doppler-gpu 0.4.8 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CHANGELOG.md +10 -0
  2. package/models/catalog.json +21 -7
  3. package/package.json +8 -1
  4. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  5. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  6. package/src/config/kernels/backward-registry.json +13 -2
  7. package/src/config/kernels/kernel-ref-digests.js +8 -4
  8. package/src/config/kernels/registry.json +1035 -2677
  9. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  10. package/src/config/schema/training-metrics.schema.js +4 -0
  11. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  12. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  13. package/src/experimental/training/attention-backward.js +123 -55
  14. package/src/experimental/training/autograd.js +101 -5
  15. package/src/experimental/training/clip.js +6 -1
  16. package/src/experimental/training/datasets/text-pairs.js +80 -8
  17. package/src/experimental/training/distillation/student-fixture.js +470 -114
  18. package/src/experimental/training/lora-pipeline.js +202 -27
  19. package/src/experimental/training/lora.js +10 -4
  20. package/src/experimental/training/optimizer.js +9 -1
  21. package/src/experimental/training/runner.js +18 -4
  22. package/src/experimental/training/workloads.js +11 -0
  23. package/src/gpu/kernels/backward/adam.js +6 -4
  24. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  25. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  26. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  27. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  28. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  29. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  30. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  31. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  32. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  33. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  34. package/src/gpu/kernels/backward/index.d.ts +3 -0
  35. package/src/gpu/kernels/backward/index.js +3 -0
  36. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  37. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  38. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  39. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  40. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  41. package/src/gpu/kernels/backward/utils.js +9 -2
  42. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  43. package/src/gpu/kernels/index.d.ts +5 -0
  44. package/src/gpu/kernels/index.js +5 -0
  45. package/src/gpu/kernels/matmul-selection.js +5 -0
  46. package/src/gpu/kernels/silu.d.ts +1 -0
  47. package/src/gpu/kernels/silu.js +19 -4
  48. package/src/inference/pipelines/text/embed.js +1 -1
  49. package/src/inference/pipelines/text/execution-v1.js +8 -2
  50. package/src/inference/pipelines/text/ffn/dense.js +2 -0
  51. package/src/inference/pipelines/text/generator.js +29 -0
  52. package/src/inference/pipelines/text/lora-apply.js +4 -4
  53. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  54. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  55. package/src/inference/tokenizers/bundled.js +105 -32
  56. package/src/rules/kernels/matmul.rules.json +1 -0
  57. package/src/tooling/node-converter.js +20 -5
  58. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  59. package/src/tooling-experimental-exports.shared.js +18 -0
  60. package/src/version.js +1 -1
@@ -0,0 +1,82 @@
1
+ // Matrix Multiplication Backward Kernel - dX = dY @ W^T with packed F16 W.
2
+ //
3
+ // dY and dX remain F32. W is stored as consecutive IEEE-754 binary16 values,
4
+ // packed two values per u32 storage word. unpack2x16float keeps this path
5
+ // available without requiring native shader-f16 arithmetic.
6
+
7
+ const MAX_TILE_SIZE: u32 = 16u;
8
+ const TILE_AREA: u32 = MAX_TILE_SIZE * MAX_TILE_SIZE;
9
+
10
+ override TILE_SIZE: u32 = 16u;
11
+
12
+ struct Uniforms {
13
+ M: u32,
14
+ N: u32,
15
+ K: u32,
16
+ alpha: f32,
17
+ transpose_b: u32,
18
+ _pad0: u32,
19
+ _pad1: u32,
20
+ _pad2: u32,
21
+ }
22
+
23
+ @group(0) @binding(0) var<uniform> u: Uniforms;
24
+ @group(0) @binding(1) var<storage, read> dY: array<f32>;
25
+ @group(0) @binding(2) var<storage, read> W: array<u32>;
26
+ @group(0) @binding(3) var<storage, read_write> dX: array<f32>;
27
+
28
+ var<workgroup> tileDY: array<f32, TILE_AREA>;
29
+ var<workgroup> tileW: array<f32, TILE_AREA>;
30
+
31
+ fn load_f16(index: u32) -> f32 {
32
+ let pair = unpack2x16float(W[index >> 1u]);
33
+ return select(pair.x, pair.y, (index & 1u) == 1u);
34
+ }
35
+
36
+ @compute @workgroup_size(TILE_SIZE, TILE_SIZE, 1)
37
+ fn main(
38
+ @builtin(global_invocation_id) global_id: vec3<u32>,
39
+ @builtin(local_invocation_id) local_id: vec3<u32>
40
+ ) {
41
+ let row = global_id.x;
42
+ let col = global_id.y;
43
+ let local_row = local_id.x;
44
+ let local_col = local_id.y;
45
+
46
+ var sum: f32 = 0.0;
47
+ let num_tiles = (u.N + TILE_SIZE - 1u) / TILE_SIZE;
48
+
49
+ for (var t: u32 = 0u; t < num_tiles; t = t + 1u) {
50
+ let tile_n = t * TILE_SIZE;
51
+ let tile_idx = local_row * TILE_SIZE + local_col;
52
+ let dy_col = tile_n + local_col;
53
+ if (row < u.M && dy_col < u.N) {
54
+ tileDY[tile_idx] = dY[row * u.N + dy_col];
55
+ } else {
56
+ tileDY[tile_idx] = 0.0;
57
+ }
58
+
59
+ let wt_row = tile_n + local_row;
60
+ if (wt_row < u.N && col < u.K) {
61
+ if (u.transpose_b == 0u) {
62
+ tileW[tile_idx] = load_f16(col * u.N + wt_row);
63
+ } else {
64
+ tileW[tile_idx] = load_f16(wt_row * u.K + col);
65
+ }
66
+ } else {
67
+ tileW[tile_idx] = 0.0;
68
+ }
69
+
70
+ workgroupBarrier();
71
+
72
+ for (var k: u32 = 0u; k < TILE_SIZE; k = k + 1u) {
73
+ sum += tileDY[local_row * TILE_SIZE + k] * tileW[k * TILE_SIZE + local_col];
74
+ }
75
+
76
+ workgroupBarrier();
77
+ }
78
+
79
+ if (row < u.M && col < u.K) {
80
+ dX[row * u.K + col] = sum * u.alpha;
81
+ }
82
+ }
@@ -9,7 +9,13 @@ import { DEFAULT_HIGH_PRECISION_EPS } from '../../../config/schema/index.js';
9
9
 
10
10
  export async function runRmsNormBackward(input, weight, gradOutput, options = {}) {
11
11
  const device = getDevice();
12
- const { numTokens, hiddenSize, eps = DEFAULT_HIGH_PRECISION_EPS, outputBuffer = null } = options;
12
+ const {
13
+ numTokens,
14
+ hiddenSize,
15
+ eps = DEFAULT_HIGH_PRECISION_EPS,
16
+ rmsNormWeightOffset = false,
17
+ outputBuffer = null,
18
+ } = options;
13
19
 
14
20
  if (!numTokens || !hiddenSize) {
15
21
  throw new Error('rmsnorm backward requires numTokens and hiddenSize');
@@ -22,7 +28,12 @@ export async function runRmsNormBackward(input, weight, gradOutput, options = {}
22
28
  const outputSize = numTokens * hiddenSize * 4;
23
29
  const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'rmsnorm_backward_output');
24
30
 
25
- const pipeline = await createPipeline('rmsnorm_backward', 'default');
31
+ const pipeline = await createPipeline(
32
+ 'rmsnorm_backward',
33
+ 'default',
34
+ null,
35
+ { RMS_NORM_OFFSET: rmsNormWeightOffset }
36
+ );
26
37
  const uniformBuffer = createUniformBufferWithView(
27
38
  'rmsnorm_backward_uniforms',
28
39
  16,
@@ -57,7 +68,13 @@ export async function runRmsNormBackward(input, weight, gradOutput, options = {}
57
68
 
58
69
  export async function recordRmsNormBackward(recorder, input, weight, gradOutput, options = {}) {
59
70
  const device = recorder.device;
60
- const { numTokens, hiddenSize, eps = DEFAULT_HIGH_PRECISION_EPS, outputBuffer = null } = options;
71
+ const {
72
+ numTokens,
73
+ hiddenSize,
74
+ eps = DEFAULT_HIGH_PRECISION_EPS,
75
+ rmsNormWeightOffset = false,
76
+ outputBuffer = null,
77
+ } = options;
61
78
 
62
79
  if (!numTokens || !hiddenSize) {
63
80
  throw new Error('rmsnorm backward requires numTokens and hiddenSize');
@@ -70,7 +87,12 @@ export async function recordRmsNormBackward(recorder, input, weight, gradOutput,
70
87
  const outputSize = numTokens * hiddenSize * 4;
71
88
  const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'rmsnorm_backward_output');
72
89
 
73
- const pipeline = await createPipeline('rmsnorm_backward', 'default');
90
+ const pipeline = await createPipeline(
91
+ 'rmsnorm_backward',
92
+ 'default',
93
+ null,
94
+ { RMS_NORM_OFFSET: rmsNormWeightOffset }
95
+ );
74
96
  const uniformBuffer = createUniformBufferWithView(
75
97
  'rmsnorm_backward_uniforms',
76
98
  16,
@@ -7,6 +7,7 @@
7
7
  * Uses shared-memory reduction for per-token stats.
8
8
  */
9
9
  override WORKGROUP_SIZE: u32 = 256u;
10
+ override RMS_NORM_OFFSET: bool = false;
10
11
  const MAX_WORKGROUP_SIZE: u32 = 256u;
11
12
 
12
13
  struct Uniforms {
@@ -44,7 +45,8 @@ fn main(
44
45
 
45
46
  for (var i: u32 = local_id; i < hidden_size; i = i + WORKGROUP_SIZE) {
46
47
  let x = input[base + i];
47
- let g = grad_output[base + i] * weight[i];
48
+ let effective_weight = select(weight[i], 1.0 + weight[i], RMS_NORM_OFFSET);
49
+ let g = grad_output[base + i] * effective_weight;
48
50
  sum_sq = sum_sq + x * x;
49
51
  sum_gx = sum_gx + g * x;
50
52
  }
@@ -72,7 +74,8 @@ fn main(
72
74
 
73
75
  for (var i: u32 = local_id; i < hidden_size; i = i + WORKGROUP_SIZE) {
74
76
  let x = input[base + i];
75
- let g = grad_output[base + i] * weight[i];
77
+ let effective_weight = select(weight[i], 1.0 + weight[i], RMS_NORM_OFFSET);
78
+ let g = grad_output[base + i] * effective_weight;
76
79
  output[base + i] = g * inv_rms - x * coeff;
77
80
  }
78
- }
81
+ }
@@ -56,6 +56,16 @@ export declare function createBackwardKernel(
56
56
  record: (recorder: CommandRecorder, ...args: unknown[]) => Promise<Tensor>;
57
57
  };
58
58
 
59
+ export declare function resolveMatmulBackwardDxVariant(
60
+ weight: { dtype?: string | null } | null
61
+ ): 'default' | 'f16_weight';
62
+
63
+ export interface MatmulBackwardDxOptions {
64
+ alpha?: number;
65
+ transposeB?: boolean;
66
+ outputBuffer?: GPUBuffer | null;
67
+ }
68
+
59
69
  /**
60
70
  * Matmul-backward gradient w.r.t. input (dX = dY · Wᵀ). Submit + wait.
61
71
  */
@@ -65,7 +75,7 @@ export declare function runMatmulBackwardDx(
65
75
  M: number,
66
76
  K: number,
67
77
  N: number,
68
- options?: { outputBuffer?: GPUBuffer | null }
78
+ options?: MatmulBackwardDxOptions
69
79
  ): Promise<Tensor>;
70
80
 
71
81
  /** Record runMatmulBackwardDx onto a CommandRecorder. */
@@ -76,5 +86,5 @@ export declare function recordMatmulBackwardDx(
76
86
  M: number,
77
87
  K: number,
78
88
  N: number,
79
- options?: { outputBuffer?: GPUBuffer | null }
89
+ options?: MatmulBackwardDxOptions
80
90
  ): Promise<Tensor>;
@@ -192,13 +192,20 @@ export function createBackwardKernel(opName, spec) {
192
192
  return { run, record };
193
193
  }
194
194
 
195
+ export function resolveMatmulBackwardDxVariant(weight) {
196
+ const dtype = String(weight?.dtype || 'f32').toLowerCase();
197
+ if (dtype === 'f16') return 'f16_weight';
198
+ if (dtype === 'f32') return 'default';
199
+ throw new Error(`matmul backward dX does not support weight dtype "${dtype}".`);
200
+ }
201
+
195
202
  export async function runMatmulBackwardDx(dY, W, M, K, N, options = {}) {
196
203
  const { alpha = 1.0, transposeB = false, outputBuffer = null } = options;
197
204
  const device = getDevice();
198
205
  const outputSize = M * K * 4;
199
206
  const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
200
207
 
201
- const pipeline = await createPipeline('matmul_backward', 'default');
208
+ const pipeline = await createPipeline('matmul_backward', resolveMatmulBackwardDxVariant(W));
202
209
 
203
210
  const uniformBuffer = createUniformBufferWithView(
204
211
  'matmul_backward_uniforms',
@@ -244,7 +251,7 @@ export async function recordMatmulBackwardDx(recorder, dY, W, M, K, N, options =
244
251
  const outputSize = M * K * 4;
245
252
  const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
246
253
 
247
- const pipeline = await createPipeline('matmul_backward', 'default');
254
+ const pipeline = await createPipeline('matmul_backward', resolveMatmulBackwardDxVariant(W));
248
255
 
249
256
  const uniformBuffer = createUniformBufferWithView(
250
257
  'matmul_backward_uniforms',
@@ -4,6 +4,7 @@
4
4
  * Cast F16 to F32 Kernel
5
5
  *
6
6
  * Converts a buffer of f16 values to f32.
7
+ * Supports 2D dispatch for tensors larger than one workgroup dimension.
7
8
  */
8
9
 
9
10
  enable f16;
@@ -22,8 +23,11 @@ struct Uniforms {
22
23
  @group(0) @binding(2) var<storage, read_write> output: array<f32>;
23
24
 
24
25
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
25
- fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
26
- let idx = gid.x;
26
+ fn main(
27
+ @builtin(global_invocation_id) gid: vec3<u32>,
28
+ @builtin(num_workgroups) num_wg: vec3<u32>
29
+ ) {
30
+ let idx = gid.x + gid.y * num_wg.x * WORKGROUP_SIZE;
27
31
  if (idx >= u.count) {
28
32
  return;
29
33
  }
@@ -460,6 +460,11 @@ export {
460
460
  recordSoftmaxBackward,
461
461
  runRmsNormBackward,
462
462
  recordRmsNormBackward,
463
+ runGatedRmsNormBackward,
464
+ recordGatedRmsNormBackward,
465
+ runCausalConv1dSiluBackward,
466
+ recordCausalConv1dSiluBackward,
467
+ runGatedDeltaRecurrentBackward,
463
468
  runLayerNormBackward,
464
469
  recordLayerNormBackward,
465
470
  runAttentionBackward,
@@ -370,6 +370,11 @@ export {
370
370
  recordSoftmaxBackward,
371
371
  runRmsNormBackward,
372
372
  recordRmsNormBackward,
373
+ runGatedRmsNormBackward,
374
+ recordGatedRmsNormBackward,
375
+ runCausalConv1dSiluBackward,
376
+ recordCausalConv1dSiluBackward,
377
+ runGatedDeltaRecurrentBackward,
373
378
  runLayerNormBackward,
374
379
  recordLayerNormBackward,
375
380
  runAttentionBackward,
@@ -460,6 +460,11 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
460
460
  variant: pathVariant,
461
461
  reason: 'path_override_quantized_fallthrough',
462
462
  });
463
+ } else if (bDtype === 'f16' && isQ4KFusedVariant(pathVariant)) {
464
+ logKernelSelectionOnce('matmul', {
465
+ variant: pathVariant,
466
+ reason: 'path_override_f16_fallthrough',
467
+ });
463
468
  } else {
464
469
  throw new Error(`[Matmul] Path variant "${pathVariant}" rejected for role=${options.role ?? '?'} layerIdx=${options.layerIdx ?? '?'} phase=${phase} M=${M} K=${K} aDtype=${aDtype} bDtype=${bDtype} outDtype=${requestedOutputDtype}`);
465
470
  }
@@ -16,6 +16,7 @@ export interface SiLUOptions extends OutputBufferOptions {
16
16
  size?: number | null;
17
17
  gate?: Tensor | null;
18
18
  gateActivation?: 'silu' | 'sigmoid';
19
+ /** Required when gate is present; ungated SiLU defaults to silu. */
19
20
  inputActivation?: 'silu' | 'identity';
20
21
  useVec4?: boolean;
21
22
  biasOffset?: number;
@@ -49,6 +49,19 @@ function resolveSwigluLimit(value, context) {
49
49
  return value;
50
50
  }
51
51
 
52
+ function resolveInputActivation(value, hasGate) {
53
+ if (value === undefined) {
54
+ if (hasGate) {
55
+ throw new Error('Gated SiLU requires an explicit inputActivation (identity or silu).');
56
+ }
57
+ return 'silu';
58
+ }
59
+ if (value !== 'identity' && value !== 'silu') {
60
+ throw new Error(`Unsupported SiLU inputActivation: ${value}`);
61
+ }
62
+ return value;
63
+ }
64
+
52
65
 
53
66
  function createSiLUBindGroupEntries(uniformBuffer, input, output, gate) {
54
67
  const gateBuffer = gate?.buffer ?? input.buffer;
@@ -91,9 +104,10 @@ export async function runSiLU(
91
104
  useVec4 = false,
92
105
  swigluLimit,
93
106
  gateActivation = 'silu',
94
- inputActivation = 'silu',
107
+ inputActivation,
95
108
  } = options;
96
109
  const resolvedSwigluLimit = resolveSwigluLimit(swigluLimit, 'SiLU');
110
+ const resolvedInputActivation = resolveInputActivation(inputActivation, Boolean(gate));
97
111
 
98
112
  const isF16 = canUseF16(input);
99
113
  const bytesPerElement = dtypeBytes(input.dtype);
@@ -109,7 +123,7 @@ export async function runSiLU(
109
123
  const constants = {
110
124
  ...(overrides || {}),
111
125
  ...(gate && gateActivation === 'sigmoid' ? { GATE_USE_SIGMOID: true } : {}),
112
- ...(inputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
126
+ ...(resolvedInputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
113
127
  };
114
128
  const pipeline = await getPipelineFast('silu', variant, null, constants);
115
129
 
@@ -344,9 +358,10 @@ export async function recordSiLU(
344
358
  outputBuffer = null,
345
359
  swigluLimit,
346
360
  gateActivation = 'silu',
347
- inputActivation = 'silu',
361
+ inputActivation,
348
362
  } = options;
349
363
  const resolvedSwigluLimit = resolveSwigluLimit(swigluLimit, 'SiLU');
364
+ const resolvedInputActivation = resolveInputActivation(inputActivation, Boolean(gate));
350
365
 
351
366
  const isF16 = canUseF16(input);
352
367
  const bytesPerElement = dtypeBytes(input.dtype);
@@ -362,7 +377,7 @@ export async function recordSiLU(
362
377
  const constants = {
363
378
  ...(overrides || {}),
364
379
  ...(gate && gateActivation === 'sigmoid' ? { GATE_USE_SIGMOID: true } : {}),
365
- ...(inputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
380
+ ...(resolvedInputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
366
381
  };
367
382
  const pipeline = await getPipelineFast('silu', variant, null, constants);
368
383
 
@@ -63,7 +63,7 @@ export function decodeRangeChunkIntoOutput(bytes, sourceDtype, output, dstOffset
63
63
  }
64
64
  }
65
65
 
66
- function resolveEmbeddingScale(config, hiddenSize) {
66
+ export function resolveEmbeddingScale(config, hiddenSize) {
67
67
  const embeddingScale = config.embeddingScale;
68
68
  const scaleEmbeddings = config.scaleEmbeddings;
69
69
  if (embeddingScale === undefined) {
@@ -530,10 +530,14 @@ function collectCoveredLayers(entries, targetLayers, predicate) {
530
530
  return covered;
531
531
  }
532
532
 
533
- function assertHybridLinearProjectionIsolation(execution, layerTypes, modelId) {
533
+ function assertHybridLinearProjectionIsolation(execution, layerTypes, modelId, weightDtype) {
534
534
  if (!Array.isArray(layerTypes) || layerTypes.length === 0) {
535
535
  return;
536
536
  }
537
+ const normalizedWeightDtype = String(weightDtype || '').trim().toLowerCase();
538
+ if (normalizedWeightDtype === 'f16' || normalizedWeightDtype === 'f32') {
539
+ return;
540
+ }
537
541
 
538
542
  const linearLayers = layerTypes
539
543
  .map((layerType, layerIdx) => ({ layerType, layerIdx }))
@@ -730,6 +734,7 @@ export function compileExecutionV1(options = {}) {
730
734
  const modelId = options.modelId ?? 'model';
731
735
  const numLayers = options.numLayers ?? 0;
732
736
  const headDim = Number.isFinite(options.headDim) ? Math.floor(options.headDim) : null;
737
+ const weightDtype = options.weightDtype ?? null;
733
738
  const hasCapabilityProof = options.capabilities && typeof options.capabilities === 'object';
734
739
  const hasUseGPUOption = hasOwnProperty(options, 'useGPU');
735
740
  const useGPU = hasUseGPUOption
@@ -861,7 +866,7 @@ export function compileExecutionV1(options = {}) {
861
866
 
862
867
  }
863
868
 
864
- assertHybridLinearProjectionIsolation(execution, layerTypes, modelId);
869
+ assertHybridLinearProjectionIsolation(execution, layerTypes, modelId, weightDtype);
865
870
 
866
871
  // Transformed graphs have null digests for derived kernels — skip digest
867
872
  // validation since the original graph was already validated above.
@@ -1054,6 +1059,7 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
1054
1059
  modelId: manifest.modelId ?? options.modelId,
1055
1060
  numLayers: options.numLayers ?? manifest.architecture?.numLayers ?? 0,
1056
1061
  headDim: options.headDim ?? manifest.architecture?.headDim ?? null,
1062
+ weightDtype: manifest.quantizationInfo?.weights ?? null,
1057
1063
  capabilities: options.capabilities ?? null,
1058
1064
  platform: options.platform ?? null,
1059
1065
  runtimeSession: hasExplicitRuntimeOverrides
@@ -1257,6 +1257,7 @@ export async function runDenseFFNGPU(
1257
1257
  const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
1258
1258
  size: numTokens * intermediateSize,
1259
1259
  gate: gateOutput,
1260
+ inputActivation: 'identity',
1260
1261
  swigluLimit,
1261
1262
  label: `L${layerIdx}.ffn_activation`,
1262
1263
  layerIdx,
@@ -1646,6 +1647,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
1646
1647
  activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
1647
1648
  size: numTokens * intermediateSize,
1648
1649
  gate: gateOutput,
1650
+ inputActivation: 'identity',
1649
1651
  swigluLimit,
1650
1652
  }, recorder);
1651
1653
 
@@ -2277,6 +2277,27 @@ export class PipelineGenerator {
2277
2277
  this.#state.stats.gpuTimePrefillMs = undefined;
2278
2278
  this.#state.stats.prefillProfileSteps = [];
2279
2279
  const opts = resolvePrefillOptions(this.#state, options);
2280
+ const diagnosticsEnabled = options?.diagnostics?.enabled === true
2281
+ || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
2282
+ if (diagnosticsEnabled) {
2283
+ const captureConfig = {
2284
+ ...createDefaultCaptureConfig(),
2285
+ enabled: true,
2286
+ defaultLevel: CAPTURE_LEVELS.SLICE,
2287
+ ...(options?.diagnostics?.captureConfig ?? {}),
2288
+ };
2289
+ validateCaptureConfig(captureConfig);
2290
+ this.#state.operatorDiagnostics = {
2291
+ enabled: true,
2292
+ captureConfig,
2293
+ emitter: new OperatorEventEmitter({
2294
+ modelHash: this.#state.manifest?.modelId ?? null,
2295
+ runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
2296
+ executionPlanHash: opts.executionPlan?.id ?? null,
2297
+ }),
2298
+ tsirFixture: null,
2299
+ };
2300
+ }
2280
2301
  try {
2281
2302
  const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
2282
2303
 
@@ -2298,6 +2319,14 @@ export class PipelineGenerator {
2298
2319
  };
2299
2320
  } finally {
2300
2321
  this._closeFinitenessFallbackWindow(opts);
2322
+ this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
2323
+ ? {
2324
+ enabled: true,
2325
+ timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
2326
+ recordCount: this.#state.operatorDiagnostics.emitter.length,
2327
+ }
2328
+ : null;
2329
+ this.#state.operatorDiagnostics = null;
2301
2330
  }
2302
2331
  }
2303
2332
 
@@ -46,12 +46,12 @@ export async function applyLoRA(input, baseOutput, lora, dims, getWeightBuffer,
46
46
  let scaled = null;
47
47
  try {
48
48
  loraIntermediate = recorder
49
- ? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath, outputDtype: intermediateDtype })
50
- : await runMatmul(input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath, outputDtype: intermediateDtype });
49
+ ? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: false, role: 'lora_a', kernelPath, outputDtype: intermediateDtype })
50
+ : await runMatmul(input, aBuf, M, rank, K, { transposeB: false, role: 'lora_a', kernelPath, outputDtype: intermediateDtype });
51
51
 
52
52
  loraOutput = recorder
53
- ? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath, outputDtype })
54
- : await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath, outputDtype });
53
+ ? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: false, role: 'lora_b', kernelPath, outputDtype })
54
+ : await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: false, role: 'lora_b', kernelPath, outputDtype });
55
55
 
56
56
  scaled = recorder
57
57
  ? await recordScale(recorder, loraOutput, lora.scale, { outputBuffer: null })
@@ -122,6 +122,7 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
122
122
  activatedOutput = await activationFn(upOutput, {
123
123
  size: numTokens * intermediateSize,
124
124
  gate: gateOutput,
125
+ inputActivation: 'identity',
125
126
  swigluLimit,
126
127
  });
127
128
 
@@ -1117,6 +1117,7 @@ async function runMixtralExpert(
1117
1117
  const activated = await activationFn(upOut, {
1118
1118
  size: count * intermediateSize,
1119
1119
  gate: gateOut,
1120
+ inputActivation: 'identity',
1120
1121
  swigluLimit,
1121
1122
  });
1122
1123
  releaseBuffer(gateOut.buffer);