doppler-gpu 0.4.8 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/README.md +62 -12
  3. package/models/catalog.json +21 -7
  4. package/package.json +25 -1
  5. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  6. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  7. package/src/config/kernels/backward-registry.json +13 -2
  8. package/src/config/kernels/kernel-ref-digests.js +8 -4
  9. package/src/config/kernels/registry.json +1035 -2677
  10. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  11. package/src/config/schema/training-metrics.schema.js +4 -0
  12. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  13. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  14. package/src/experimental/training/attention-backward.js +123 -55
  15. package/src/experimental/training/autograd.js +101 -5
  16. package/src/experimental/training/clip.js +6 -1
  17. package/src/experimental/training/datasets/text-pairs.js +80 -8
  18. package/src/experimental/training/distillation/student-fixture.js +470 -114
  19. package/src/experimental/training/lora-pipeline.js +202 -27
  20. package/src/experimental/training/lora.js +10 -4
  21. package/src/experimental/training/optimizer.js +9 -1
  22. package/src/experimental/training/runner.js +18 -4
  23. package/src/experimental/training/workloads.js +11 -0
  24. package/src/gpu/device.js +100 -32
  25. package/src/gpu/kernels/backward/adam.js +6 -4
  26. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  27. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  28. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  29. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  30. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  31. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  32. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  33. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  34. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  35. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  36. package/src/gpu/kernels/backward/index.d.ts +3 -0
  37. package/src/gpu/kernels/backward/index.js +3 -0
  38. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  39. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  40. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  41. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  42. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  43. package/src/gpu/kernels/backward/utils.js +9 -2
  44. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  45. package/src/gpu/kernels/index.d.ts +5 -0
  46. package/src/gpu/kernels/index.js +5 -0
  47. package/src/gpu/kernels/matmul-selection.js +5 -0
  48. package/src/gpu/kernels/silu.d.ts +1 -0
  49. package/src/gpu/kernels/silu.js +19 -4
  50. package/src/inference/pipelines/text/embed.js +1 -1
  51. package/src/inference/pipelines/text/execution-v1.js +8 -2
  52. package/src/inference/pipelines/text/ffn/dense.js +2 -0
  53. package/src/inference/pipelines/text/generator.js +29 -0
  54. package/src/inference/pipelines/text/lora-apply.js +4 -4
  55. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  56. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  57. package/src/inference/tokenizers/bundled.js +105 -32
  58. package/src/rules/kernels/matmul.rules.json +1 -0
  59. package/src/tooling/node-converter.js +20 -5
  60. package/src/tooling/runtime-optimization.d.ts +109 -0
  61. package/src/tooling/runtime-optimization.js +872 -0
  62. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  63. package/src/tooling-experimental-exports.shared.js +18 -0
  64. package/src/tooling-exports.shared.d.ts +19 -0
  65. package/src/tooling-exports.shared.js +11 -0
  66. package/src/version.js +1 -1
@@ -4,6 +4,7 @@
4
4
  * Cast F16 to F32 Kernel
5
5
  *
6
6
  * Converts a buffer of f16 values to f32.
7
+ * Supports 2D dispatch for tensors larger than one workgroup dimension.
7
8
  */
8
9
 
9
10
  enable f16;
@@ -22,8 +23,11 @@ struct Uniforms {
22
23
  @group(0) @binding(2) var<storage, read_write> output: array<f32>;
23
24
 
24
25
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
25
- fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
26
- let idx = gid.x;
26
+ fn main(
27
+ @builtin(global_invocation_id) gid: vec3<u32>,
28
+ @builtin(num_workgroups) num_wg: vec3<u32>
29
+ ) {
30
+ let idx = gid.x + gid.y * num_wg.x * WORKGROUP_SIZE;
27
31
  if (idx >= u.count) {
28
32
  return;
29
33
  }
@@ -460,6 +460,11 @@ export {
460
460
  recordSoftmaxBackward,
461
461
  runRmsNormBackward,
462
462
  recordRmsNormBackward,
463
+ runGatedRmsNormBackward,
464
+ recordGatedRmsNormBackward,
465
+ runCausalConv1dSiluBackward,
466
+ recordCausalConv1dSiluBackward,
467
+ runGatedDeltaRecurrentBackward,
463
468
  runLayerNormBackward,
464
469
  recordLayerNormBackward,
465
470
  runAttentionBackward,
@@ -370,6 +370,11 @@ export {
370
370
  recordSoftmaxBackward,
371
371
  runRmsNormBackward,
372
372
  recordRmsNormBackward,
373
+ runGatedRmsNormBackward,
374
+ recordGatedRmsNormBackward,
375
+ runCausalConv1dSiluBackward,
376
+ recordCausalConv1dSiluBackward,
377
+ runGatedDeltaRecurrentBackward,
373
378
  runLayerNormBackward,
374
379
  recordLayerNormBackward,
375
380
  runAttentionBackward,
@@ -460,6 +460,11 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
460
460
  variant: pathVariant,
461
461
  reason: 'path_override_quantized_fallthrough',
462
462
  });
463
+ } else if (bDtype === 'f16' && isQ4KFusedVariant(pathVariant)) {
464
+ logKernelSelectionOnce('matmul', {
465
+ variant: pathVariant,
466
+ reason: 'path_override_f16_fallthrough',
467
+ });
463
468
  } else {
464
469
  throw new Error(`[Matmul] Path variant "${pathVariant}" rejected for role=${options.role ?? '?'} layerIdx=${options.layerIdx ?? '?'} phase=${phase} M=${M} K=${K} aDtype=${aDtype} bDtype=${bDtype} outDtype=${requestedOutputDtype}`);
465
470
  }
@@ -16,6 +16,7 @@ export interface SiLUOptions extends OutputBufferOptions {
16
16
  size?: number | null;
17
17
  gate?: Tensor | null;
18
18
  gateActivation?: 'silu' | 'sigmoid';
19
+ /** Required when gate is present; ungated SiLU defaults to silu. */
19
20
  inputActivation?: 'silu' | 'identity';
20
21
  useVec4?: boolean;
21
22
  biasOffset?: number;
@@ -49,6 +49,19 @@ function resolveSwigluLimit(value, context) {
49
49
  return value;
50
50
  }
51
51
 
52
+ function resolveInputActivation(value, hasGate) {
53
+ if (value === undefined) {
54
+ if (hasGate) {
55
+ throw new Error('Gated SiLU requires an explicit inputActivation (identity or silu).');
56
+ }
57
+ return 'silu';
58
+ }
59
+ if (value !== 'identity' && value !== 'silu') {
60
+ throw new Error(`Unsupported SiLU inputActivation: ${value}`);
61
+ }
62
+ return value;
63
+ }
64
+
52
65
 
53
66
  function createSiLUBindGroupEntries(uniformBuffer, input, output, gate) {
54
67
  const gateBuffer = gate?.buffer ?? input.buffer;
@@ -91,9 +104,10 @@ export async function runSiLU(
91
104
  useVec4 = false,
92
105
  swigluLimit,
93
106
  gateActivation = 'silu',
94
- inputActivation = 'silu',
107
+ inputActivation,
95
108
  } = options;
96
109
  const resolvedSwigluLimit = resolveSwigluLimit(swigluLimit, 'SiLU');
110
+ const resolvedInputActivation = resolveInputActivation(inputActivation, Boolean(gate));
97
111
 
98
112
  const isF16 = canUseF16(input);
99
113
  const bytesPerElement = dtypeBytes(input.dtype);
@@ -109,7 +123,7 @@ export async function runSiLU(
109
123
  const constants = {
110
124
  ...(overrides || {}),
111
125
  ...(gate && gateActivation === 'sigmoid' ? { GATE_USE_SIGMOID: true } : {}),
112
- ...(inputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
126
+ ...(resolvedInputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
113
127
  };
114
128
  const pipeline = await getPipelineFast('silu', variant, null, constants);
115
129
 
@@ -344,9 +358,10 @@ export async function recordSiLU(
344
358
  outputBuffer = null,
345
359
  swigluLimit,
346
360
  gateActivation = 'silu',
347
- inputActivation = 'silu',
361
+ inputActivation,
348
362
  } = options;
349
363
  const resolvedSwigluLimit = resolveSwigluLimit(swigluLimit, 'SiLU');
364
+ const resolvedInputActivation = resolveInputActivation(inputActivation, Boolean(gate));
350
365
 
351
366
  const isF16 = canUseF16(input);
352
367
  const bytesPerElement = dtypeBytes(input.dtype);
@@ -362,7 +377,7 @@ export async function recordSiLU(
362
377
  const constants = {
363
378
  ...(overrides || {}),
364
379
  ...(gate && gateActivation === 'sigmoid' ? { GATE_USE_SIGMOID: true } : {}),
365
- ...(inputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
380
+ ...(resolvedInputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
366
381
  };
367
382
  const pipeline = await getPipelineFast('silu', variant, null, constants);
368
383
 
@@ -63,7 +63,7 @@ export function decodeRangeChunkIntoOutput(bytes, sourceDtype, output, dstOffset
63
63
  }
64
64
  }
65
65
 
66
- function resolveEmbeddingScale(config, hiddenSize) {
66
+ export function resolveEmbeddingScale(config, hiddenSize) {
67
67
  const embeddingScale = config.embeddingScale;
68
68
  const scaleEmbeddings = config.scaleEmbeddings;
69
69
  if (embeddingScale === undefined) {
@@ -530,10 +530,14 @@ function collectCoveredLayers(entries, targetLayers, predicate) {
530
530
  return covered;
531
531
  }
532
532
 
533
- function assertHybridLinearProjectionIsolation(execution, layerTypes, modelId) {
533
+ function assertHybridLinearProjectionIsolation(execution, layerTypes, modelId, weightDtype) {
534
534
  if (!Array.isArray(layerTypes) || layerTypes.length === 0) {
535
535
  return;
536
536
  }
537
+ const normalizedWeightDtype = String(weightDtype || '').trim().toLowerCase();
538
+ if (normalizedWeightDtype === 'f16' || normalizedWeightDtype === 'f32') {
539
+ return;
540
+ }
537
541
 
538
542
  const linearLayers = layerTypes
539
543
  .map((layerType, layerIdx) => ({ layerType, layerIdx }))
@@ -730,6 +734,7 @@ export function compileExecutionV1(options = {}) {
730
734
  const modelId = options.modelId ?? 'model';
731
735
  const numLayers = options.numLayers ?? 0;
732
736
  const headDim = Number.isFinite(options.headDim) ? Math.floor(options.headDim) : null;
737
+ const weightDtype = options.weightDtype ?? null;
733
738
  const hasCapabilityProof = options.capabilities && typeof options.capabilities === 'object';
734
739
  const hasUseGPUOption = hasOwnProperty(options, 'useGPU');
735
740
  const useGPU = hasUseGPUOption
@@ -861,7 +866,7 @@ export function compileExecutionV1(options = {}) {
861
866
 
862
867
  }
863
868
 
864
- assertHybridLinearProjectionIsolation(execution, layerTypes, modelId);
869
+ assertHybridLinearProjectionIsolation(execution, layerTypes, modelId, weightDtype);
865
870
 
866
871
  // Transformed graphs have null digests for derived kernels — skip digest
867
872
  // validation since the original graph was already validated above.
@@ -1054,6 +1059,7 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
1054
1059
  modelId: manifest.modelId ?? options.modelId,
1055
1060
  numLayers: options.numLayers ?? manifest.architecture?.numLayers ?? 0,
1056
1061
  headDim: options.headDim ?? manifest.architecture?.headDim ?? null,
1062
+ weightDtype: manifest.quantizationInfo?.weights ?? null,
1057
1063
  capabilities: options.capabilities ?? null,
1058
1064
  platform: options.platform ?? null,
1059
1065
  runtimeSession: hasExplicitRuntimeOverrides
@@ -1257,6 +1257,7 @@ export async function runDenseFFNGPU(
1257
1257
  const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
1258
1258
  size: numTokens * intermediateSize,
1259
1259
  gate: gateOutput,
1260
+ inputActivation: 'identity',
1260
1261
  swigluLimit,
1261
1262
  label: `L${layerIdx}.ffn_activation`,
1262
1263
  layerIdx,
@@ -1646,6 +1647,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
1646
1647
  activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
1647
1648
  size: numTokens * intermediateSize,
1648
1649
  gate: gateOutput,
1650
+ inputActivation: 'identity',
1649
1651
  swigluLimit,
1650
1652
  }, recorder);
1651
1653
 
@@ -2277,6 +2277,27 @@ export class PipelineGenerator {
2277
2277
  this.#state.stats.gpuTimePrefillMs = undefined;
2278
2278
  this.#state.stats.prefillProfileSteps = [];
2279
2279
  const opts = resolvePrefillOptions(this.#state, options);
2280
+ const diagnosticsEnabled = options?.diagnostics?.enabled === true
2281
+ || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
2282
+ if (diagnosticsEnabled) {
2283
+ const captureConfig = {
2284
+ ...createDefaultCaptureConfig(),
2285
+ enabled: true,
2286
+ defaultLevel: CAPTURE_LEVELS.SLICE,
2287
+ ...(options?.diagnostics?.captureConfig ?? {}),
2288
+ };
2289
+ validateCaptureConfig(captureConfig);
2290
+ this.#state.operatorDiagnostics = {
2291
+ enabled: true,
2292
+ captureConfig,
2293
+ emitter: new OperatorEventEmitter({
2294
+ modelHash: this.#state.manifest?.modelId ?? null,
2295
+ runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
2296
+ executionPlanHash: opts.executionPlan?.id ?? null,
2297
+ }),
2298
+ tsirFixture: null,
2299
+ };
2300
+ }
2280
2301
  try {
2281
2302
  const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
2282
2303
 
@@ -2298,6 +2319,14 @@ export class PipelineGenerator {
2298
2319
  };
2299
2320
  } finally {
2300
2321
  this._closeFinitenessFallbackWindow(opts);
2322
+ this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
2323
+ ? {
2324
+ enabled: true,
2325
+ timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
2326
+ recordCount: this.#state.operatorDiagnostics.emitter.length,
2327
+ }
2328
+ : null;
2329
+ this.#state.operatorDiagnostics = null;
2301
2330
  }
2302
2331
  }
2303
2332
 
@@ -46,12 +46,12 @@ export async function applyLoRA(input, baseOutput, lora, dims, getWeightBuffer,
46
46
  let scaled = null;
47
47
  try {
48
48
  loraIntermediate = recorder
49
- ? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath, outputDtype: intermediateDtype })
50
- : await runMatmul(input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath, outputDtype: intermediateDtype });
49
+ ? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: false, role: 'lora_a', kernelPath, outputDtype: intermediateDtype })
50
+ : await runMatmul(input, aBuf, M, rank, K, { transposeB: false, role: 'lora_a', kernelPath, outputDtype: intermediateDtype });
51
51
 
52
52
  loraOutput = recorder
53
- ? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath, outputDtype })
54
- : await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath, outputDtype });
53
+ ? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: false, role: 'lora_b', kernelPath, outputDtype })
54
+ : await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: false, role: 'lora_b', kernelPath, outputDtype });
55
55
 
56
56
  scaled = recorder
57
57
  ? await recordScale(recorder, loraOutput, lora.scale, { outputBuffer: null })
@@ -122,6 +122,7 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
122
122
  activatedOutput = await activationFn(upOutput, {
123
123
  size: numTokens * intermediateSize,
124
124
  gate: gateOutput,
125
+ inputActivation: 'identity',
125
126
  swigluLimit,
126
127
  });
127
128
 
@@ -1117,6 +1117,7 @@ async function runMixtralExpert(
1117
1117
  const activated = await activationFn(upOut, {
1118
1118
  size: count * intermediateSize,
1119
1119
  gate: gateOut,
1120
+ inputActivation: 'identity',
1120
1121
  swigluLimit,
1121
1122
  });
1122
1123
  releaseBuffer(gateOut.buffer);
@@ -95,6 +95,67 @@ function resolveByteLevelPretokenizerConfig(preTokenizer) {
95
95
  };
96
96
  }
97
97
 
98
+ function normalizeInlineCaseInsensitiveGroups(pattern) {
99
+ let source = String(pattern || '');
100
+ let caseInsensitive = false;
101
+ let start = source.indexOf('(?i:');
102
+ while (start !== -1) {
103
+ let depth = 1;
104
+ let escaped = false;
105
+ let end = start + 4;
106
+ for (; end < source.length; end++) {
107
+ const char = source[end];
108
+ if (escaped) {
109
+ escaped = false;
110
+ continue;
111
+ }
112
+ if (char === '\\') {
113
+ escaped = true;
114
+ continue;
115
+ }
116
+ if (char === '(') depth += 1;
117
+ else if (char === ')') {
118
+ depth -= 1;
119
+ if (depth === 0) break;
120
+ }
121
+ }
122
+ if (depth !== 0) {
123
+ throw new Error('[Tokenizer] Unterminated inline case-insensitive regex group.');
124
+ }
125
+ source = `${source.slice(0, start)}(?:${source.slice(start + 4, end)})${source.slice(end + 1)}`;
126
+ caseInsensitive = true;
127
+ start = source.indexOf('(?i:', start + 3);
128
+ }
129
+ return { source, caseInsensitive };
130
+ }
131
+
132
+ function resolveIsolatedSplitPretokenizer(preTokenizer) {
133
+ if (!preTokenizer || typeof preTokenizer !== 'object') return null;
134
+ if (preTokenizer.type === 'Sequence' && Array.isArray(preTokenizer.pretokenizers)) {
135
+ for (const entry of preTokenizer.pretokenizers) {
136
+ const resolved = resolveIsolatedSplitPretokenizer(entry);
137
+ if (resolved) return resolved;
138
+ }
139
+ return null;
140
+ }
141
+ if (
142
+ preTokenizer.type !== 'Split'
143
+ || preTokenizer.behavior !== 'Isolated'
144
+ || preTokenizer.invert === true
145
+ || typeof preTokenizer.pattern?.Regex !== 'string'
146
+ ) {
147
+ return null;
148
+ }
149
+ const normalized = normalizeInlineCaseInsensitiveGroups(preTokenizer.pattern.Regex);
150
+ const flags = `gu${normalized.caseInsensitive ? 'i' : ''}`;
151
+ try {
152
+ new RegExp(normalized.source, flags);
153
+ } catch (error) {
154
+ throw new Error(`[Tokenizer] Unsupported split pre-tokenizer regex: ${error.message}`);
155
+ }
156
+ return { source: normalized.source, flags };
157
+ }
158
+
98
159
  function hexNibble(code) {
99
160
  if (code >= 48 && code <= 57) return code - 48;
100
161
  if (code >= 65 && code <= 70) return code - 55;
@@ -388,6 +449,8 @@ export class BundledTokenizer extends BaseTokenizer {
388
449
 
389
450
  #useByteLevelEncoding = false;
390
451
 
452
+ #splitPretokenizer = null;
453
+
391
454
 
392
455
  constructor(config = {}) {
393
456
  // BundledTokenizer gets vocabSize from load(), so defer validation
@@ -412,6 +475,7 @@ export class BundledTokenizer extends BaseTokenizer {
412
475
  this.#byteDecoder = null;
413
476
  this.#byteEncoder = null;
414
477
  this.#useByteLevelEncoding = false;
478
+ this.#splitPretokenizer = null;
415
479
  this.vocabSize = 0;
416
480
  }
417
481
 
@@ -574,6 +638,7 @@ export class BundledTokenizer extends BaseTokenizer {
574
638
  // Handle behavior flags (use HF config if present, else runtime defaults)
575
639
  const runtimeDefaults = getRuntimeConfig().inference.tokenizer;
576
640
  const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(hf.pre_tokenizer);
641
+ this.#splitPretokenizer = resolveIsolatedSplitPretokenizer(hf.pre_tokenizer);
577
642
  const configuredAddBosToken = this.addBosToken;
578
643
  const configuredAddEosToken = this.addEosToken;
579
644
  const inferredFlags = inferBundledTokenizerBehaviorFlags(hf, this.specialTokens);
@@ -741,6 +806,7 @@ export class BundledTokenizer extends BaseTokenizer {
741
806
 
742
807
  const runtimeDefaults = getRuntimeConfig().inference.tokenizer;
743
808
  const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(tokenizerJson.pre_tokenizer);
809
+ this.#splitPretokenizer = resolveIsolatedSplitPretokenizer(tokenizerJson.pre_tokenizer);
744
810
  const configuredAddBosToken = this.addBosToken;
745
811
  const configuredAddEosToken = this.addEosToken;
746
812
  const inferredFlags = inferBundledTokenizerBehaviorFlags(tokenizerJson, this.specialTokens);
@@ -953,48 +1019,55 @@ export class BundledTokenizer extends BaseTokenizer {
953
1019
  if (text.length === 0) return [];
954
1020
 
955
1021
  let normalized = text;
956
- let prefixed;
957
- if (this.#useByteLevelEncoding) {
958
- if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
959
- normalized = ` ${normalized}`;
960
- }
961
- prefixed = this.#encodeByteLevelText(normalized);
962
- } else {
963
- if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
964
- normalized = ` ${normalized}`;
965
- }
966
- const sp = this.#spacePrefixChar;
967
- prefixed = normalized.replace(/ /g, sp);
968
- }
969
-
970
- if (this.#mergeRanks.size === 0) {
971
- return this.#encodeBPEGreedy(prefixed);
972
- }
973
-
974
- const tokens = this.#bpeTokenize(prefixed);
975
-
976
1022
  const ids = [];
977
- for (const token of tokens) {
978
- const id = this.#vocab.get(token);
979
- if (id !== undefined) {
980
- ids.push(id);
1023
+ if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
1024
+ normalized = ` ${normalized}`;
1025
+ }
1026
+ const pretokens = this.#splitBpePretokens(normalized);
1027
+ for (const pretoken of pretokens) {
1028
+ const encoded = this.#useByteLevelEncoding
1029
+ ? this.#encodeByteLevelText(pretoken)
1030
+ : pretoken.replace(/ /g, this.#spacePrefixChar);
1031
+ if (this.#mergeRanks.size === 0) {
1032
+ ids.push(...this.#encodeBPEGreedy(encoded));
981
1033
  continue;
982
1034
  }
983
- const bytes = new TextEncoder().encode(token);
984
- for (const b of bytes) {
985
- const byteId = this.#byteTokens.get(b);
986
- if (byteId !== undefined) {
987
- ids.push(byteId);
1035
+ for (const token of this.#bpeTokenize(encoded)) {
1036
+ const id = this.#vocab.get(token);
1037
+ if (id !== undefined) {
1038
+ ids.push(id);
988
1039
  continue;
989
1040
  }
990
- const byteToken = `<0x${b.toString(16).padStart(2, '0').toUpperCase()}>`;
991
- ids.push(this.#vocab.get(byteToken) ?? this.#getUnkTokenId());
1041
+ const bytes = new TextEncoder().encode(token);
1042
+ for (const b of bytes) {
1043
+ const byteId = this.#byteTokens.get(b);
1044
+ if (byteId !== undefined) {
1045
+ ids.push(byteId);
1046
+ continue;
1047
+ }
1048
+ const byteToken = `<0x${b.toString(16).padStart(2, '0').toUpperCase()}>`;
1049
+ ids.push(this.#vocab.get(byteToken) ?? this.#getUnkTokenId());
1050
+ }
992
1051
  }
993
1052
  }
994
-
995
1053
  return ids;
996
1054
  }
997
1055
 
1056
+ #splitBpePretokens(text) {
1057
+ if (!this.#useByteLevelEncoding || !this.#splitPretokenizer) return [text];
1058
+ const regex = new RegExp(this.#splitPretokenizer.source, this.#splitPretokenizer.flags);
1059
+ const pretokens = [];
1060
+ let cursor = 0;
1061
+ for (const match of text.matchAll(regex)) {
1062
+ const index = match.index ?? cursor;
1063
+ if (index > cursor) pretokens.push(text.slice(cursor, index));
1064
+ if (match[0]) pretokens.push(match[0]);
1065
+ cursor = index + match[0].length;
1066
+ }
1067
+ if (cursor < text.length) pretokens.push(text.slice(cursor));
1068
+ return pretokens.length > 0 ? pretokens : [text];
1069
+ }
1070
+
998
1071
 
999
1072
  #encodeBPEGreedy(text) {
1000
1073
 
@@ -74,6 +74,7 @@
74
74
  { "match": {}, "value": { "context": "variantWeightDtype" } }
75
75
  ],
76
76
  "weightDtypeMismatchPolicy": [
77
+ { "match": { "actualWeightDtype": "f16", "requiredWeightDtype": "q4k" }, "value": "fallthrough" },
77
78
  { "match": { "actualWeightDtype": "q4k", "requiredWeightDtype": "f16", "hasSubgroups": true }, "value": "fallthrough" },
78
79
  { "match": { "actualWeightDtype": "q4k", "requiredWeightDtype": "f32", "hasSubgroups": true }, "value": "fallthrough" },
79
80
  { "match": { "actualWeightDtype": "litert_int4", "requiredWeightDtype": "f16" }, "value": "fallthrough" },
@@ -275,10 +275,10 @@ async function loadNodeGpuCastRuntime() {
275
275
  gpuCastRuntimePromise = (async () => {
276
276
  await bootstrapNodeWebGPU();
277
277
  const [
278
- { initDevice, getDevice },
278
+ { initDevice, getDevice, destroyDevice, resetDeviceState },
279
279
  { castF32ToF16, runBF16ToF16 },
280
280
  { createTensor },
281
- { acquireBuffer, releaseBuffer, getBufferPool },
281
+ { acquireBuffer, releaseBuffer, getBufferPool, destroyBufferPool },
282
282
  ] = await Promise.all([
283
283
  import('../gpu/device.js'),
284
284
  import('../gpu/kernel-selector.js'),
@@ -293,12 +293,15 @@ async function loadNodeGpuCastRuntime() {
293
293
  }
294
294
  return {
295
295
  getDevice,
296
+ destroyDevice,
297
+ resetDeviceState,
296
298
  castF32ToF16,
297
299
  runBF16ToF16,
298
300
  createTensor,
299
301
  acquireBuffer,
300
302
  releaseBuffer,
301
303
  getBufferPool,
304
+ destroyBufferPool,
302
305
  };
303
306
  })();
304
307
  }
@@ -1056,6 +1059,7 @@ export async function convertSafetensorsDirectory(options) {
1056
1059
  installNodeFileFetchShim();
1057
1060
  const fileRangeReader = createFileRangeReader();
1058
1061
  const totalTimer = createStageTimer('Total');
1062
+ let gpuRuntimeForCleanup = null;
1059
1063
  try {
1060
1064
 
1061
1065
  const [
@@ -1457,9 +1461,8 @@ export async function convertSafetensorsDirectory(options) {
1457
1461
  let result = null;
1458
1462
  try {
1459
1463
  if (executionPlan.useGpuCast) {
1460
- let gpuRuntime;
1461
1464
  try {
1462
- gpuRuntime = await loadNodeGpuCastRuntime();
1465
+ gpuRuntimeForCleanup = await loadNodeGpuCastRuntime();
1463
1466
  } catch (error) {
1464
1467
  const message = error instanceof Error ? error.message : String(error);
1465
1468
  throw new Error(
@@ -1467,7 +1470,7 @@ export async function convertSafetensorsDirectory(options) {
1467
1470
  );
1468
1471
  }
1469
1472
  gpuTensorTransformer = createNodeGpuTensorTransformer({
1470
- runtime: gpuRuntime,
1473
+ runtime: gpuRuntimeForCleanup,
1471
1474
  gpuCastMinTensorBytes: executionPlan.gpuCastMinTensorBytes,
1472
1475
  requireGpuCast: executionPlan.gpuCastRequestedExplicitly === true,
1473
1476
  resolveTensorTargetQuant,
@@ -1609,6 +1612,18 @@ export async function convertSafetensorsDirectory(options) {
1609
1612
  };
1610
1613
  } finally {
1611
1614
  await fileRangeReader.closeAll();
1615
+ if (gpuRuntimeForCleanup) {
1616
+ try {
1617
+ gpuRuntimeForCleanup.destroyBufferPool();
1618
+ } finally {
1619
+ try {
1620
+ gpuRuntimeForCleanup.destroyDevice();
1621
+ } finally {
1622
+ gpuRuntimeForCleanup.resetDeviceState();
1623
+ gpuCastRuntimePromise = null;
1624
+ }
1625
+ }
1626
+ }
1612
1627
  totalTimer.stop();
1613
1628
  }
1614
1629
  }
@@ -0,0 +1,109 @@
1
+ export type RuntimeOptimizationWorkload = 'inference' | 'embedding' | 'rerank';
2
+ export type RuntimeOptimizationDirection = 'maximize' | 'minimize';
3
+
4
+ export interface RuntimeOptimizationContract {
5
+ schema: 'doppler.runtime-optimization-contract/v1';
6
+ contractId: string;
7
+ kind: 'runtime_profile';
8
+ model: {
9
+ modelId: string;
10
+ modelUrl: string | null;
11
+ expectedExecutionContractHash: `sha256:${string}` | null;
12
+ };
13
+ baseline: {
14
+ runtimeProfile: null;
15
+ runtimeConfig: Record<string, unknown>;
16
+ };
17
+ workload: {
18
+ type: RuntimeOptimizationWorkload;
19
+ request: {
20
+ inferenceInput?: Record<string, unknown> | null;
21
+ cacheMode?: 'cold' | 'warm' | null;
22
+ loadMode?: 'opfs' | 'http' | 'memory' | 'file' | null;
23
+ };
24
+ };
25
+ mutationPolicy: {
26
+ dimensions: Array<{ path: string; values: unknown[] }>;
27
+ maxCandidates: number;
28
+ };
29
+ verification: {
30
+ comparisons: Array<{
31
+ path: 'result.output'
32
+ | 'result.metrics.referenceTranscript.tokens.generatedTokenIdsHash'
33
+ | 'result.metrics.referenceTranscript.output.textHash';
34
+ mode: 'canonical_exact';
35
+ }>;
36
+ };
37
+ measurement: {
38
+ metricPath: 'result.metrics.decodeTokensPerSec'
39
+ | 'result.metrics.embeddingMs'
40
+ | 'result.metrics.rerankMs'
41
+ | 'result.timing.decodeTokensPerSec'
42
+ | 'result.timing.totalRunMs';
43
+ direction: RuntimeOptimizationDirection;
44
+ pairCount: number;
45
+ minValidPairs: number;
46
+ minImprovementPercent: number;
47
+ requirePositiveConfidence: boolean;
48
+ maxRelativeStdDevPercent: number | null;
49
+ };
50
+ }
51
+
52
+ export interface RuntimeOptimizationCandidate {
53
+ schema: 'doppler.runtime-optimization-candidate/v1';
54
+ candidateId: string;
55
+ contractHash: `sha256:${string}`;
56
+ parentHash: `sha256:${string}`;
57
+ patch: Array<{ op: 'set'; path: string; value: unknown }>;
58
+ }
59
+
60
+ export interface RuntimeOptimizationReceipt {
61
+ schema: 'doppler.runtime-optimization-receipt/v1';
62
+ contractId: string;
63
+ contractHash: `sha256:${string}`;
64
+ candidateId: string;
65
+ candidateHash: `sha256:${string}`;
66
+ parentHash: `sha256:${string}`;
67
+ model: RuntimeOptimizationContract['model'];
68
+ runtimeInputs: Record<string, unknown>;
69
+ verification: Record<string, unknown>;
70
+ measurement: Record<string, unknown>;
71
+ decision: {
72
+ accepted: boolean;
73
+ status: 'accepted' | 'rejected' | 'invalid';
74
+ reasons: string[];
75
+ };
76
+ receiptHash: `sha256:${string}`;
77
+ }
78
+
79
+ export interface RuntimeOptimizationEvaluationOptions {
80
+ runCommand?: (request: Record<string, unknown>, options?: Record<string, unknown>) => Promise<Record<string, unknown>>;
81
+ commandOptions?: Record<string, unknown>;
82
+ signal?: AbortSignal | null;
83
+ onEvent?: (event: Record<string, unknown>) => void;
84
+ }
85
+
86
+ export declare const RUNTIME_OPTIMIZATION_CONTRACT_SCHEMA: 'doppler.runtime-optimization-contract/v1';
87
+ export declare const RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA: 'doppler.runtime-optimization-candidate/v1';
88
+ export declare const RUNTIME_OPTIMIZATION_RECEIPT_SCHEMA: 'doppler.runtime-optimization-receipt/v1';
89
+
90
+ export declare function validateRuntimeOptimizationContract(
91
+ input: RuntimeOptimizationContract
92
+ ): RuntimeOptimizationContract;
93
+ export declare function hashRuntimeOptimizationContract(input: RuntimeOptimizationContract): `sha256:${string}`;
94
+ export declare function enumerateRuntimeOptimizationCandidates(
95
+ input: RuntimeOptimizationContract
96
+ ): RuntimeOptimizationCandidate[];
97
+ export declare function validateRuntimeOptimizationCandidate(
98
+ candidate: RuntimeOptimizationCandidate,
99
+ contract: RuntimeOptimizationContract
100
+ ): RuntimeOptimizationCandidate;
101
+ export declare function materializeRuntimeOptimizationCandidate(
102
+ contract: RuntimeOptimizationContract,
103
+ candidate: RuntimeOptimizationCandidate
104
+ ): { runtimeProfile: null; runtimeConfig: Record<string, unknown> };
105
+ export declare function evaluateBrowserRuntimeOptimizationCandidate(
106
+ contract: RuntimeOptimizationContract,
107
+ candidate: RuntimeOptimizationCandidate,
108
+ options?: RuntimeOptimizationEvaluationOptions
109
+ ): Promise<RuntimeOptimizationReceipt>;