doppler-gpu 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +62 -12
- package/models/catalog.json +21 -7
- package/package.json +25 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
- package/src/config/kernels/backward-registry.json +13 -2
- package/src/config/kernels/kernel-ref-digests.js +8 -4
- package/src/config/kernels/registry.json +1035 -2677
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
- package/src/config/schema/training-metrics.schema.js +4 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
- package/src/experimental/training/attention-backward.js +123 -55
- package/src/experimental/training/autograd.js +101 -5
- package/src/experimental/training/clip.js +6 -1
- package/src/experimental/training/datasets/text-pairs.js +80 -8
- package/src/experimental/training/distillation/student-fixture.js +470 -114
- package/src/experimental/training/lora-pipeline.js +202 -27
- package/src/experimental/training/lora.js +10 -4
- package/src/experimental/training/optimizer.js +9 -1
- package/src/experimental/training/runner.js +18 -4
- package/src/experimental/training/workloads.js +11 -0
- package/src/gpu/device.js +100 -32
- package/src/gpu/kernels/backward/adam.js +6 -4
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
- package/src/gpu/kernels/backward/index.d.ts +3 -0
- package/src/gpu/kernels/backward/index.js +3 -0
- package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
- package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
- package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
- package/src/gpu/kernels/backward/utils.d.ts +12 -2
- package/src/gpu/kernels/backward/utils.js +9 -2
- package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
- package/src/gpu/kernels/index.d.ts +5 -0
- package/src/gpu/kernels/index.js +5 -0
- package/src/gpu/kernels/matmul-selection.js +5 -0
- package/src/gpu/kernels/silu.d.ts +1 -0
- package/src/gpu/kernels/silu.js +19 -4
- package/src/inference/pipelines/text/embed.js +1 -1
- package/src/inference/pipelines/text/execution-v1.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +2 -0
- package/src/inference/pipelines/text/generator.js +29 -0
- package/src/inference/pipelines/text/lora-apply.js +4 -4
- package/src/inference/pipelines/text/moe-cpu.js +1 -0
- package/src/inference/pipelines/text/moe-gpu.js +1 -0
- package/src/inference/tokenizers/bundled.js +105 -32
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/tooling/node-converter.js +20 -5
- package/src/tooling/runtime-optimization.d.ts +109 -0
- package/src/tooling/runtime-optimization.js +872 -0
- package/src/tooling-experimental-exports.shared.d.ts +25 -0
- package/src/tooling-experimental-exports.shared.js +18 -0
- package/src/tooling-exports.shared.d.ts +19 -0
- package/src/tooling-exports.shared.js +11 -0
- package/src/version.js +1 -1
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Cast F16 to F32 Kernel
|
|
5
5
|
*
|
|
6
6
|
* Converts a buffer of f16 values to f32.
|
|
7
|
+
* Supports 2D dispatch for tensors larger than one workgroup dimension.
|
|
7
8
|
*/
|
|
8
9
|
|
|
9
10
|
enable f16;
|
|
@@ -22,8 +23,11 @@ struct Uniforms {
|
|
|
22
23
|
@group(0) @binding(2) var<storage, read_write> output: array<f32>;
|
|
23
24
|
|
|
24
25
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
25
|
-
fn main(
|
|
26
|
-
|
|
26
|
+
fn main(
|
|
27
|
+
@builtin(global_invocation_id) gid: vec3<u32>,
|
|
28
|
+
@builtin(num_workgroups) num_wg: vec3<u32>
|
|
29
|
+
) {
|
|
30
|
+
let idx = gid.x + gid.y * num_wg.x * WORKGROUP_SIZE;
|
|
27
31
|
if (idx >= u.count) {
|
|
28
32
|
return;
|
|
29
33
|
}
|
|
@@ -460,6 +460,11 @@ export {
|
|
|
460
460
|
recordSoftmaxBackward,
|
|
461
461
|
runRmsNormBackward,
|
|
462
462
|
recordRmsNormBackward,
|
|
463
|
+
runGatedRmsNormBackward,
|
|
464
|
+
recordGatedRmsNormBackward,
|
|
465
|
+
runCausalConv1dSiluBackward,
|
|
466
|
+
recordCausalConv1dSiluBackward,
|
|
467
|
+
runGatedDeltaRecurrentBackward,
|
|
463
468
|
runLayerNormBackward,
|
|
464
469
|
recordLayerNormBackward,
|
|
465
470
|
runAttentionBackward,
|
package/src/gpu/kernels/index.js
CHANGED
|
@@ -370,6 +370,11 @@ export {
|
|
|
370
370
|
recordSoftmaxBackward,
|
|
371
371
|
runRmsNormBackward,
|
|
372
372
|
recordRmsNormBackward,
|
|
373
|
+
runGatedRmsNormBackward,
|
|
374
|
+
recordGatedRmsNormBackward,
|
|
375
|
+
runCausalConv1dSiluBackward,
|
|
376
|
+
recordCausalConv1dSiluBackward,
|
|
377
|
+
runGatedDeltaRecurrentBackward,
|
|
373
378
|
runLayerNormBackward,
|
|
374
379
|
recordLayerNormBackward,
|
|
375
380
|
runAttentionBackward,
|
|
@@ -460,6 +460,11 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
|
|
|
460
460
|
variant: pathVariant,
|
|
461
461
|
reason: 'path_override_quantized_fallthrough',
|
|
462
462
|
});
|
|
463
|
+
} else if (bDtype === 'f16' && isQ4KFusedVariant(pathVariant)) {
|
|
464
|
+
logKernelSelectionOnce('matmul', {
|
|
465
|
+
variant: pathVariant,
|
|
466
|
+
reason: 'path_override_f16_fallthrough',
|
|
467
|
+
});
|
|
463
468
|
} else {
|
|
464
469
|
throw new Error(`[Matmul] Path variant "${pathVariant}" rejected for role=${options.role ?? '?'} layerIdx=${options.layerIdx ?? '?'} phase=${phase} M=${M} K=${K} aDtype=${aDtype} bDtype=${bDtype} outDtype=${requestedOutputDtype}`);
|
|
465
470
|
}
|
|
@@ -16,6 +16,7 @@ export interface SiLUOptions extends OutputBufferOptions {
|
|
|
16
16
|
size?: number | null;
|
|
17
17
|
gate?: Tensor | null;
|
|
18
18
|
gateActivation?: 'silu' | 'sigmoid';
|
|
19
|
+
/** Required when gate is present; ungated SiLU defaults to silu. */
|
|
19
20
|
inputActivation?: 'silu' | 'identity';
|
|
20
21
|
useVec4?: boolean;
|
|
21
22
|
biasOffset?: number;
|
package/src/gpu/kernels/silu.js
CHANGED
|
@@ -49,6 +49,19 @@ function resolveSwigluLimit(value, context) {
|
|
|
49
49
|
return value;
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
function resolveInputActivation(value, hasGate) {
|
|
53
|
+
if (value === undefined) {
|
|
54
|
+
if (hasGate) {
|
|
55
|
+
throw new Error('Gated SiLU requires an explicit inputActivation (identity or silu).');
|
|
56
|
+
}
|
|
57
|
+
return 'silu';
|
|
58
|
+
}
|
|
59
|
+
if (value !== 'identity' && value !== 'silu') {
|
|
60
|
+
throw new Error(`Unsupported SiLU inputActivation: ${value}`);
|
|
61
|
+
}
|
|
62
|
+
return value;
|
|
63
|
+
}
|
|
64
|
+
|
|
52
65
|
|
|
53
66
|
function createSiLUBindGroupEntries(uniformBuffer, input, output, gate) {
|
|
54
67
|
const gateBuffer = gate?.buffer ?? input.buffer;
|
|
@@ -91,9 +104,10 @@ export async function runSiLU(
|
|
|
91
104
|
useVec4 = false,
|
|
92
105
|
swigluLimit,
|
|
93
106
|
gateActivation = 'silu',
|
|
94
|
-
inputActivation
|
|
107
|
+
inputActivation,
|
|
95
108
|
} = options;
|
|
96
109
|
const resolvedSwigluLimit = resolveSwigluLimit(swigluLimit, 'SiLU');
|
|
110
|
+
const resolvedInputActivation = resolveInputActivation(inputActivation, Boolean(gate));
|
|
97
111
|
|
|
98
112
|
const isF16 = canUseF16(input);
|
|
99
113
|
const bytesPerElement = dtypeBytes(input.dtype);
|
|
@@ -109,7 +123,7 @@ export async function runSiLU(
|
|
|
109
123
|
const constants = {
|
|
110
124
|
...(overrides || {}),
|
|
111
125
|
...(gate && gateActivation === 'sigmoid' ? { GATE_USE_SIGMOID: true } : {}),
|
|
112
|
-
...(
|
|
126
|
+
...(resolvedInputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
|
|
113
127
|
};
|
|
114
128
|
const pipeline = await getPipelineFast('silu', variant, null, constants);
|
|
115
129
|
|
|
@@ -344,9 +358,10 @@ export async function recordSiLU(
|
|
|
344
358
|
outputBuffer = null,
|
|
345
359
|
swigluLimit,
|
|
346
360
|
gateActivation = 'silu',
|
|
347
|
-
inputActivation
|
|
361
|
+
inputActivation,
|
|
348
362
|
} = options;
|
|
349
363
|
const resolvedSwigluLimit = resolveSwigluLimit(swigluLimit, 'SiLU');
|
|
364
|
+
const resolvedInputActivation = resolveInputActivation(inputActivation, Boolean(gate));
|
|
350
365
|
|
|
351
366
|
const isF16 = canUseF16(input);
|
|
352
367
|
const bytesPerElement = dtypeBytes(input.dtype);
|
|
@@ -362,7 +377,7 @@ export async function recordSiLU(
|
|
|
362
377
|
const constants = {
|
|
363
378
|
...(overrides || {}),
|
|
364
379
|
...(gate && gateActivation === 'sigmoid' ? { GATE_USE_SIGMOID: true } : {}),
|
|
365
|
-
...(
|
|
380
|
+
...(resolvedInputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
|
|
366
381
|
};
|
|
367
382
|
const pipeline = await getPipelineFast('silu', variant, null, constants);
|
|
368
383
|
|
|
@@ -63,7 +63,7 @@ export function decodeRangeChunkIntoOutput(bytes, sourceDtype, output, dstOffset
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
-
function resolveEmbeddingScale(config, hiddenSize) {
|
|
66
|
+
export function resolveEmbeddingScale(config, hiddenSize) {
|
|
67
67
|
const embeddingScale = config.embeddingScale;
|
|
68
68
|
const scaleEmbeddings = config.scaleEmbeddings;
|
|
69
69
|
if (embeddingScale === undefined) {
|
|
@@ -530,10 +530,14 @@ function collectCoveredLayers(entries, targetLayers, predicate) {
|
|
|
530
530
|
return covered;
|
|
531
531
|
}
|
|
532
532
|
|
|
533
|
-
function assertHybridLinearProjectionIsolation(execution, layerTypes, modelId) {
|
|
533
|
+
function assertHybridLinearProjectionIsolation(execution, layerTypes, modelId, weightDtype) {
|
|
534
534
|
if (!Array.isArray(layerTypes) || layerTypes.length === 0) {
|
|
535
535
|
return;
|
|
536
536
|
}
|
|
537
|
+
const normalizedWeightDtype = String(weightDtype || '').trim().toLowerCase();
|
|
538
|
+
if (normalizedWeightDtype === 'f16' || normalizedWeightDtype === 'f32') {
|
|
539
|
+
return;
|
|
540
|
+
}
|
|
537
541
|
|
|
538
542
|
const linearLayers = layerTypes
|
|
539
543
|
.map((layerType, layerIdx) => ({ layerType, layerIdx }))
|
|
@@ -730,6 +734,7 @@ export function compileExecutionV1(options = {}) {
|
|
|
730
734
|
const modelId = options.modelId ?? 'model';
|
|
731
735
|
const numLayers = options.numLayers ?? 0;
|
|
732
736
|
const headDim = Number.isFinite(options.headDim) ? Math.floor(options.headDim) : null;
|
|
737
|
+
const weightDtype = options.weightDtype ?? null;
|
|
733
738
|
const hasCapabilityProof = options.capabilities && typeof options.capabilities === 'object';
|
|
734
739
|
const hasUseGPUOption = hasOwnProperty(options, 'useGPU');
|
|
735
740
|
const useGPU = hasUseGPUOption
|
|
@@ -861,7 +866,7 @@ export function compileExecutionV1(options = {}) {
|
|
|
861
866
|
|
|
862
867
|
}
|
|
863
868
|
|
|
864
|
-
assertHybridLinearProjectionIsolation(execution, layerTypes, modelId);
|
|
869
|
+
assertHybridLinearProjectionIsolation(execution, layerTypes, modelId, weightDtype);
|
|
865
870
|
|
|
866
871
|
// Transformed graphs have null digests for derived kernels — skip digest
|
|
867
872
|
// validation since the original graph was already validated above.
|
|
@@ -1054,6 +1059,7 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
|
|
|
1054
1059
|
modelId: manifest.modelId ?? options.modelId,
|
|
1055
1060
|
numLayers: options.numLayers ?? manifest.architecture?.numLayers ?? 0,
|
|
1056
1061
|
headDim: options.headDim ?? manifest.architecture?.headDim ?? null,
|
|
1062
|
+
weightDtype: manifest.quantizationInfo?.weights ?? null,
|
|
1057
1063
|
capabilities: options.capabilities ?? null,
|
|
1058
1064
|
platform: options.platform ?? null,
|
|
1059
1065
|
runtimeSession: hasExplicitRuntimeOverrides
|
|
@@ -1257,6 +1257,7 @@ export async function runDenseFFNGPU(
|
|
|
1257
1257
|
const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
|
|
1258
1258
|
size: numTokens * intermediateSize,
|
|
1259
1259
|
gate: gateOutput,
|
|
1260
|
+
inputActivation: 'identity',
|
|
1260
1261
|
swigluLimit,
|
|
1261
1262
|
label: `L${layerIdx}.ffn_activation`,
|
|
1262
1263
|
layerIdx,
|
|
@@ -1646,6 +1647,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
|
|
|
1646
1647
|
activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
|
|
1647
1648
|
size: numTokens * intermediateSize,
|
|
1648
1649
|
gate: gateOutput,
|
|
1650
|
+
inputActivation: 'identity',
|
|
1649
1651
|
swigluLimit,
|
|
1650
1652
|
}, recorder);
|
|
1651
1653
|
|
|
@@ -2277,6 +2277,27 @@ export class PipelineGenerator {
|
|
|
2277
2277
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2278
2278
|
this.#state.stats.prefillProfileSteps = [];
|
|
2279
2279
|
const opts = resolvePrefillOptions(this.#state, options);
|
|
2280
|
+
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
2281
|
+
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
2282
|
+
if (diagnosticsEnabled) {
|
|
2283
|
+
const captureConfig = {
|
|
2284
|
+
...createDefaultCaptureConfig(),
|
|
2285
|
+
enabled: true,
|
|
2286
|
+
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
2287
|
+
...(options?.diagnostics?.captureConfig ?? {}),
|
|
2288
|
+
};
|
|
2289
|
+
validateCaptureConfig(captureConfig);
|
|
2290
|
+
this.#state.operatorDiagnostics = {
|
|
2291
|
+
enabled: true,
|
|
2292
|
+
captureConfig,
|
|
2293
|
+
emitter: new OperatorEventEmitter({
|
|
2294
|
+
modelHash: this.#state.manifest?.modelId ?? null,
|
|
2295
|
+
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
2296
|
+
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
2297
|
+
}),
|
|
2298
|
+
tsirFixture: null,
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
2280
2301
|
try {
|
|
2281
2302
|
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
2282
2303
|
|
|
@@ -2298,6 +2319,14 @@ export class PipelineGenerator {
|
|
|
2298
2319
|
};
|
|
2299
2320
|
} finally {
|
|
2300
2321
|
this._closeFinitenessFallbackWindow(opts);
|
|
2322
|
+
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2323
|
+
? {
|
|
2324
|
+
enabled: true,
|
|
2325
|
+
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2326
|
+
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2327
|
+
}
|
|
2328
|
+
: null;
|
|
2329
|
+
this.#state.operatorDiagnostics = null;
|
|
2301
2330
|
}
|
|
2302
2331
|
}
|
|
2303
2332
|
|
|
@@ -46,12 +46,12 @@ export async function applyLoRA(input, baseOutput, lora, dims, getWeightBuffer,
|
|
|
46
46
|
let scaled = null;
|
|
47
47
|
try {
|
|
48
48
|
loraIntermediate = recorder
|
|
49
|
-
? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB:
|
|
50
|
-
: await runMatmul(input, aBuf, M, rank, K, { transposeB:
|
|
49
|
+
? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: false, role: 'lora_a', kernelPath, outputDtype: intermediateDtype })
|
|
50
|
+
: await runMatmul(input, aBuf, M, rank, K, { transposeB: false, role: 'lora_a', kernelPath, outputDtype: intermediateDtype });
|
|
51
51
|
|
|
52
52
|
loraOutput = recorder
|
|
53
|
-
? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB:
|
|
54
|
-
: await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB:
|
|
53
|
+
? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: false, role: 'lora_b', kernelPath, outputDtype })
|
|
54
|
+
: await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: false, role: 'lora_b', kernelPath, outputDtype });
|
|
55
55
|
|
|
56
56
|
scaled = recorder
|
|
57
57
|
? await recordScale(recorder, loraOutput, lora.scale, { outputBuffer: null })
|
|
@@ -122,6 +122,7 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
|
|
|
122
122
|
activatedOutput = await activationFn(upOutput, {
|
|
123
123
|
size: numTokens * intermediateSize,
|
|
124
124
|
gate: gateOutput,
|
|
125
|
+
inputActivation: 'identity',
|
|
125
126
|
swigluLimit,
|
|
126
127
|
});
|
|
127
128
|
|
|
@@ -95,6 +95,67 @@ function resolveByteLevelPretokenizerConfig(preTokenizer) {
|
|
|
95
95
|
};
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
+
function normalizeInlineCaseInsensitiveGroups(pattern) {
|
|
99
|
+
let source = String(pattern || '');
|
|
100
|
+
let caseInsensitive = false;
|
|
101
|
+
let start = source.indexOf('(?i:');
|
|
102
|
+
while (start !== -1) {
|
|
103
|
+
let depth = 1;
|
|
104
|
+
let escaped = false;
|
|
105
|
+
let end = start + 4;
|
|
106
|
+
for (; end < source.length; end++) {
|
|
107
|
+
const char = source[end];
|
|
108
|
+
if (escaped) {
|
|
109
|
+
escaped = false;
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (char === '\\') {
|
|
113
|
+
escaped = true;
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
if (char === '(') depth += 1;
|
|
117
|
+
else if (char === ')') {
|
|
118
|
+
depth -= 1;
|
|
119
|
+
if (depth === 0) break;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (depth !== 0) {
|
|
123
|
+
throw new Error('[Tokenizer] Unterminated inline case-insensitive regex group.');
|
|
124
|
+
}
|
|
125
|
+
source = `${source.slice(0, start)}(?:${source.slice(start + 4, end)})${source.slice(end + 1)}`;
|
|
126
|
+
caseInsensitive = true;
|
|
127
|
+
start = source.indexOf('(?i:', start + 3);
|
|
128
|
+
}
|
|
129
|
+
return { source, caseInsensitive };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function resolveIsolatedSplitPretokenizer(preTokenizer) {
|
|
133
|
+
if (!preTokenizer || typeof preTokenizer !== 'object') return null;
|
|
134
|
+
if (preTokenizer.type === 'Sequence' && Array.isArray(preTokenizer.pretokenizers)) {
|
|
135
|
+
for (const entry of preTokenizer.pretokenizers) {
|
|
136
|
+
const resolved = resolveIsolatedSplitPretokenizer(entry);
|
|
137
|
+
if (resolved) return resolved;
|
|
138
|
+
}
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
if (
|
|
142
|
+
preTokenizer.type !== 'Split'
|
|
143
|
+
|| preTokenizer.behavior !== 'Isolated'
|
|
144
|
+
|| preTokenizer.invert === true
|
|
145
|
+
|| typeof preTokenizer.pattern?.Regex !== 'string'
|
|
146
|
+
) {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
const normalized = normalizeInlineCaseInsensitiveGroups(preTokenizer.pattern.Regex);
|
|
150
|
+
const flags = `gu${normalized.caseInsensitive ? 'i' : ''}`;
|
|
151
|
+
try {
|
|
152
|
+
new RegExp(normalized.source, flags);
|
|
153
|
+
} catch (error) {
|
|
154
|
+
throw new Error(`[Tokenizer] Unsupported split pre-tokenizer regex: ${error.message}`);
|
|
155
|
+
}
|
|
156
|
+
return { source: normalized.source, flags };
|
|
157
|
+
}
|
|
158
|
+
|
|
98
159
|
function hexNibble(code) {
|
|
99
160
|
if (code >= 48 && code <= 57) return code - 48;
|
|
100
161
|
if (code >= 65 && code <= 70) return code - 55;
|
|
@@ -388,6 +449,8 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
388
449
|
|
|
389
450
|
#useByteLevelEncoding = false;
|
|
390
451
|
|
|
452
|
+
#splitPretokenizer = null;
|
|
453
|
+
|
|
391
454
|
|
|
392
455
|
constructor(config = {}) {
|
|
393
456
|
// BundledTokenizer gets vocabSize from load(), so defer validation
|
|
@@ -412,6 +475,7 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
412
475
|
this.#byteDecoder = null;
|
|
413
476
|
this.#byteEncoder = null;
|
|
414
477
|
this.#useByteLevelEncoding = false;
|
|
478
|
+
this.#splitPretokenizer = null;
|
|
415
479
|
this.vocabSize = 0;
|
|
416
480
|
}
|
|
417
481
|
|
|
@@ -574,6 +638,7 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
574
638
|
// Handle behavior flags (use HF config if present, else runtime defaults)
|
|
575
639
|
const runtimeDefaults = getRuntimeConfig().inference.tokenizer;
|
|
576
640
|
const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(hf.pre_tokenizer);
|
|
641
|
+
this.#splitPretokenizer = resolveIsolatedSplitPretokenizer(hf.pre_tokenizer);
|
|
577
642
|
const configuredAddBosToken = this.addBosToken;
|
|
578
643
|
const configuredAddEosToken = this.addEosToken;
|
|
579
644
|
const inferredFlags = inferBundledTokenizerBehaviorFlags(hf, this.specialTokens);
|
|
@@ -741,6 +806,7 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
741
806
|
|
|
742
807
|
const runtimeDefaults = getRuntimeConfig().inference.tokenizer;
|
|
743
808
|
const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(tokenizerJson.pre_tokenizer);
|
|
809
|
+
this.#splitPretokenizer = resolveIsolatedSplitPretokenizer(tokenizerJson.pre_tokenizer);
|
|
744
810
|
const configuredAddBosToken = this.addBosToken;
|
|
745
811
|
const configuredAddEosToken = this.addEosToken;
|
|
746
812
|
const inferredFlags = inferBundledTokenizerBehaviorFlags(tokenizerJson, this.specialTokens);
|
|
@@ -953,48 +1019,55 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
953
1019
|
if (text.length === 0) return [];
|
|
954
1020
|
|
|
955
1021
|
let normalized = text;
|
|
956
|
-
let prefixed;
|
|
957
|
-
if (this.#useByteLevelEncoding) {
|
|
958
|
-
if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
|
|
959
|
-
normalized = ` ${normalized}`;
|
|
960
|
-
}
|
|
961
|
-
prefixed = this.#encodeByteLevelText(normalized);
|
|
962
|
-
} else {
|
|
963
|
-
if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
|
|
964
|
-
normalized = ` ${normalized}`;
|
|
965
|
-
}
|
|
966
|
-
const sp = this.#spacePrefixChar;
|
|
967
|
-
prefixed = normalized.replace(/ /g, sp);
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
if (this.#mergeRanks.size === 0) {
|
|
971
|
-
return this.#encodeBPEGreedy(prefixed);
|
|
972
|
-
}
|
|
973
|
-
|
|
974
|
-
const tokens = this.#bpeTokenize(prefixed);
|
|
975
|
-
|
|
976
1022
|
const ids = [];
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
1023
|
+
if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
|
|
1024
|
+
normalized = ` ${normalized}`;
|
|
1025
|
+
}
|
|
1026
|
+
const pretokens = this.#splitBpePretokens(normalized);
|
|
1027
|
+
for (const pretoken of pretokens) {
|
|
1028
|
+
const encoded = this.#useByteLevelEncoding
|
|
1029
|
+
? this.#encodeByteLevelText(pretoken)
|
|
1030
|
+
: pretoken.replace(/ /g, this.#spacePrefixChar);
|
|
1031
|
+
if (this.#mergeRanks.size === 0) {
|
|
1032
|
+
ids.push(...this.#encodeBPEGreedy(encoded));
|
|
981
1033
|
continue;
|
|
982
1034
|
}
|
|
983
|
-
const
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
ids.push(byteId);
|
|
1035
|
+
for (const token of this.#bpeTokenize(encoded)) {
|
|
1036
|
+
const id = this.#vocab.get(token);
|
|
1037
|
+
if (id !== undefined) {
|
|
1038
|
+
ids.push(id);
|
|
988
1039
|
continue;
|
|
989
1040
|
}
|
|
990
|
-
const
|
|
991
|
-
|
|
1041
|
+
const bytes = new TextEncoder().encode(token);
|
|
1042
|
+
for (const b of bytes) {
|
|
1043
|
+
const byteId = this.#byteTokens.get(b);
|
|
1044
|
+
if (byteId !== undefined) {
|
|
1045
|
+
ids.push(byteId);
|
|
1046
|
+
continue;
|
|
1047
|
+
}
|
|
1048
|
+
const byteToken = `<0x${b.toString(16).padStart(2, '0').toUpperCase()}>`;
|
|
1049
|
+
ids.push(this.#vocab.get(byteToken) ?? this.#getUnkTokenId());
|
|
1050
|
+
}
|
|
992
1051
|
}
|
|
993
1052
|
}
|
|
994
|
-
|
|
995
1053
|
return ids;
|
|
996
1054
|
}
|
|
997
1055
|
|
|
1056
|
+
#splitBpePretokens(text) {
|
|
1057
|
+
if (!this.#useByteLevelEncoding || !this.#splitPretokenizer) return [text];
|
|
1058
|
+
const regex = new RegExp(this.#splitPretokenizer.source, this.#splitPretokenizer.flags);
|
|
1059
|
+
const pretokens = [];
|
|
1060
|
+
let cursor = 0;
|
|
1061
|
+
for (const match of text.matchAll(regex)) {
|
|
1062
|
+
const index = match.index ?? cursor;
|
|
1063
|
+
if (index > cursor) pretokens.push(text.slice(cursor, index));
|
|
1064
|
+
if (match[0]) pretokens.push(match[0]);
|
|
1065
|
+
cursor = index + match[0].length;
|
|
1066
|
+
}
|
|
1067
|
+
if (cursor < text.length) pretokens.push(text.slice(cursor));
|
|
1068
|
+
return pretokens.length > 0 ? pretokens : [text];
|
|
1069
|
+
}
|
|
1070
|
+
|
|
998
1071
|
|
|
999
1072
|
#encodeBPEGreedy(text) {
|
|
1000
1073
|
|
|
@@ -74,6 +74,7 @@
|
|
|
74
74
|
{ "match": {}, "value": { "context": "variantWeightDtype" } }
|
|
75
75
|
],
|
|
76
76
|
"weightDtypeMismatchPolicy": [
|
|
77
|
+
{ "match": { "actualWeightDtype": "f16", "requiredWeightDtype": "q4k" }, "value": "fallthrough" },
|
|
77
78
|
{ "match": { "actualWeightDtype": "q4k", "requiredWeightDtype": "f16", "hasSubgroups": true }, "value": "fallthrough" },
|
|
78
79
|
{ "match": { "actualWeightDtype": "q4k", "requiredWeightDtype": "f32", "hasSubgroups": true }, "value": "fallthrough" },
|
|
79
80
|
{ "match": { "actualWeightDtype": "litert_int4", "requiredWeightDtype": "f16" }, "value": "fallthrough" },
|
|
@@ -275,10 +275,10 @@ async function loadNodeGpuCastRuntime() {
|
|
|
275
275
|
gpuCastRuntimePromise = (async () => {
|
|
276
276
|
await bootstrapNodeWebGPU();
|
|
277
277
|
const [
|
|
278
|
-
{ initDevice, getDevice },
|
|
278
|
+
{ initDevice, getDevice, destroyDevice, resetDeviceState },
|
|
279
279
|
{ castF32ToF16, runBF16ToF16 },
|
|
280
280
|
{ createTensor },
|
|
281
|
-
{ acquireBuffer, releaseBuffer, getBufferPool },
|
|
281
|
+
{ acquireBuffer, releaseBuffer, getBufferPool, destroyBufferPool },
|
|
282
282
|
] = await Promise.all([
|
|
283
283
|
import('../gpu/device.js'),
|
|
284
284
|
import('../gpu/kernel-selector.js'),
|
|
@@ -293,12 +293,15 @@ async function loadNodeGpuCastRuntime() {
|
|
|
293
293
|
}
|
|
294
294
|
return {
|
|
295
295
|
getDevice,
|
|
296
|
+
destroyDevice,
|
|
297
|
+
resetDeviceState,
|
|
296
298
|
castF32ToF16,
|
|
297
299
|
runBF16ToF16,
|
|
298
300
|
createTensor,
|
|
299
301
|
acquireBuffer,
|
|
300
302
|
releaseBuffer,
|
|
301
303
|
getBufferPool,
|
|
304
|
+
destroyBufferPool,
|
|
302
305
|
};
|
|
303
306
|
})();
|
|
304
307
|
}
|
|
@@ -1056,6 +1059,7 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1056
1059
|
installNodeFileFetchShim();
|
|
1057
1060
|
const fileRangeReader = createFileRangeReader();
|
|
1058
1061
|
const totalTimer = createStageTimer('Total');
|
|
1062
|
+
let gpuRuntimeForCleanup = null;
|
|
1059
1063
|
try {
|
|
1060
1064
|
|
|
1061
1065
|
const [
|
|
@@ -1457,9 +1461,8 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1457
1461
|
let result = null;
|
|
1458
1462
|
try {
|
|
1459
1463
|
if (executionPlan.useGpuCast) {
|
|
1460
|
-
let gpuRuntime;
|
|
1461
1464
|
try {
|
|
1462
|
-
|
|
1465
|
+
gpuRuntimeForCleanup = await loadNodeGpuCastRuntime();
|
|
1463
1466
|
} catch (error) {
|
|
1464
1467
|
const message = error instanceof Error ? error.message : String(error);
|
|
1465
1468
|
throw new Error(
|
|
@@ -1467,7 +1470,7 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1467
1470
|
);
|
|
1468
1471
|
}
|
|
1469
1472
|
gpuTensorTransformer = createNodeGpuTensorTransformer({
|
|
1470
|
-
runtime:
|
|
1473
|
+
runtime: gpuRuntimeForCleanup,
|
|
1471
1474
|
gpuCastMinTensorBytes: executionPlan.gpuCastMinTensorBytes,
|
|
1472
1475
|
requireGpuCast: executionPlan.gpuCastRequestedExplicitly === true,
|
|
1473
1476
|
resolveTensorTargetQuant,
|
|
@@ -1609,6 +1612,18 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1609
1612
|
};
|
|
1610
1613
|
} finally {
|
|
1611
1614
|
await fileRangeReader.closeAll();
|
|
1615
|
+
if (gpuRuntimeForCleanup) {
|
|
1616
|
+
try {
|
|
1617
|
+
gpuRuntimeForCleanup.destroyBufferPool();
|
|
1618
|
+
} finally {
|
|
1619
|
+
try {
|
|
1620
|
+
gpuRuntimeForCleanup.destroyDevice();
|
|
1621
|
+
} finally {
|
|
1622
|
+
gpuRuntimeForCleanup.resetDeviceState();
|
|
1623
|
+
gpuCastRuntimePromise = null;
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1612
1627
|
totalTimer.stop();
|
|
1613
1628
|
}
|
|
1614
1629
|
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
export type RuntimeOptimizationWorkload = 'inference' | 'embedding' | 'rerank';
|
|
2
|
+
export type RuntimeOptimizationDirection = 'maximize' | 'minimize';
|
|
3
|
+
|
|
4
|
+
export interface RuntimeOptimizationContract {
|
|
5
|
+
schema: 'doppler.runtime-optimization-contract/v1';
|
|
6
|
+
contractId: string;
|
|
7
|
+
kind: 'runtime_profile';
|
|
8
|
+
model: {
|
|
9
|
+
modelId: string;
|
|
10
|
+
modelUrl: string | null;
|
|
11
|
+
expectedExecutionContractHash: `sha256:${string}` | null;
|
|
12
|
+
};
|
|
13
|
+
baseline: {
|
|
14
|
+
runtimeProfile: null;
|
|
15
|
+
runtimeConfig: Record<string, unknown>;
|
|
16
|
+
};
|
|
17
|
+
workload: {
|
|
18
|
+
type: RuntimeOptimizationWorkload;
|
|
19
|
+
request: {
|
|
20
|
+
inferenceInput?: Record<string, unknown> | null;
|
|
21
|
+
cacheMode?: 'cold' | 'warm' | null;
|
|
22
|
+
loadMode?: 'opfs' | 'http' | 'memory' | 'file' | null;
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
mutationPolicy: {
|
|
26
|
+
dimensions: Array<{ path: string; values: unknown[] }>;
|
|
27
|
+
maxCandidates: number;
|
|
28
|
+
};
|
|
29
|
+
verification: {
|
|
30
|
+
comparisons: Array<{
|
|
31
|
+
path: 'result.output'
|
|
32
|
+
| 'result.metrics.referenceTranscript.tokens.generatedTokenIdsHash'
|
|
33
|
+
| 'result.metrics.referenceTranscript.output.textHash';
|
|
34
|
+
mode: 'canonical_exact';
|
|
35
|
+
}>;
|
|
36
|
+
};
|
|
37
|
+
measurement: {
|
|
38
|
+
metricPath: 'result.metrics.decodeTokensPerSec'
|
|
39
|
+
| 'result.metrics.embeddingMs'
|
|
40
|
+
| 'result.metrics.rerankMs'
|
|
41
|
+
| 'result.timing.decodeTokensPerSec'
|
|
42
|
+
| 'result.timing.totalRunMs';
|
|
43
|
+
direction: RuntimeOptimizationDirection;
|
|
44
|
+
pairCount: number;
|
|
45
|
+
minValidPairs: number;
|
|
46
|
+
minImprovementPercent: number;
|
|
47
|
+
requirePositiveConfidence: boolean;
|
|
48
|
+
maxRelativeStdDevPercent: number | null;
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface RuntimeOptimizationCandidate {
|
|
53
|
+
schema: 'doppler.runtime-optimization-candidate/v1';
|
|
54
|
+
candidateId: string;
|
|
55
|
+
contractHash: `sha256:${string}`;
|
|
56
|
+
parentHash: `sha256:${string}`;
|
|
57
|
+
patch: Array<{ op: 'set'; path: string; value: unknown }>;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface RuntimeOptimizationReceipt {
|
|
61
|
+
schema: 'doppler.runtime-optimization-receipt/v1';
|
|
62
|
+
contractId: string;
|
|
63
|
+
contractHash: `sha256:${string}`;
|
|
64
|
+
candidateId: string;
|
|
65
|
+
candidateHash: `sha256:${string}`;
|
|
66
|
+
parentHash: `sha256:${string}`;
|
|
67
|
+
model: RuntimeOptimizationContract['model'];
|
|
68
|
+
runtimeInputs: Record<string, unknown>;
|
|
69
|
+
verification: Record<string, unknown>;
|
|
70
|
+
measurement: Record<string, unknown>;
|
|
71
|
+
decision: {
|
|
72
|
+
accepted: boolean;
|
|
73
|
+
status: 'accepted' | 'rejected' | 'invalid';
|
|
74
|
+
reasons: string[];
|
|
75
|
+
};
|
|
76
|
+
receiptHash: `sha256:${string}`;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface RuntimeOptimizationEvaluationOptions {
|
|
80
|
+
runCommand?: (request: Record<string, unknown>, options?: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
|
81
|
+
commandOptions?: Record<string, unknown>;
|
|
82
|
+
signal?: AbortSignal | null;
|
|
83
|
+
onEvent?: (event: Record<string, unknown>) => void;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export declare const RUNTIME_OPTIMIZATION_CONTRACT_SCHEMA: 'doppler.runtime-optimization-contract/v1';
|
|
87
|
+
export declare const RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA: 'doppler.runtime-optimization-candidate/v1';
|
|
88
|
+
export declare const RUNTIME_OPTIMIZATION_RECEIPT_SCHEMA: 'doppler.runtime-optimization-receipt/v1';
|
|
89
|
+
|
|
90
|
+
export declare function validateRuntimeOptimizationContract(
|
|
91
|
+
input: RuntimeOptimizationContract
|
|
92
|
+
): RuntimeOptimizationContract;
|
|
93
|
+
export declare function hashRuntimeOptimizationContract(input: RuntimeOptimizationContract): `sha256:${string}`;
|
|
94
|
+
export declare function enumerateRuntimeOptimizationCandidates(
|
|
95
|
+
input: RuntimeOptimizationContract
|
|
96
|
+
): RuntimeOptimizationCandidate[];
|
|
97
|
+
export declare function validateRuntimeOptimizationCandidate(
|
|
98
|
+
candidate: RuntimeOptimizationCandidate,
|
|
99
|
+
contract: RuntimeOptimizationContract
|
|
100
|
+
): RuntimeOptimizationCandidate;
|
|
101
|
+
export declare function materializeRuntimeOptimizationCandidate(
|
|
102
|
+
contract: RuntimeOptimizationContract,
|
|
103
|
+
candidate: RuntimeOptimizationCandidate
|
|
104
|
+
): { runtimeProfile: null; runtimeConfig: Record<string, unknown> };
|
|
105
|
+
export declare function evaluateBrowserRuntimeOptimizationCandidate(
|
|
106
|
+
contract: RuntimeOptimizationContract,
|
|
107
|
+
candidate: RuntimeOptimizationCandidate,
|
|
108
|
+
options?: RuntimeOptimizationEvaluationOptions
|
|
109
|
+
): Promise<RuntimeOptimizationReceipt>;
|