doppler-gpu 0.4.10 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +40 -1
- package/models/catalog.json +637 -32
- package/package.json +24 -3
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +135 -13
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +69 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +73 -11
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/adapter.schema.d.ts +6 -1
- package/src/config/schema/adapter.schema.js +5 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/artifact-contract.d.ts +83 -0
- package/src/experimental/adapters/artifact-contract.js +254 -0
- package/src/experimental/adapters/index.d.ts +25 -0
- package/src/experimental/adapters/index.js +12 -0
- package/src/experimental/adapters/lora-loader.js +24 -4
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/experimental/training/artifacts.d.ts +160 -0
- package/src/experimental/training/attention-backward.d.ts +64 -0
- package/src/experimental/training/attention-forward.d.ts +22 -0
- package/src/experimental/training/attention-forward.js +82 -0
- package/src/experimental/training/autograd.d.ts +76 -0
- package/src/experimental/training/checkpoint.d.ts +36 -0
- package/src/experimental/training/clip.d.ts +9 -0
- package/src/experimental/training/dataloader.d.ts +8 -0
- package/src/experimental/training/distillation/student-fixture.js +1 -0
- package/src/experimental/training/export.d.ts +41 -0
- package/src/experimental/training/lora-pipeline.d.ts +186 -0
- package/src/experimental/training/lora-pipeline.js +5 -0
- package/src/experimental/training/lora.d.ts +20 -0
- package/src/experimental/training/lora.js +4 -3
- package/src/experimental/training/loss-scaling.d.ts +21 -0
- package/src/experimental/training/loss.d.ts +10 -0
- package/src/experimental/training/native-qwen-lora.d.ts +106 -0
- package/src/experimental/training/native-qwen-lora.js +449 -0
- package/src/experimental/training/objectives/base.d.ts +58 -0
- package/src/experimental/training/optimizer.d.ts +22 -0
- package/src/experimental/training/optimizer.js +1 -0
- package/src/experimental/training/runner.d.ts +249 -0
- package/src/experimental/training/trainer.d.ts +89 -0
- package/src/experimental/training/workloads.d.ts +205 -0
- package/src/gpu/kernels/backward/adam.d.ts +1 -0
- package/src/gpu/kernels/backward/adam.js +6 -2
- package/src/gpu/kernels/backward/adam.wgsl +3 -2
- package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
- package/src/gpu/kernels/backward/utils.js +18 -4
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +205 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.d.ts +5 -1
- package/src/inference/pipelines/text/generator.js +196 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
- package/src/inference/pipelines/text/linear-attention.js +77 -6
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/lora-types.d.ts +6 -1
- package/src/inference/pipelines/text/lora-types.js +5 -0
- package/src/inference/pipelines/text/ops.js +33 -0
- package/src/inference/pipelines/text/types.d.ts +66 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +9 -2
- package/src/inference/pipelines/text.js +116 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/models/diffusiongemma.js +1 -1
- package/src/models/embeddinggemma.js +1 -1
- package/src/models/gemma3.js +1 -1
- package/src/models/gemma4.js +1 -1
- package/src/models/qwen3.js +1 -1
- package/src/tooling/hf-registry-utils.js +1 -1
- package/src/tooling/node-browser-command-runner.d.ts +16 -0
- package/src/tooling/node-browser-command-runner.js +1 -1
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/training-browser.d.ts +19 -0
- package/src/training-browser.js +13 -0
- package/src/training.d.ts +137 -0
- package/src/training.js +219 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +9 -1
- package/tests/kernels/browser/test-page.js +4 -2
- package/tests/kernels/reference/adam.js +4 -2
- package/NOTICE +0 -5
|
@@ -63,6 +63,7 @@ import {
|
|
|
63
63
|
resolveAdvanceEmbeddingMode,
|
|
64
64
|
getFinalNormWeights,
|
|
65
65
|
extractEmbeddingFromHidden,
|
|
66
|
+
extractTokenEmbeddingsFromHidden,
|
|
66
67
|
} from './generator-runtime.js';
|
|
67
68
|
|
|
68
69
|
import { resolveSamplingConfig } from './sampling-config.js';
|
|
@@ -642,7 +643,20 @@ export class PipelineGenerator {
|
|
|
642
643
|
embeddingMode,
|
|
643
644
|
finalNormWeights,
|
|
644
645
|
config,
|
|
645
|
-
this.#state.embeddingPostprocessor
|
|
646
|
+
this.#state.embeddingPostprocessor,
|
|
647
|
+
null,
|
|
648
|
+
this.#state.weights.get('final_norm_bias') ?? null
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
_extractTokenEmbeddingsFromHidden(hiddenStates, numTokens, hiddenSize, finalNormWeights, config) {
|
|
653
|
+
return extractTokenEmbeddingsFromHidden(
|
|
654
|
+
hiddenStates,
|
|
655
|
+
numTokens,
|
|
656
|
+
hiddenSize,
|
|
657
|
+
finalNormWeights,
|
|
658
|
+
config,
|
|
659
|
+
this.#state.weights.get('final_norm_bias') ?? null
|
|
646
660
|
);
|
|
647
661
|
}
|
|
648
662
|
|
|
@@ -2101,6 +2115,111 @@ export class PipelineGenerator {
|
|
|
2101
2115
|
}
|
|
2102
2116
|
}
|
|
2103
2117
|
|
|
2118
|
+
async prefillForLoRATraining(inputIds, options = {}) {
|
|
2119
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
2120
|
+
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
2121
|
+
if (!Array.isArray(inputIds) && !(inputIds instanceof Int32Array) && !(inputIds instanceof Uint32Array)) {
|
|
2122
|
+
throw new Error('prefillForLoRATraining requires token IDs.');
|
|
2123
|
+
}
|
|
2124
|
+
const tokens = Array.from(inputIds, (value) => Number(value));
|
|
2125
|
+
if (tokens.length === 0) throw new Error('prefillForLoRATraining requires at least one token.');
|
|
2126
|
+
this._assertTokenIdsInRange(tokens, 'prefillForLoRATraining.inputIds');
|
|
2127
|
+
|
|
2128
|
+
const config = this.#state.modelConfig;
|
|
2129
|
+
const layerIdx = Number(options.layerIdx);
|
|
2130
|
+
if (!Number.isInteger(layerIdx) || layerIdx !== config.numLayers - 1) {
|
|
2131
|
+
throw new Error(`native_lora_target_not_supported: layerIdx must be the final layer (${config.numLayers - 1}).`);
|
|
2132
|
+
}
|
|
2133
|
+
if (options.module !== 'down_proj') {
|
|
2134
|
+
throw new Error('native_lora_target_not_supported: module must be down_proj.');
|
|
2135
|
+
}
|
|
2136
|
+
if (config.layerTypes?.[layerIdx] !== 'full_attention') {
|
|
2137
|
+
throw new Error('native_lora_target_not_supported: final layer must be full_attention.');
|
|
2138
|
+
}
|
|
2139
|
+
if (config.postFeedforwardNorm === true) {
|
|
2140
|
+
throw new Error('native_lora_target_not_supported: final-layer post-feedforward normalization is not supported.');
|
|
2141
|
+
}
|
|
2142
|
+
if (this.#state.lora) {
|
|
2143
|
+
throw new Error('native_lora_base_capture_requires_inactive_adapter.');
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
this.resetToSeqLen(0);
|
|
2147
|
+
this._resetDecodeRuntimeState();
|
|
2148
|
+
const opts = resolvePrefillOptions(this.#state, {
|
|
2149
|
+
...options,
|
|
2150
|
+
inputIds: tokens,
|
|
2151
|
+
useChatTemplate: false,
|
|
2152
|
+
});
|
|
2153
|
+
let activationTensor = null;
|
|
2154
|
+
let hiddenTensor = null;
|
|
2155
|
+
let disposed = false;
|
|
2156
|
+
const releaseOwned = () => {
|
|
2157
|
+
if (disposed) return;
|
|
2158
|
+
disposed = true;
|
|
2159
|
+
if (activationTensor?.buffer) releaseBuffer(activationTensor.buffer);
|
|
2160
|
+
if (hiddenTensor?.buffer) releaseBuffer(hiddenTensor.buffer);
|
|
2161
|
+
};
|
|
2162
|
+
const capture = async ({ tensor, numTokens, hiddenSize, recorder }) => {
|
|
2163
|
+
if (activationTensor) {
|
|
2164
|
+
throw new Error('native_lora_training_capture_duplicated.');
|
|
2165
|
+
}
|
|
2166
|
+
const bytesPerElement = tensor.dtype === 'f16' ? 2 : 4;
|
|
2167
|
+
const byteLength = Math.ceil((numTokens * hiddenSize * bytesPerElement) / 4) * 4;
|
|
2168
|
+
const buffer = acquireBuffer(byteLength, undefined, `L${layerIdx}.native_lora_ffn_act`);
|
|
2169
|
+
if (recorder) {
|
|
2170
|
+
recorder.getEncoder().copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
|
|
2171
|
+
} else {
|
|
2172
|
+
const device = getDevice();
|
|
2173
|
+
const encoder = device.createCommandEncoder({ label: `L${layerIdx}.native_lora_capture` });
|
|
2174
|
+
encoder.copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
|
|
2175
|
+
device.queue.submit([encoder.finish()]);
|
|
2176
|
+
}
|
|
2177
|
+
activationTensor = createTensor(
|
|
2178
|
+
buffer,
|
|
2179
|
+
tensor.dtype,
|
|
2180
|
+
[numTokens, hiddenSize],
|
|
2181
|
+
`L${layerIdx}.native_lora_ffn_act`
|
|
2182
|
+
);
|
|
2183
|
+
};
|
|
2184
|
+
|
|
2185
|
+
try {
|
|
2186
|
+
const prefill = await this._prefillToHidden(tokens, {
|
|
2187
|
+
...opts,
|
|
2188
|
+
_trainingCapture: { layerIdx, stage: 'ffn_act', capture },
|
|
2189
|
+
});
|
|
2190
|
+
if (prefill.currentRecorder) {
|
|
2191
|
+
recordPrefillRecorderStats(this.#state, prefill.currentRecorder);
|
|
2192
|
+
await prefill.currentRecorder.submitAndWait();
|
|
2193
|
+
await prefill.recordProfile(prefill.currentRecorder);
|
|
2194
|
+
} else {
|
|
2195
|
+
await getDevice().queue.onSubmittedWorkDone();
|
|
2196
|
+
}
|
|
2197
|
+
if (!activationTensor) {
|
|
2198
|
+
releaseBuffer(prefill.currentHiddenBuffer);
|
|
2199
|
+
throw new Error('native_lora_training_activation_not_captured.');
|
|
2200
|
+
}
|
|
2201
|
+
hiddenTensor = createTensor(
|
|
2202
|
+
prefill.currentHiddenBuffer,
|
|
2203
|
+
prefill.activationDtype,
|
|
2204
|
+
[prefill.numTokens, config.hiddenSize],
|
|
2205
|
+
`L${layerIdx}.native_lora_base_hidden`
|
|
2206
|
+
);
|
|
2207
|
+
this.resetToSeqLen(0);
|
|
2208
|
+
return {
|
|
2209
|
+
inputIds: tokens,
|
|
2210
|
+
layerIdx,
|
|
2211
|
+
module: 'down_proj',
|
|
2212
|
+
activation: activationTensor,
|
|
2213
|
+
baseHidden: hiddenTensor,
|
|
2214
|
+
dispose: releaseOwned,
|
|
2215
|
+
};
|
|
2216
|
+
} catch (error) {
|
|
2217
|
+
releaseOwned();
|
|
2218
|
+
this.resetToSeqLen(0);
|
|
2219
|
+
throw error;
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2104
2223
|
async prefillWithEmbedding(prompt, options = {}) {
|
|
2105
2224
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
2106
2225
|
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
@@ -2112,6 +2231,34 @@ export class PipelineGenerator {
|
|
|
2112
2231
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2113
2232
|
this.#state.stats.prefillProfileSteps = [];
|
|
2114
2233
|
const opts = resolvePrefillEmbeddingOptions(this.#state, options);
|
|
2234
|
+
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
2235
|
+
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
2236
|
+
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
2237
|
+
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
2238
|
+
const captureConfig = {
|
|
2239
|
+
...createDefaultCaptureConfig(),
|
|
2240
|
+
enabled: true,
|
|
2241
|
+
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
2242
|
+
...(options?.diagnostics?.captureConfig ?? {}),
|
|
2243
|
+
};
|
|
2244
|
+
validateCaptureConfig(captureConfig);
|
|
2245
|
+
this.#state.operatorDiagnostics = {
|
|
2246
|
+
enabled: diagnosticsEnabled === true,
|
|
2247
|
+
captureConfig,
|
|
2248
|
+
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
2249
|
+
modelHash: this.#state.manifest?.modelId ?? null,
|
|
2250
|
+
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
2251
|
+
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
2252
|
+
}) : null,
|
|
2253
|
+
tsirFixture: tsirFixtureCfg ? {
|
|
2254
|
+
dir: tsirFixtureCfg.dir,
|
|
2255
|
+
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
2256
|
+
? tsirFixtureCfg.layerFilter
|
|
2257
|
+
: null,
|
|
2258
|
+
records: [],
|
|
2259
|
+
} : null,
|
|
2260
|
+
};
|
|
2261
|
+
}
|
|
2115
2262
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
2116
2263
|
const inputStart = performance.now();
|
|
2117
2264
|
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
|
|
@@ -2178,12 +2325,32 @@ export class PipelineGenerator {
|
|
|
2178
2325
|
}
|
|
2179
2326
|
|
|
2180
2327
|
let embedding;
|
|
2328
|
+
let tokenEmbeddings = null;
|
|
2329
|
+
let logits = null;
|
|
2181
2330
|
let hiddenBytes = 0;
|
|
2182
2331
|
let readbackMs = 0;
|
|
2183
2332
|
let decodeHiddenMs = 0;
|
|
2184
2333
|
let finalNormMs = 0;
|
|
2185
2334
|
let extractMs = 0;
|
|
2335
|
+
let logitsMs = 0;
|
|
2186
2336
|
try {
|
|
2337
|
+
if (options.__returnSequenceLogits === true) {
|
|
2338
|
+
const logitsStart = performance.now();
|
|
2339
|
+
logits = await computeLogits(
|
|
2340
|
+
currentHiddenBuffer,
|
|
2341
|
+
numTokens,
|
|
2342
|
+
getLogitsWeights(this.#state),
|
|
2343
|
+
getLogitsConfig(this.#state),
|
|
2344
|
+
this.#state.useGPU,
|
|
2345
|
+
this.#state.debugFlags,
|
|
2346
|
+
undefined,
|
|
2347
|
+
undefined,
|
|
2348
|
+
this.#state.runtimeConfig.shared.debug.probes,
|
|
2349
|
+
{ lastPositionOnly: false },
|
|
2350
|
+
this.#state.operatorDiagnostics
|
|
2351
|
+
);
|
|
2352
|
+
logitsMs = performance.now() - logitsStart;
|
|
2353
|
+
}
|
|
2187
2354
|
const hiddenSize = config.hiddenSize;
|
|
2188
2355
|
hiddenBytes = numTokens * hiddenSize * activationBytes;
|
|
2189
2356
|
const readbackStart = performance.now();
|
|
@@ -2199,13 +2366,25 @@ export class PipelineGenerator {
|
|
|
2199
2366
|
const finalNormWeights = await this._getFinalNormWeights();
|
|
2200
2367
|
finalNormMs = performance.now() - finalNormStart;
|
|
2201
2368
|
const extractStart = performance.now();
|
|
2202
|
-
|
|
2369
|
+
if (options.__returnTokenEmbeddings === true) {
|
|
2370
|
+
tokenEmbeddings = this._extractTokenEmbeddingsFromHidden(
|
|
2371
|
+
hiddenStates,
|
|
2372
|
+
numTokens,
|
|
2373
|
+
hiddenSize,
|
|
2374
|
+
finalNormWeights,
|
|
2375
|
+
config
|
|
2376
|
+
);
|
|
2377
|
+
}
|
|
2378
|
+
embedding = extractEmbeddingFromHidden(
|
|
2203
2379
|
hiddenStates,
|
|
2204
2380
|
numTokens,
|
|
2205
2381
|
hiddenSize,
|
|
2206
2382
|
opts.embeddingMode,
|
|
2207
2383
|
finalNormWeights,
|
|
2208
|
-
config
|
|
2384
|
+
config,
|
|
2385
|
+
this.#state.embeddingPostprocessor,
|
|
2386
|
+
tokenEmbeddings,
|
|
2387
|
+
this.#state.weights.get('final_norm_bias') ?? null
|
|
2209
2388
|
);
|
|
2210
2389
|
extractMs = performance.now() - extractStart;
|
|
2211
2390
|
} finally {
|
|
@@ -2222,6 +2401,7 @@ export class PipelineGenerator {
|
|
|
2222
2401
|
decodeHiddenMs,
|
|
2223
2402
|
finalNormMs,
|
|
2224
2403
|
extractMs,
|
|
2404
|
+
logitsMs,
|
|
2225
2405
|
hiddenBytes,
|
|
2226
2406
|
tokens: numTokens,
|
|
2227
2407
|
activationDtype,
|
|
@@ -2241,6 +2421,8 @@ export class PipelineGenerator {
|
|
|
2241
2421
|
seqLen: this.#state.currentSeqLen,
|
|
2242
2422
|
tokens: inputIds,
|
|
2243
2423
|
embedding,
|
|
2424
|
+
tokenEmbeddings,
|
|
2425
|
+
logits,
|
|
2244
2426
|
embeddingMode: opts.embeddingMode,
|
|
2245
2427
|
phase,
|
|
2246
2428
|
linearAttention: null,
|
|
@@ -2257,12 +2439,22 @@ export class PipelineGenerator {
|
|
|
2257
2439
|
seqLen: this.#state.currentSeqLen,
|
|
2258
2440
|
tokens: inputIds,
|
|
2259
2441
|
embedding,
|
|
2442
|
+
tokenEmbeddings,
|
|
2443
|
+
logits,
|
|
2260
2444
|
embeddingMode: opts.embeddingMode,
|
|
2261
2445
|
phase,
|
|
2262
2446
|
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2263
2447
|
};
|
|
2264
2448
|
} finally {
|
|
2265
2449
|
this._closeFinitenessFallbackWindow(opts);
|
|
2450
|
+
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2451
|
+
? {
|
|
2452
|
+
enabled: true,
|
|
2453
|
+
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2454
|
+
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2455
|
+
}
|
|
2456
|
+
: null;
|
|
2457
|
+
this.#state.operatorDiagnostics = null;
|
|
2266
2458
|
}
|
|
2267
2459
|
}
|
|
2268
2460
|
|
|
@@ -3092,6 +3284,7 @@ export class PipelineGenerator {
|
|
|
3092
3284
|
debugCheckBuffer,
|
|
3093
3285
|
opts.executionPlan
|
|
3094
3286
|
);
|
|
3287
|
+
context.trainingCapture = opts?._trainingCapture ?? null;
|
|
3095
3288
|
context.currentTokenIds = inputIds;
|
|
3096
3289
|
context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
|
|
3097
3290
|
context.skipKVCacheWrites = returnHidden
|
|
@@ -2,7 +2,13 @@ import { getDevice } from '../../../gpu/device.js';
|
|
|
2
2
|
import { Q4K_BLOCK_BYTES, q4kBlockCount } from '../../../config/schema/index.js';
|
|
3
3
|
import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
|
|
4
4
|
import { getKernelPathMatmulVariant } from '../../../config/kernel-path-loader.js';
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
createWeightBuffer,
|
|
7
|
+
getWeightDtype,
|
|
8
|
+
isGpuBufferInstance,
|
|
9
|
+
isWeightBuffer,
|
|
10
|
+
tagBufferDtype,
|
|
11
|
+
} from '../../../gpu/weight-buffer.js';
|
|
6
12
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
7
13
|
import { log } from '../../../debug/index.js';
|
|
8
14
|
|
|
@@ -112,6 +118,26 @@ function formatsMatch(a, b) {
|
|
|
112
118
|
&& a?.bytesPerRow === b?.bytesPerRow;
|
|
113
119
|
}
|
|
114
120
|
|
|
121
|
+
function resolveBiasStorageFormat(weight, expectedElements) {
|
|
122
|
+
if (!weight || !Number.isFinite(expectedElements) || expectedElements <= 0) {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
const buffer = isWeightBuffer(weight) ? weight.buffer : weight;
|
|
126
|
+
if (!isGpuBufferInstance(buffer)) {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
const dtype = String(getWeightDtype(weight) ?? getWeightDtype(buffer) ?? '').toLowerCase();
|
|
130
|
+
if (dtype !== 'f16' && dtype !== 'f32') {
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
const bytesPerElement = dtype === 'f16' ? 2 : 4;
|
|
134
|
+
const byteLength = expectedElements * bytesPerElement;
|
|
135
|
+
if (buffer.size < byteLength) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
return { buffer, dtype, bytesPerElement, byteLength };
|
|
139
|
+
}
|
|
140
|
+
|
|
115
141
|
export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, options = {}) {
|
|
116
142
|
const device = getDevice();
|
|
117
143
|
if (!device) {
|
|
@@ -174,6 +200,17 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
174
200
|
continue;
|
|
175
201
|
}
|
|
176
202
|
|
|
203
|
+
const projectionBiases = [weights.qProjBias, weights.kProjBias, weights.vProjBias];
|
|
204
|
+
const presentProjectionBiases = projectionBiases.filter(Boolean).length;
|
|
205
|
+
if (presentProjectionBiases !== 0 && presentProjectionBiases !== projectionBiases.length) {
|
|
206
|
+
log.debug('QKV Fusion', `Layer ${l}: partial Q/K/V projection biases require split projections`);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
if (hasAttentionOutputGate && presentProjectionBiases > 0) {
|
|
210
|
+
log.debug('QKV Fusion', `Layer ${l}: gated Q projection biases require split projections`);
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
|
|
177
214
|
const qFormat = resolveProjectionStorageFormat(qProj, qProjSize, hiddenSize);
|
|
178
215
|
const kFormat = resolveProjectionStorageFormat(kProj, kSize, hiddenSize);
|
|
179
216
|
const vFormat = resolveProjectionStorageFormat(vProj, vSize, hiddenSize);
|
|
@@ -228,6 +265,32 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
228
265
|
})
|
|
229
266
|
: null;
|
|
230
267
|
|
|
268
|
+
let qBiasFormat = null;
|
|
269
|
+
let kBiasFormat = null;
|
|
270
|
+
let vBiasFormat = null;
|
|
271
|
+
let qkvBiasBuffer = null;
|
|
272
|
+
if (presentProjectionBiases === projectionBiases.length) {
|
|
273
|
+
qBiasFormat = resolveBiasStorageFormat(weights.qProjBias, qSize);
|
|
274
|
+
kBiasFormat = resolveBiasStorageFormat(weights.kProjBias, kSize);
|
|
275
|
+
vBiasFormat = resolveBiasStorageFormat(weights.vProjBias, vSize);
|
|
276
|
+
const biasFormatsMatch = qBiasFormat
|
|
277
|
+
&& kBiasFormat
|
|
278
|
+
&& vBiasFormat
|
|
279
|
+
&& qBiasFormat.dtype === kBiasFormat.dtype
|
|
280
|
+
&& qBiasFormat.dtype === vBiasFormat.dtype;
|
|
281
|
+
if (!biasFormatsMatch) {
|
|
282
|
+
log.debug('QKV Fusion', `Layer ${l}: inconsistent Q/K/V projection bias formats, skipping`);
|
|
283
|
+
qkvBuffer.destroy();
|
|
284
|
+
qGateBuffer?.destroy();
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
qkvBiasBuffer = device.createBuffer({
|
|
288
|
+
label: `layer_${l}_qkv_proj_bias`,
|
|
289
|
+
size: qBiasFormat.byteLength + kBiasFormat.byteLength + vBiasFormat.byteLength,
|
|
290
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
|
|
231
294
|
const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
|
|
232
295
|
if (hasAttentionOutputGate) {
|
|
233
296
|
for (let head = 0; head < numHeads; head++) {
|
|
@@ -266,6 +329,23 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
266
329
|
qkvBuffer, qBytes + kFormat.byteLength,
|
|
267
330
|
vFormat.byteLength
|
|
268
331
|
);
|
|
332
|
+
if (qkvBiasBuffer) {
|
|
333
|
+
encoder.copyBufferToBuffer(qBiasFormat.buffer, 0, qkvBiasBuffer, 0, qBiasFormat.byteLength);
|
|
334
|
+
encoder.copyBufferToBuffer(
|
|
335
|
+
kBiasFormat.buffer,
|
|
336
|
+
0,
|
|
337
|
+
qkvBiasBuffer,
|
|
338
|
+
qBiasFormat.byteLength,
|
|
339
|
+
kBiasFormat.byteLength
|
|
340
|
+
);
|
|
341
|
+
encoder.copyBufferToBuffer(
|
|
342
|
+
vBiasFormat.buffer,
|
|
343
|
+
0,
|
|
344
|
+
qkvBiasBuffer,
|
|
345
|
+
qBiasFormat.byteLength + kBiasFormat.byteLength,
|
|
346
|
+
vBiasFormat.byteLength
|
|
347
|
+
);
|
|
348
|
+
}
|
|
269
349
|
device.queue.submit([encoder.finish()]);
|
|
270
350
|
|
|
271
351
|
// Store fused buffer, sizes, and dtype
|
|
@@ -278,6 +358,10 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
278
358
|
);
|
|
279
359
|
weights.qkvSizes = [qSize, kSize, vSize];
|
|
280
360
|
weights.qkvDtype = dtype;
|
|
361
|
+
if (qkvBiasBuffer) {
|
|
362
|
+
tagBufferDtype(qkvBiasBuffer, qBiasFormat.dtype);
|
|
363
|
+
weights.qkvProjBias = qkvBiasBuffer;
|
|
364
|
+
}
|
|
281
365
|
if (qGateBuffer) {
|
|
282
366
|
weights.qGateProj = createWeightBuffer(
|
|
283
367
|
qGateBuffer,
|
|
@@ -237,7 +237,9 @@ export interface WeightLoadResult {
|
|
|
237
237
|
layerWeights: Map<string, LayerWeights>;
|
|
238
238
|
embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
239
239
|
lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
240
|
+
lmHeadBias: Float32Array | null;
|
|
240
241
|
finalNorm: GPUBuffer | Float32Array | null;
|
|
242
|
+
finalNormBias: Float32Array | null;
|
|
241
243
|
embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
|
|
242
244
|
diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
|
|
243
245
|
perLayerInputWeights: PerLayerInputWeights | null;
|
|
@@ -1020,7 +1020,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
1020
1020
|
layerWeights,
|
|
1021
1021
|
embeddings: dopplerLoader.embeddings,
|
|
1022
1022
|
lmHead: dopplerLoader.lmHead,
|
|
1023
|
+
lmHeadBias: dopplerLoader.lmHeadBias,
|
|
1023
1024
|
finalNorm: dopplerLoader.finalNorm,
|
|
1025
|
+
finalNormBias: dopplerLoader.finalNormBias,
|
|
1024
1026
|
embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
|
|
1025
1027
|
diffusionGemmaSelfConditioning: dopplerLoader.diffusionGemmaSelfConditioning,
|
|
1026
1028
|
perLayerInputWeights: dopplerLoader.perLayerInputWeights,
|
|
@@ -57,6 +57,7 @@ function shouldUseSandwichRMSNormPairFusion({
|
|
|
57
57
|
attnOutput,
|
|
58
58
|
inputTensor,
|
|
59
59
|
}) {
|
|
60
|
+
if (context.config?.normalizationType === 'layernorm') return false;
|
|
60
61
|
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
61
62
|
if (mergedSession?.useSandwichRMSNormPairFusion !== true) {
|
|
62
63
|
return false;
|
|
@@ -123,6 +124,7 @@ function shouldUsePostFfnNextInputRMSNormPairFusion({
|
|
|
123
124
|
activationDtype,
|
|
124
125
|
layerScalar,
|
|
125
126
|
}) {
|
|
127
|
+
if (config.normalizationType === 'layernorm') return false;
|
|
126
128
|
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
127
129
|
if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
|
|
128
130
|
return false;
|
|
@@ -176,6 +178,7 @@ function shouldUseStandardPostFfnNextInputRMSNormPairFusion({
|
|
|
176
178
|
layerScalar,
|
|
177
179
|
residualBranchScale,
|
|
178
180
|
}) {
|
|
181
|
+
if (config.normalizationType === 'layernorm') return false;
|
|
179
182
|
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
180
183
|
if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
|
|
181
184
|
return false;
|
|
@@ -851,6 +854,8 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
851
854
|
attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
|
|
852
855
|
queryPreAttnScalar: config.queryPreAttnScalar,
|
|
853
856
|
queryKeyNorm,
|
|
857
|
+
queryKeyNormType: config.queryKeyNormType,
|
|
858
|
+
queryKeyNormAxis: config.queryKeyNormAxis,
|
|
854
859
|
queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
|
|
855
860
|
valueNorm: config.valueNorm,
|
|
856
861
|
attentionOutputGate: config.attentionOutputGate,
|
|
@@ -860,6 +865,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
860
865
|
? (context.multimodalBidirectionalSpan ?? null)
|
|
861
866
|
: null,
|
|
862
867
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
868
|
+
normalizationType: config.normalizationType,
|
|
863
869
|
ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
|
|
864
870
|
ropeFrequencyBaseDim: resolveAttentionFrequencyBaseDim(config, layerType),
|
|
865
871
|
ropeInterleaved: config.ropeInterleaved,
|
|
@@ -6,6 +6,7 @@ import type { LinearNormMode } from '../../../config/schema/index.js';
|
|
|
6
6
|
import type { ProbeConfigSchema } from '../../../config/schema/index.js';
|
|
7
7
|
import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
|
|
8
8
|
import type { KernelPathSchema } from '../../../config/schema/kernel-path.schema.js';
|
|
9
|
+
import type { LoRAAdapter } from './lora.js';
|
|
9
10
|
|
|
10
11
|
export interface LinearLayerRuntimeState {
|
|
11
12
|
layerIdx: number;
|
|
@@ -76,6 +77,7 @@ export interface RunLinearAttentionLayerOptions {
|
|
|
76
77
|
precomputedInputNorm?: Tensor | null;
|
|
77
78
|
debugProbes?: ProbeConfigSchema[] | null;
|
|
78
79
|
recorder?: CommandRecorder | null;
|
|
80
|
+
lora?: LoRAAdapter | null;
|
|
79
81
|
}
|
|
80
82
|
|
|
81
83
|
export declare function hasLinearAttentionLayers(layerTypes: unknown): boolean;
|
|
@@ -23,6 +23,8 @@ import { dequantizeQ4KM } from '../../../converter/quantizer.js';
|
|
|
23
23
|
import { getKernelPathMatmulPrecision } from '../../../config/kernel-path-loader.js';
|
|
24
24
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
25
25
|
import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
|
|
26
|
+
import { applyLoRA } from './lora-apply.js';
|
|
27
|
+
import { getLoRAModule } from './lora.js';
|
|
26
28
|
|
|
27
29
|
const LINEAR_RUNTIME_SCHEMA_VERSION = 1;
|
|
28
30
|
const QK_L2NORM_EPS = 1e-6;
|
|
@@ -172,6 +174,38 @@ function releaseOrTrackBuffer(recorder, buffer) {
|
|
|
172
174
|
}
|
|
173
175
|
}
|
|
174
176
|
|
|
177
|
+
async function applyProjectionLoRA({
|
|
178
|
+
inputTensor,
|
|
179
|
+
baseOutput,
|
|
180
|
+
loraModule,
|
|
181
|
+
dims,
|
|
182
|
+
getWeightBuffer,
|
|
183
|
+
recorder,
|
|
184
|
+
kernelPath,
|
|
185
|
+
}) {
|
|
186
|
+
if (!loraModule) {
|
|
187
|
+
return baseOutput;
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
const combined = await applyLoRA(
|
|
191
|
+
inputTensor,
|
|
192
|
+
baseOutput,
|
|
193
|
+
loraModule,
|
|
194
|
+
dims,
|
|
195
|
+
getWeightBuffer,
|
|
196
|
+
recorder ?? undefined,
|
|
197
|
+
{ kernelPath }
|
|
198
|
+
);
|
|
199
|
+
if (combined.buffer !== baseOutput.buffer) {
|
|
200
|
+
releaseOrTrackBuffer(recorder, baseOutput.buffer);
|
|
201
|
+
}
|
|
202
|
+
return combined;
|
|
203
|
+
} catch (error) {
|
|
204
|
+
releaseOrTrackBuffer(recorder, baseOutput.buffer);
|
|
205
|
+
throw error;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
175
209
|
function releaseResolvedWeightBuffer(originalWeight, resolvedWeight, recorder) {
|
|
176
210
|
if (isResolvedWeightShared(originalWeight)) {
|
|
177
211
|
return;
|
|
@@ -637,6 +671,7 @@ async function projectLinearTensor({
|
|
|
637
671
|
getWeightBuffer,
|
|
638
672
|
recorder,
|
|
639
673
|
executionPolicies = null,
|
|
674
|
+
loraModule = null,
|
|
640
675
|
}) {
|
|
641
676
|
const resolvedWeight = getWeightBuffer(sourceWeight, role);
|
|
642
677
|
const resolvedInputDtype = resolveMatmulStepDtype(
|
|
@@ -702,7 +737,16 @@ async function projectLinearTensor({
|
|
|
702
737
|
outputDtype: resolvedOutputDtype,
|
|
703
738
|
executionPolicies,
|
|
704
739
|
});
|
|
705
|
-
|
|
740
|
+
const projected = await settleProjectionOutputDtype(result);
|
|
741
|
+
return await applyProjectionLoRA({
|
|
742
|
+
inputTensor: matmulInput,
|
|
743
|
+
baseOutput: projected,
|
|
744
|
+
loraModule,
|
|
745
|
+
dims: { M: numTokens, N: outDim, K: hiddenSize },
|
|
746
|
+
getWeightBuffer,
|
|
747
|
+
recorder,
|
|
748
|
+
kernelPath,
|
|
749
|
+
});
|
|
706
750
|
}
|
|
707
751
|
const result = await runMatmul(matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
|
|
708
752
|
transposeB: 'auto',
|
|
@@ -712,7 +756,16 @@ async function projectLinearTensor({
|
|
|
712
756
|
outputDtype: resolvedOutputDtype,
|
|
713
757
|
executionPolicies,
|
|
714
758
|
});
|
|
715
|
-
|
|
759
|
+
const projected = await settleProjectionOutputDtype(result);
|
|
760
|
+
return await applyProjectionLoRA({
|
|
761
|
+
inputTensor: matmulInput,
|
|
762
|
+
baseOutput: projected,
|
|
763
|
+
loraModule,
|
|
764
|
+
dims: { M: numTokens, N: outDim, K: hiddenSize },
|
|
765
|
+
getWeightBuffer,
|
|
766
|
+
recorder: null,
|
|
767
|
+
kernelPath,
|
|
768
|
+
});
|
|
716
769
|
} finally {
|
|
717
770
|
if (matmulInput !== inputTensor) {
|
|
718
771
|
releaseOrTrackBuffer(recorder, matmulInput.buffer);
|
|
@@ -835,6 +888,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
835
888
|
recorder,
|
|
836
889
|
executionPolicies = null,
|
|
837
890
|
precomputedInputNorm = null,
|
|
891
|
+
lora = null,
|
|
838
892
|
} = options;
|
|
839
893
|
|
|
840
894
|
if (!layerWeights) {
|
|
@@ -909,7 +963,12 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
909
963
|
}
|
|
910
964
|
}
|
|
911
965
|
|
|
912
|
-
const
|
|
966
|
+
const qkvLoRA = getLoRAModule(lora, layerIdx, 'in_proj_qkv');
|
|
967
|
+
const zLoRA = getLoRAModule(lora, layerIdx, 'in_proj_z');
|
|
968
|
+
const aLoRA = getLoRAModule(lora, layerIdx, 'in_proj_a');
|
|
969
|
+
const bLoRA = getLoRAModule(lora, layerIdx, 'in_proj_b');
|
|
970
|
+
const outLoRA = getLoRAModule(lora, layerIdx, 'out_proj');
|
|
971
|
+
const qkvzProjection = qkvLoRA || zLoRA ? null : resolveLinearAttentionQKVZProjection(layerWeights, {
|
|
913
972
|
phase,
|
|
914
973
|
numTokens,
|
|
915
974
|
hiddenSize,
|
|
@@ -951,6 +1010,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
951
1010
|
getWeightBuffer,
|
|
952
1011
|
recorder,
|
|
953
1012
|
executionPolicies,
|
|
1013
|
+
loraModule: qkvLoRA,
|
|
954
1014
|
});
|
|
955
1015
|
const zTensor = qkvzTensor ?? await projectLinearTensor({
|
|
956
1016
|
inputTensor: normedTensor,
|
|
@@ -966,8 +1026,9 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
966
1026
|
getWeightBuffer,
|
|
967
1027
|
recorder,
|
|
968
1028
|
executionPolicies,
|
|
1029
|
+
loraModule: zLoRA,
|
|
969
1030
|
});
|
|
970
|
-
const abProjection = resolveLinearAttentionABProjection(layerWeights, {
|
|
1031
|
+
const abProjection = aLoRA || bLoRA ? null : resolveLinearAttentionABProjection(layerWeights, {
|
|
971
1032
|
phase,
|
|
972
1033
|
numTokens,
|
|
973
1034
|
hiddenSize,
|
|
@@ -1007,6 +1068,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
1007
1068
|
getWeightBuffer,
|
|
1008
1069
|
recorder,
|
|
1009
1070
|
executionPolicies,
|
|
1071
|
+
loraModule: aLoRA,
|
|
1010
1072
|
});
|
|
1011
1073
|
const bTensor = abTensor ?? await projectLinearTensor({
|
|
1012
1074
|
inputTensor: normedTensor,
|
|
@@ -1022,6 +1084,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
1022
1084
|
getWeightBuffer,
|
|
1023
1085
|
recorder,
|
|
1024
1086
|
executionPolicies,
|
|
1087
|
+
loraModule: bLoRA,
|
|
1025
1088
|
});
|
|
1026
1089
|
|
|
1027
1090
|
const outProjInputDtype = resolveMatmulStepDtype(
|
|
@@ -1160,9 +1223,17 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
1160
1223
|
? (recorder ? await recordCastF32ToF16(recorder, result) : await castF32ToF16(result))
|
|
1161
1224
|
: (recorder ? await recordCastF16ToF32(recorder, result) : await castF16ToF32(result));
|
|
1162
1225
|
releaseOrTrackBuffer(recorder, result.buffer);
|
|
1163
|
-
|
|
1226
|
+
result = casted;
|
|
1164
1227
|
}
|
|
1165
|
-
return
|
|
1228
|
+
return await applyProjectionLoRA({
|
|
1229
|
+
inputTensor: coreTensor,
|
|
1230
|
+
baseOutput: result,
|
|
1231
|
+
loraModule: outLoRA,
|
|
1232
|
+
dims: { M: numTokens, N: hiddenSize, K: projectionLayout.valueDim },
|
|
1233
|
+
getWeightBuffer,
|
|
1234
|
+
recorder,
|
|
1235
|
+
kernelPath,
|
|
1236
|
+
});
|
|
1166
1237
|
} finally {
|
|
1167
1238
|
releaseOrTrackBuffer(recorder, coreTensor.buffer);
|
|
1168
1239
|
releaseResolvedWeightBuffer(layerWeights.oProj, outProjWeight, recorder);
|
|
@@ -25,6 +25,12 @@ export function rmsNormCPU(
|
|
|
25
25
|
eps?: number,
|
|
26
26
|
rmsNormWeightOffset?: boolean
|
|
27
27
|
): Float32Array;
|
|
28
|
+
export function layerNormCPU(
|
|
29
|
+
x: Float32Array,
|
|
30
|
+
weight: Float32Array,
|
|
31
|
+
bias: Float32Array,
|
|
32
|
+
eps?: number
|
|
33
|
+
): Float32Array;
|
|
28
34
|
|
|
29
35
|
/**
|
|
30
36
|
* Convert a single float16 value to float32.
|