doppler-gpu 0.4.10 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +40 -1
- package/models/catalog.json +637 -32
- package/package.json +24 -3
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +135 -13
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +69 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +73 -11
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/adapter.schema.d.ts +6 -1
- package/src/config/schema/adapter.schema.js +5 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/artifact-contract.d.ts +83 -0
- package/src/experimental/adapters/artifact-contract.js +254 -0
- package/src/experimental/adapters/index.d.ts +25 -0
- package/src/experimental/adapters/index.js +12 -0
- package/src/experimental/adapters/lora-loader.js +24 -4
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/experimental/training/artifacts.d.ts +160 -0
- package/src/experimental/training/attention-backward.d.ts +64 -0
- package/src/experimental/training/attention-forward.d.ts +22 -0
- package/src/experimental/training/attention-forward.js +82 -0
- package/src/experimental/training/autograd.d.ts +76 -0
- package/src/experimental/training/checkpoint.d.ts +36 -0
- package/src/experimental/training/clip.d.ts +9 -0
- package/src/experimental/training/dataloader.d.ts +8 -0
- package/src/experimental/training/distillation/student-fixture.js +1 -0
- package/src/experimental/training/export.d.ts +41 -0
- package/src/experimental/training/lora-pipeline.d.ts +186 -0
- package/src/experimental/training/lora-pipeline.js +5 -0
- package/src/experimental/training/lora.d.ts +20 -0
- package/src/experimental/training/lora.js +4 -3
- package/src/experimental/training/loss-scaling.d.ts +21 -0
- package/src/experimental/training/loss.d.ts +10 -0
- package/src/experimental/training/native-qwen-lora.d.ts +106 -0
- package/src/experimental/training/native-qwen-lora.js +449 -0
- package/src/experimental/training/objectives/base.d.ts +58 -0
- package/src/experimental/training/optimizer.d.ts +22 -0
- package/src/experimental/training/optimizer.js +1 -0
- package/src/experimental/training/runner.d.ts +249 -0
- package/src/experimental/training/trainer.d.ts +89 -0
- package/src/experimental/training/workloads.d.ts +205 -0
- package/src/gpu/kernels/backward/adam.d.ts +1 -0
- package/src/gpu/kernels/backward/adam.js +6 -2
- package/src/gpu/kernels/backward/adam.wgsl +3 -2
- package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
- package/src/gpu/kernels/backward/utils.js +18 -4
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +205 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.d.ts +5 -1
- package/src/inference/pipelines/text/generator.js +196 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
- package/src/inference/pipelines/text/linear-attention.js +77 -6
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/lora-types.d.ts +6 -1
- package/src/inference/pipelines/text/lora-types.js +5 -0
- package/src/inference/pipelines/text/ops.js +33 -0
- package/src/inference/pipelines/text/types.d.ts +66 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +9 -2
- package/src/inference/pipelines/text.js +116 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/models/diffusiongemma.js +1 -1
- package/src/models/embeddinggemma.js +1 -1
- package/src/models/gemma3.js +1 -1
- package/src/models/gemma4.js +1 -1
- package/src/models/qwen3.js +1 -1
- package/src/tooling/hf-registry-utils.js +1 -1
- package/src/tooling/node-browser-command-runner.d.ts +16 -0
- package/src/tooling/node-browser-command-runner.js +1 -1
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/training-browser.d.ts +19 -0
- package/src/training-browser.js +13 -0
- package/src/training.d.ts +137 -0
- package/src/training.js +219 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +9 -1
- package/tests/kernels/browser/test-page.js +4 -2
- package/tests/kernels/reference/adam.js +4 -2
- package/NOTICE +0 -5
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
CausalLmLoraTrainer,
|
|
3
|
+
CausalLmLoraTrainerResult,
|
|
4
|
+
} from './experimental/training/lora-pipeline.js';
|
|
5
|
+
import type {
|
|
6
|
+
LoadedTrainingWorkload,
|
|
7
|
+
TrainingWorkloadPack,
|
|
8
|
+
} from './experimental/training/workloads.js';
|
|
9
|
+
|
|
10
|
+
export { AutogradTape, OpType } from './experimental/training/autograd.js';
|
|
11
|
+
export { buildAttentionSoftmaxCache } from './experimental/training/attention-backward.js';
|
|
12
|
+
export { recordAttentionForward } from './experimental/training/attention-forward.js';
|
|
13
|
+
export { LoraAdapter } from './experimental/training/lora.js';
|
|
14
|
+
export { AdamOptimizer } from './experimental/training/optimizer.js';
|
|
15
|
+
export { trainStep } from './experimental/training/trainer.js';
|
|
16
|
+
export { crossEntropyLoss } from './experimental/training/loss.js';
|
|
17
|
+
export { clipGradients } from './experimental/training/clip.js';
|
|
18
|
+
export { exportLoRAAdapter, serializeLoRASafetensors } from './experimental/training/export.js';
|
|
19
|
+
export { DynamicLossScaler, detectOverflow } from './experimental/training/loss-scaling.js';
|
|
20
|
+
export { TrainingRunner, runTraining } from './experimental/training/runner.js';
|
|
21
|
+
export { DataLoader } from './experimental/training/dataloader.js';
|
|
22
|
+
export { saveCheckpoint, loadCheckpoint } from './experimental/training/checkpoint.js';
|
|
23
|
+
export {
|
|
24
|
+
NativeQwenLoRATrainer,
|
|
25
|
+
createNativeQwenLoRATrainer,
|
|
26
|
+
loadNativeQwenTrainingPipeline,
|
|
27
|
+
trainNativeQwenSftLoRA,
|
|
28
|
+
} from './experimental/training/native-qwen-lora.js';
|
|
29
|
+
export type {
|
|
30
|
+
NativeQwenLoRATrainerOptions,
|
|
31
|
+
NativeQwenLoRACheckpoint,
|
|
32
|
+
NativeQwenLoRAStepResult,
|
|
33
|
+
NativeQwenSftLoRAOptions,
|
|
34
|
+
} from './experimental/training/native-qwen-lora.js';
|
|
35
|
+
export {
|
|
36
|
+
LORA_RUNNER_BASE_MODEL_REGISTRY,
|
|
37
|
+
LORA_RUNNER_DATASET_FORMAT_REGISTRY,
|
|
38
|
+
LORA_RUNNER_SUPPORT_CONTRACT,
|
|
39
|
+
compareLoraRun,
|
|
40
|
+
evaluateLoraCheckpoint,
|
|
41
|
+
exportLoraCheckpoint,
|
|
42
|
+
getLoraRunnerCompatibility,
|
|
43
|
+
qualityGateLoraRun,
|
|
44
|
+
watchLoraCheckpoints,
|
|
45
|
+
} from './experimental/training/lora-pipeline.js';
|
|
46
|
+
export {
|
|
47
|
+
loadTrainingWorkloadPack,
|
|
48
|
+
normalizeTrainingWorkloadPack,
|
|
49
|
+
serializeTrainingWorkloadLock,
|
|
50
|
+
} from './experimental/training/workloads.js';
|
|
51
|
+
export type {
|
|
52
|
+
CausalLmLoraTrainer,
|
|
53
|
+
CausalLmLoraTrainerInput,
|
|
54
|
+
CausalLmLoraTrainerResult,
|
|
55
|
+
CausalLmLoraTrainerTensor,
|
|
56
|
+
} from './experimental/training/lora-pipeline.js';
|
|
57
|
+
export type {
|
|
58
|
+
LoadedTrainingWorkload,
|
|
59
|
+
LoRAWorkloadPipelineConfig,
|
|
60
|
+
TrainingWorkloadPack,
|
|
61
|
+
} from './experimental/training/workloads.js';
|
|
62
|
+
|
|
63
|
+
export type TrainingBackend = 'webgpu_native' | 'external';
|
|
64
|
+
|
|
65
|
+
export interface TrainingBackendCapability {
|
|
66
|
+
id: TrainingBackend;
|
|
67
|
+
supported: boolean;
|
|
68
|
+
blockedReasons: readonly string[];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface TrainingCapabilities {
|
|
72
|
+
schemaVersion: 1;
|
|
73
|
+
scope: 'completion_masked_sft_lora';
|
|
74
|
+
supported: boolean;
|
|
75
|
+
operatorSurfaces: readonly ['browser', 'node', 'bun'];
|
|
76
|
+
runnerKey: string;
|
|
77
|
+
baseModelId: string;
|
|
78
|
+
baseModelFamily: string | null;
|
|
79
|
+
datasetFormat: string;
|
|
80
|
+
taskType: string;
|
|
81
|
+
backends: Readonly<{
|
|
82
|
+
webgpuNative: TrainingBackendCapability;
|
|
83
|
+
external: TrainingBackendCapability;
|
|
84
|
+
}>;
|
|
85
|
+
adapterExport: Readonly<{
|
|
86
|
+
format: 'safetensors';
|
|
87
|
+
manifest: 'rdrr_lora_adapter';
|
|
88
|
+
runtimeLoadable: true;
|
|
89
|
+
}>;
|
|
90
|
+
compatibility: ReturnType<typeof import('./experimental/training/lora-pipeline.js').getLoraRunnerCompatibility>;
|
|
91
|
+
nativeTarget: Readonly<{ module: 'down_proj'; layer: 'last' }> | null;
|
|
92
|
+
blockedReasons: readonly string[];
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export interface TrainSftLoRAOptions {
|
|
96
|
+
backend: TrainingBackend;
|
|
97
|
+
loadedWorkload: LoadedTrainingWorkload;
|
|
98
|
+
runRoot?: string | null;
|
|
99
|
+
timestamp?: string | Date | null;
|
|
100
|
+
causalLmTrainer?: CausalLmLoraTrainer;
|
|
101
|
+
fetch?: (url: string) => Promise<string>;
|
|
102
|
+
readFile?: (path: string) => Promise<string>;
|
|
103
|
+
pipeline?: import('./inference/pipelines/text.js').InferencePipeline;
|
|
104
|
+
samples?: Array<{
|
|
105
|
+
inputIds: readonly number[];
|
|
106
|
+
targetIds: readonly number[];
|
|
107
|
+
supervisedTokenCount: number;
|
|
108
|
+
}>;
|
|
109
|
+
export?: { id: string; name: string; weightsPath: string } | null;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export declare const TRAINING_BACKENDS: readonly TrainingBackend[];
|
|
113
|
+
|
|
114
|
+
export declare function bootstrapNativeTrainingHost(): Promise<{
|
|
115
|
+
ok: true;
|
|
116
|
+
provider: string;
|
|
117
|
+
detail: null;
|
|
118
|
+
}>;
|
|
119
|
+
|
|
120
|
+
export declare function releaseNativeTrainingHost(): Promise<{
|
|
121
|
+
released: boolean;
|
|
122
|
+
provider: string | null;
|
|
123
|
+
reason: string | null;
|
|
124
|
+
}>;
|
|
125
|
+
|
|
126
|
+
export declare function getTrainingCapabilities(
|
|
127
|
+
workload: TrainingWorkloadPack | Record<string, unknown>
|
|
128
|
+
): TrainingCapabilities;
|
|
129
|
+
|
|
130
|
+
export declare function assertTrainingBackend(
|
|
131
|
+
workload: TrainingWorkloadPack | Record<string, unknown>,
|
|
132
|
+
backend: TrainingBackend
|
|
133
|
+
): TrainingCapabilities;
|
|
134
|
+
|
|
135
|
+
export declare function trainSftLoRA(
|
|
136
|
+
options: TrainSftLoRAOptions
|
|
137
|
+
): Promise<CausalLmLoraTrainerResult | Record<string, unknown>>;
|
package/src/training.js
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import {
|
|
2
|
+
LORA_RUNNER_BASE_MODEL_REGISTRY,
|
|
3
|
+
LORA_RUNNER_DATASET_FORMAT_REGISTRY,
|
|
4
|
+
LORA_RUNNER_SUPPORT_CONTRACT,
|
|
5
|
+
compareLoraRun,
|
|
6
|
+
evaluateLoraCheckpoint,
|
|
7
|
+
exportLoraCheckpoint,
|
|
8
|
+
getLoraRunnerCompatibility,
|
|
9
|
+
qualityGateLoraRun,
|
|
10
|
+
runLoraPipeline,
|
|
11
|
+
watchLoraCheckpoints,
|
|
12
|
+
} from './experimental/training/lora-pipeline.js';
|
|
13
|
+
import {
|
|
14
|
+
loadTrainingWorkloadPack,
|
|
15
|
+
normalizeTrainingWorkloadPack,
|
|
16
|
+
serializeTrainingWorkloadLock,
|
|
17
|
+
} from './experimental/training/workloads.js';
|
|
18
|
+
import { trainNativeQwenSftLoRA } from './experimental/training/native-qwen-lora.js';
|
|
19
|
+
|
|
20
|
+
export { AutogradTape, OpType } from './experimental/training/autograd.js';
|
|
21
|
+
export { buildAttentionSoftmaxCache } from './experimental/training/attention-backward.js';
|
|
22
|
+
export { recordAttentionForward } from './experimental/training/attention-forward.js';
|
|
23
|
+
export { LoraAdapter } from './experimental/training/lora.js';
|
|
24
|
+
export { AdamOptimizer } from './experimental/training/optimizer.js';
|
|
25
|
+
export { trainStep } from './experimental/training/trainer.js';
|
|
26
|
+
export { crossEntropyLoss } from './experimental/training/loss.js';
|
|
27
|
+
export { clipGradients } from './experimental/training/clip.js';
|
|
28
|
+
export { exportLoRAAdapter, serializeLoRASafetensors } from './experimental/training/export.js';
|
|
29
|
+
export { DynamicLossScaler, detectOverflow } from './experimental/training/loss-scaling.js';
|
|
30
|
+
export { TrainingRunner, runTraining } from './experimental/training/runner.js';
|
|
31
|
+
export { DataLoader } from './experimental/training/dataloader.js';
|
|
32
|
+
export { saveCheckpoint, loadCheckpoint } from './experimental/training/checkpoint.js';
|
|
33
|
+
export {
|
|
34
|
+
NativeQwenLoRATrainer,
|
|
35
|
+
createNativeQwenLoRATrainer,
|
|
36
|
+
loadNativeQwenTrainingPipeline,
|
|
37
|
+
trainNativeQwenSftLoRA,
|
|
38
|
+
} from './experimental/training/native-qwen-lora.js';
|
|
39
|
+
|
|
40
|
+
export {
|
|
41
|
+
LORA_RUNNER_BASE_MODEL_REGISTRY,
|
|
42
|
+
LORA_RUNNER_DATASET_FORMAT_REGISTRY,
|
|
43
|
+
LORA_RUNNER_SUPPORT_CONTRACT,
|
|
44
|
+
compareLoraRun,
|
|
45
|
+
evaluateLoraCheckpoint,
|
|
46
|
+
exportLoraCheckpoint,
|
|
47
|
+
getLoraRunnerCompatibility,
|
|
48
|
+
loadTrainingWorkloadPack,
|
|
49
|
+
normalizeTrainingWorkloadPack,
|
|
50
|
+
qualityGateLoraRun,
|
|
51
|
+
serializeTrainingWorkloadLock,
|
|
52
|
+
watchLoraCheckpoints,
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
export const TRAINING_BACKENDS = Object.freeze(['webgpu_native', 'external']);
|
|
56
|
+
|
|
57
|
+
export async function bootstrapNativeTrainingHost() {
|
|
58
|
+
const [{ installNodeFileFetchShim }, { bootstrapNodeWebGPU }] = await Promise.all([
|
|
59
|
+
import('./tooling/node-file-fetch.js'),
|
|
60
|
+
import('./tooling/node-webgpu.js'),
|
|
61
|
+
]);
|
|
62
|
+
installNodeFileFetchShim();
|
|
63
|
+
const result = await bootstrapNodeWebGPU();
|
|
64
|
+
if (!result.ok) {
|
|
65
|
+
throw new Error(`native_training_webgpu_bootstrap_failed: ${result.detail || 'no WebGPU provider'}.`);
|
|
66
|
+
}
|
|
67
|
+
return result;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export async function releaseNativeTrainingHost() {
|
|
71
|
+
const [{ destroyDevice, resetDeviceState }, { releaseNodeWebGPU }] = await Promise.all([
|
|
72
|
+
import('./gpu/device.js'),
|
|
73
|
+
import('./tooling/node-webgpu.js'),
|
|
74
|
+
]);
|
|
75
|
+
destroyDevice();
|
|
76
|
+
resetDeviceState();
|
|
77
|
+
return releaseNodeWebGPU();
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function getWorkloadPipeline(workload) {
|
|
81
|
+
return workload?.pipeline || workload?.lora || {};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function buildBackendCapability(id, supported, blockedReasons) {
|
|
85
|
+
return Object.freeze({
|
|
86
|
+
id,
|
|
87
|
+
supported,
|
|
88
|
+
blockedReasons: Object.freeze(supported ? [] : [...new Set(blockedReasons)]),
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function resolveNativeLoraTarget(pipeline, compatibility) {
|
|
93
|
+
const requested = pipeline.nativeTarget;
|
|
94
|
+
const modules = Array.isArray(pipeline.adapter?.targetModules)
|
|
95
|
+
? pipeline.adapter.targetModules.map((value) => String(value))
|
|
96
|
+
: [];
|
|
97
|
+
if (!requested || modules.length !== 1 || modules[0] !== requested.module) return null;
|
|
98
|
+
const supported = compatibility.observed.nativeLoraTargets.find((target) => (
|
|
99
|
+
target.module === requested.module && target.layer === requested.layer
|
|
100
|
+
));
|
|
101
|
+
return supported ? Object.freeze({ ...supported }) : null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export function getTrainingCapabilities(workload) {
|
|
105
|
+
const compatibility = getLoraRunnerCompatibility(workload);
|
|
106
|
+
const pipeline = getWorkloadPipeline(workload);
|
|
107
|
+
const blockedReasons = [...compatibility.blockedReasons];
|
|
108
|
+
if (workload?.kind !== 'lora') {
|
|
109
|
+
blockedReasons.push('workload_kind_must_be_lora');
|
|
110
|
+
}
|
|
111
|
+
if (pipeline.datasetFormat !== 'text-pairs') {
|
|
112
|
+
blockedReasons.push('dataset_format_must_be_text_pairs');
|
|
113
|
+
}
|
|
114
|
+
if (pipeline.taskType !== 'text_generation') {
|
|
115
|
+
blockedReasons.push('task_type_must_be_text_generation');
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const sftLoraSupported = blockedReasons.length === 0;
|
|
119
|
+
const nativeBlockedReasons = [...blockedReasons];
|
|
120
|
+
const nativeTarget = resolveNativeLoraTarget(pipeline, compatibility);
|
|
121
|
+
if (compatibility.observed.requiresExternalTrainer && !nativeTarget) {
|
|
122
|
+
nativeBlockedReasons.push('native_full_graph_runner_unavailable_for_base_model');
|
|
123
|
+
}
|
|
124
|
+
const webgpuNativeSupported = nativeBlockedReasons.length === 0;
|
|
125
|
+
|
|
126
|
+
return Object.freeze({
|
|
127
|
+
schemaVersion: 1,
|
|
128
|
+
scope: 'completion_masked_sft_lora',
|
|
129
|
+
supported: sftLoraSupported,
|
|
130
|
+
operatorSurfaces: Object.freeze(['browser', 'node', 'bun']),
|
|
131
|
+
runnerKey: compatibility.observed.runnerKey,
|
|
132
|
+
baseModelId: compatibility.observed.baseModelId,
|
|
133
|
+
baseModelFamily: compatibility.observed.baseModelFamily,
|
|
134
|
+
datasetFormat: compatibility.observed.datasetFormat,
|
|
135
|
+
taskType: compatibility.observed.taskType,
|
|
136
|
+
backends: Object.freeze({
|
|
137
|
+
webgpuNative: buildBackendCapability(
|
|
138
|
+
'webgpu_native',
|
|
139
|
+
webgpuNativeSupported,
|
|
140
|
+
nativeBlockedReasons
|
|
141
|
+
),
|
|
142
|
+
external: buildBackendCapability('external', sftLoraSupported, blockedReasons),
|
|
143
|
+
}),
|
|
144
|
+
adapterExport: Object.freeze({
|
|
145
|
+
format: 'safetensors',
|
|
146
|
+
manifest: 'rdrr_lora_adapter',
|
|
147
|
+
runtimeLoadable: true,
|
|
148
|
+
}),
|
|
149
|
+
compatibility,
|
|
150
|
+
nativeTarget,
|
|
151
|
+
blockedReasons: Object.freeze([...new Set(blockedReasons)]),
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function assertTrainingBackend(workload, backend) {
|
|
156
|
+
if (!TRAINING_BACKENDS.includes(backend)) {
|
|
157
|
+
throw new Error(`training_backend_invalid: expected one of ${TRAINING_BACKENDS.join(', ')}, got ${String(backend)}.`);
|
|
158
|
+
}
|
|
159
|
+
const capabilities = getTrainingCapabilities(workload);
|
|
160
|
+
const selected = backend === 'webgpu_native'
|
|
161
|
+
? capabilities.backends.webgpuNative
|
|
162
|
+
: capabilities.backends.external;
|
|
163
|
+
if (!selected.supported) {
|
|
164
|
+
throw new Error(
|
|
165
|
+
`training_backend_not_supported: backend=${backend} runnerKey=${capabilities.runnerKey} `
|
|
166
|
+
+ `blockedReasons=${selected.blockedReasons.join(',')}.`
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
return capabilities;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export async function trainSftLoRA(options) {
|
|
173
|
+
if (!options || typeof options !== 'object') {
|
|
174
|
+
throw new Error('trainSftLoRA requires an options object.');
|
|
175
|
+
}
|
|
176
|
+
const loadedWorkload = options.loadedWorkload;
|
|
177
|
+
if (!loadedWorkload?.workload) {
|
|
178
|
+
throw new Error('trainSftLoRA requires options.loadedWorkload.');
|
|
179
|
+
}
|
|
180
|
+
const backend = options.backend;
|
|
181
|
+
const capabilities = assertTrainingBackend(loadedWorkload.workload, backend);
|
|
182
|
+
|
|
183
|
+
const pipeline = getWorkloadPipeline(loadedWorkload.workload);
|
|
184
|
+
const externalTrainerConfigured = typeof options.causalLmTrainer === 'function'
|
|
185
|
+
|| Boolean(pipeline.trainer);
|
|
186
|
+
if (backend === 'external' && !externalTrainerConfigured) {
|
|
187
|
+
throw new Error(
|
|
188
|
+
'external_training_backend_not_configured: provide options.causalLmTrainer or workload.pipeline.trainer.'
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
if (backend === 'webgpu_native' && externalTrainerConfigured) {
|
|
192
|
+
throw new Error(
|
|
193
|
+
'training_backend_mismatch: webgpu_native cannot be combined with an external trainer configuration.'
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
if (backend === 'webgpu_native' && capabilities.nativeTarget) {
|
|
198
|
+
if (!options.pipeline) {
|
|
199
|
+
throw new Error('native_qwen_lora_pipeline_required: provide the loaded Doppler pipeline.');
|
|
200
|
+
}
|
|
201
|
+
return trainNativeQwenSftLoRA({
|
|
202
|
+
pipeline: options.pipeline,
|
|
203
|
+
baseModelId: capabilities.baseModelId,
|
|
204
|
+
layerIdx: options.pipeline.modelConfig.numLayers - 1,
|
|
205
|
+
module: capabilities.nativeTarget.module,
|
|
206
|
+
rank: pipeline.adapter.rank,
|
|
207
|
+
alpha: pipeline.adapter.alpha,
|
|
208
|
+
optimizer: loadedWorkload.workload.training.optimizer,
|
|
209
|
+
gradient: loadedWorkload.workload.training.gradientClipping,
|
|
210
|
+
precision: loadedWorkload.workload.training.precision,
|
|
211
|
+
samples: options.samples,
|
|
212
|
+
maxSteps: loadedWorkload.workload.training.steps,
|
|
213
|
+
export: options.export || null,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const { backend: _backend, ...pipelineOptions } = options;
|
|
218
|
+
return runLoraPipeline(pipelineOptions);
|
|
219
|
+
}
|
package/src/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export const DOPPLER_VERSION = '0.4.
|
|
1
|
+
export const DOPPLER_VERSION = '0.4.14';
|
|
2
2
|
export const DOPPLER_PROVIDER_VERSION = DOPPLER_VERSION;
|
|
@@ -1562,7 +1562,15 @@ export async function runKernelSuite(harness) {
|
|
|
1562
1562
|
const grads = h.generateTestData(count, 1902);
|
|
1563
1563
|
const moment1 = h.generateTestData(count, 1903);
|
|
1564
1564
|
const moment2 = h.generateTestData(count, 1904);
|
|
1565
|
-
const options = {
|
|
1565
|
+
const options = {
|
|
1566
|
+
count,
|
|
1567
|
+
step: 2,
|
|
1568
|
+
lr: 0.001,
|
|
1569
|
+
beta1: 0.9,
|
|
1570
|
+
beta2: 0.999,
|
|
1571
|
+
eps: 1e-8,
|
|
1572
|
+
weightDecay: 0.01,
|
|
1573
|
+
};
|
|
1566
1574
|
const expected = adamRef(params, grads, moment1, moment2, options);
|
|
1567
1575
|
const actual = await h.runAdam(null, params, grads, moment1, moment2, options);
|
|
1568
1576
|
const pPass = h.compareArrays(expected.params, actual.params, h.KERNEL_TOLERANCES.residual).passed;
|
|
@@ -2156,14 +2156,16 @@ const testHarness = {
|
|
|
2156
2156
|
const outParams = new Float32Array(params);
|
|
2157
2157
|
const outM1 = new Float32Array(moment1);
|
|
2158
2158
|
const outM2 = new Float32Array(moment2);
|
|
2159
|
-
const { count, step, lr, beta1, beta2, eps } = options;
|
|
2159
|
+
const { count, step, lr, beta1, beta2, eps, weightDecay = 0 } = options;
|
|
2160
2160
|
for (let i = 0; i < count; i++) {
|
|
2161
2161
|
const g = grads[i];
|
|
2162
2162
|
outM1[i] = beta1 * outM1[i] + (1 - beta1) * g;
|
|
2163
2163
|
outM2[i] = beta2 * outM2[i] + (1 - beta2) * g * g;
|
|
2164
2164
|
const mHat = outM1[i] / (1 - Math.pow(beta1, step));
|
|
2165
2165
|
const vHat = outM2[i] / (1 - Math.pow(beta2, step));
|
|
2166
|
-
outParams[i] = outParams[i] - lr *
|
|
2166
|
+
outParams[i] = outParams[i] - lr * (
|
|
2167
|
+
mHat / (Math.sqrt(vHat) + eps) + weightDecay * outParams[i]
|
|
2168
|
+
);
|
|
2167
2169
|
}
|
|
2168
2170
|
return { params: outParams, moment1: outM1, moment2: outM2 };
|
|
2169
2171
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export function adamRef(params, grads, moment1, moment2, options) {
|
|
2
|
-
const { count, step, lr, beta1, beta2, eps } = options;
|
|
2
|
+
const { count, step, lr, beta1, beta2, eps, weightDecay = 0 } = options;
|
|
3
3
|
const outParams = new Float32Array(params);
|
|
4
4
|
const outM1 = new Float32Array(moment1);
|
|
5
5
|
const outM2 = new Float32Array(moment2);
|
|
@@ -9,7 +9,9 @@ export function adamRef(params, grads, moment1, moment2, options) {
|
|
|
9
9
|
outM2[i] = beta2 * outM2[i] + (1 - beta2) * g * g;
|
|
10
10
|
const mHat = outM1[i] / (1 - Math.pow(beta1, step));
|
|
11
11
|
const vHat = outM2[i] / (1 - Math.pow(beta2, step));
|
|
12
|
-
outParams[i] = outParams[i] - lr *
|
|
12
|
+
outParams[i] = outParams[i] - lr * (
|
|
13
|
+
mHat / (Math.sqrt(vHat) + eps) + weightDecay * outParams[i]
|
|
14
|
+
);
|
|
13
15
|
}
|
|
14
16
|
return { params: outParams, moment1: outM1, moment2: outM2 };
|
|
15
17
|
}
|