doppler-gpu 0.4.11 → 0.4.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/README.md +79 -45
- package/models/catalog.json +29 -29
- package/package.json +12 -3
- package/src/client/doppler-registry.json +9 -9
- package/src/client/runtime/lora.js +5 -0
- package/src/client/runtime/model-session.d.ts +69 -1
- package/src/client/runtime/model-session.js +231 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +21 -2
- package/src/config/schema/adapter.schema.d.ts +6 -1
- package/src/config/schema/adapter.schema.js +5 -0
- package/src/experimental/adapters/artifact-contract.d.ts +83 -0
- package/src/experimental/adapters/artifact-contract.js +254 -0
- package/src/experimental/adapters/index.d.ts +25 -0
- package/src/experimental/adapters/index.js +12 -0
- package/src/experimental/adapters/lora-loader.js +9 -4
- package/src/experimental/training/artifacts.d.ts +160 -0
- package/src/experimental/training/attention-backward.d.ts +64 -0
- package/src/experimental/training/attention-forward.d.ts +22 -0
- package/src/experimental/training/attention-forward.js +82 -0
- package/src/experimental/training/autograd.d.ts +76 -0
- package/src/experimental/training/checkpoint.d.ts +36 -0
- package/src/experimental/training/clip.d.ts +9 -0
- package/src/experimental/training/dataloader.d.ts +8 -0
- package/src/experimental/training/distillation/student-fixture.js +1 -0
- package/src/experimental/training/export.d.ts +41 -0
- package/src/experimental/training/lora-pipeline.d.ts +186 -0
- package/src/experimental/training/lora-pipeline.js +5 -0
- package/src/experimental/training/lora.d.ts +20 -0
- package/src/experimental/training/lora.js +4 -3
- package/src/experimental/training/loss-scaling.d.ts +21 -0
- package/src/experimental/training/loss.d.ts +10 -0
- package/src/experimental/training/native-qwen-lora.d.ts +106 -0
- package/src/experimental/training/native-qwen-lora.js +449 -0
- package/src/experimental/training/objectives/base.d.ts +58 -0
- package/src/experimental/training/optimizer.d.ts +22 -0
- package/src/experimental/training/optimizer.js +1 -0
- package/src/experimental/training/runner.d.ts +249 -0
- package/src/experimental/training/trainer.d.ts +89 -0
- package/src/experimental/training/workloads.d.ts +205 -0
- package/src/gpu/kernels/backward/adam.d.ts +1 -0
- package/src/gpu/kernels/backward/adam.js +6 -2
- package/src/gpu/kernels/backward/adam.wgsl +3 -2
- package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
- package/src/gpu/kernels/backward/utils.js +18 -4
- package/src/inference/pipelines/text/ffn/dense.js +16 -0
- package/src/inference/pipelines/text/generator.d.ts +5 -1
- package/src/inference/pipelines/text/generator.js +106 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
- package/src/inference/pipelines/text/linear-attention.js +77 -6
- package/src/inference/pipelines/text/lora-types.d.ts +6 -1
- package/src/inference/pipelines/text/lora-types.js +5 -0
- package/src/inference/pipelines/text/ops.js +1 -0
- package/src/inference/pipelines/text/types.d.ts +28 -0
- package/src/inference/pipelines/text.d.ts +7 -2
- package/src/inference/pipelines/text.js +4 -0
- package/src/models/diffusiongemma.js +1 -1
- package/src/models/embeddinggemma.js +1 -1
- package/src/models/gemma3.js +1 -1
- package/src/models/gemma4.js +1 -1
- package/src/models/qwen3.js +1 -1
- package/src/tooling/hf-registry-utils.js +1 -1
- package/src/tooling/node-browser-command-runner.d.ts +16 -0
- package/src/tooling/node-browser-command-runner.js +1 -1
- package/src/training-browser.d.ts +19 -0
- package/src/training-browser.js +13 -0
- package/src/training.d.ts +137 -0
- package/src/training.js +219 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +9 -1
- package/tests/kernels/browser/test-page.js +4 -2
- package/tests/kernels/reference/adam.js +4 -2
- package/NOTICE +0 -5
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
import type { TrainingConfigSchema } from '../../config/training-defaults.d.ts';
|
|
2
|
+
import type { Tensor } from '../../gpu/tensor.js';
|
|
3
|
+
import type { DynamicLossScaler } from './loss-scaling.js';
|
|
4
|
+
import type { TrainingBatch, TrainingOptimizer, ClipMetrics } from './trainer.js';
|
|
5
|
+
import type { DataLoader } from './dataloader.js';
|
|
6
|
+
import type { TrainingObjective } from './objectives/base.js';
|
|
7
|
+
import type { UlArtifactFinalizeResult, DistillArtifactFinalizeResult } from './artifacts.js';
|
|
8
|
+
|
|
9
|
+
export interface TrainingStepMetricsEntry {
|
|
10
|
+
schemaVersion: number;
|
|
11
|
+
step: number;
|
|
12
|
+
epoch: number;
|
|
13
|
+
batch: number;
|
|
14
|
+
total_loss: number;
|
|
15
|
+
step_time_ms: number;
|
|
16
|
+
forward_ms?: number;
|
|
17
|
+
backward_ms?: number;
|
|
18
|
+
optimizer_ms?: number;
|
|
19
|
+
effective_lr?: number | null;
|
|
20
|
+
lr?: number | null;
|
|
21
|
+
seed?: number;
|
|
22
|
+
model_id?: string;
|
|
23
|
+
runtime_profile?: string | null;
|
|
24
|
+
kernel_path?: string | null;
|
|
25
|
+
environment_metadata?: Record<string, unknown>;
|
|
26
|
+
memory_stats?: Record<string, unknown> | null;
|
|
27
|
+
build_provenance?: Record<string, unknown> | null;
|
|
28
|
+
scheduler_index?: number | null;
|
|
29
|
+
scheduler_phase?: string | null;
|
|
30
|
+
gradient_norm_unclipped?: number;
|
|
31
|
+
gradient_norm_clipped?: number;
|
|
32
|
+
clipped_event_count?: number;
|
|
33
|
+
total_param_count?: number;
|
|
34
|
+
trainable_param_count?: number | null;
|
|
35
|
+
trainable_groups?: string[];
|
|
36
|
+
frozen_groups?: string[];
|
|
37
|
+
nan_count?: number;
|
|
38
|
+
inf_count?: number;
|
|
39
|
+
saturation_count?: number;
|
|
40
|
+
telemetry_mode?: 'step' | 'window' | 'epoch';
|
|
41
|
+
telemetry_window_size?: number;
|
|
42
|
+
telemetry_alerts?: string[];
|
|
43
|
+
window_loss_avg?: number | null;
|
|
44
|
+
window_step_time_ms_avg?: number | null;
|
|
45
|
+
ul_stage?: string | null;
|
|
46
|
+
distill_stage?: string | null;
|
|
47
|
+
lambda?: number | null;
|
|
48
|
+
progress_shard_index?: number | null;
|
|
49
|
+
progress_shard_count?: number | null;
|
|
50
|
+
progress_step_in_shard?: number | null;
|
|
51
|
+
progress_steps_in_shard?: number | null;
|
|
52
|
+
progress_global_step?: number | null;
|
|
53
|
+
progress_global_steps?: number | null;
|
|
54
|
+
progress_percent_complete?: number | null;
|
|
55
|
+
progress_elapsed_ms?: number | null;
|
|
56
|
+
progress_eta_ms?: number | null;
|
|
57
|
+
progress_eta_iso?: string | null;
|
|
58
|
+
loss_kd?: number | null;
|
|
59
|
+
loss_triplet?: number | null;
|
|
60
|
+
distill_temperature?: number | null;
|
|
61
|
+
distill_alpha_kd?: number | null;
|
|
62
|
+
distill_alpha_ce?: number | null;
|
|
63
|
+
distill_loss_ce_aux?: number | null;
|
|
64
|
+
distill_loss_total?: number | null;
|
|
65
|
+
distill_triplet_margin?: number | null;
|
|
66
|
+
distill_triplet_active_count?: number | null;
|
|
67
|
+
distill_stage_a_step_count?: number | null;
|
|
68
|
+
distill_stage_a_kd_mean?: number | null;
|
|
69
|
+
objective?: string;
|
|
70
|
+
loss_total?: number | null;
|
|
71
|
+
coeff_ce?: number | null;
|
|
72
|
+
coeff_prior?: number | null;
|
|
73
|
+
coeff_decoder?: number | null;
|
|
74
|
+
coeff_recon?: number | null;
|
|
75
|
+
schedule_step_index?: number | null;
|
|
76
|
+
latent_clean_mean?: number | null;
|
|
77
|
+
latent_clean_std?: number | null;
|
|
78
|
+
latent_noise_mean?: number | null;
|
|
79
|
+
latent_noise_std?: number | null;
|
|
80
|
+
latent_noisy_mean?: number | null;
|
|
81
|
+
latent_noisy_std?: number | null;
|
|
82
|
+
stage1_latent_count?: number | null;
|
|
83
|
+
loss_prior?: number | null;
|
|
84
|
+
loss_decoder?: number | null;
|
|
85
|
+
loss_recon?: number | null;
|
|
86
|
+
latent_bitrate_proxy?: number | null;
|
|
87
|
+
[key: string]: unknown;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface TrainingRunnerCallbacks {
|
|
91
|
+
onStep?: (entry: TrainingStepMetricsEntry) => Promise<void> | void;
|
|
92
|
+
onEpoch?: (entry: { epoch: number; steps: number; loss: number }) => Promise<void> | void;
|
|
93
|
+
onCheckpoint?: (entry: {
|
|
94
|
+
key: string;
|
|
95
|
+
defaultCheckpointKey: string | null;
|
|
96
|
+
path: string | null;
|
|
97
|
+
metadata: Record<string, unknown> | null;
|
|
98
|
+
payload: unknown;
|
|
99
|
+
step: number;
|
|
100
|
+
epoch: number;
|
|
101
|
+
batch: number;
|
|
102
|
+
}) => Promise<void> | void;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export interface TrainingRunnerOptions extends TrainingRunnerCallbacks {
|
|
106
|
+
optimizer?: TrainingOptimizer;
|
|
107
|
+
crossEntropyLoss?: (
|
|
108
|
+
logits: Tensor,
|
|
109
|
+
targets: Tensor,
|
|
110
|
+
config: TrainingConfigSchema,
|
|
111
|
+
tape: unknown
|
|
112
|
+
) => Promise<Tensor>;
|
|
113
|
+
clipGradients?: (
|
|
114
|
+
grads: Map<Tensor, Tensor>,
|
|
115
|
+
config: TrainingConfigSchema
|
|
116
|
+
) => Promise<ClipMetrics>;
|
|
117
|
+
lossScaler?: DynamicLossScaler;
|
|
118
|
+
trainingObjective?: TrainingObjective;
|
|
119
|
+
resolveCheckpointKey?: (entry: {
|
|
120
|
+
defaultCheckpointKey: string | null;
|
|
121
|
+
step: number;
|
|
122
|
+
epoch: number;
|
|
123
|
+
batch: number;
|
|
124
|
+
}) => Promise<string> | string;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export interface TrainingRunOptions {
|
|
128
|
+
epochs?: number;
|
|
129
|
+
batchSize?: number;
|
|
130
|
+
shuffle?: boolean;
|
|
131
|
+
maxSteps?: number | null;
|
|
132
|
+
logEvery?: number;
|
|
133
|
+
prepareBatch?: (batch: unknown) => Promise<TrainingBatch> | TrainingBatch;
|
|
134
|
+
ulArtifactDir?: string | null;
|
|
135
|
+
distillArtifactDir?: string | null;
|
|
136
|
+
stageAArtifact?: string | null;
|
|
137
|
+
stageAArtifactHash?: string | null;
|
|
138
|
+
teacherModelId?: string | null;
|
|
139
|
+
studentModelId?: string | null;
|
|
140
|
+
distillDatasetId?: string | null;
|
|
141
|
+
distillDatasetPath?: string | null;
|
|
142
|
+
distillLanguagePair?: string | null;
|
|
143
|
+
distillSourceLangs?: string[] | null;
|
|
144
|
+
distillTargetLangs?: string[] | null;
|
|
145
|
+
distillPairAllowlist?: string[] | null;
|
|
146
|
+
strictPairContract?: boolean;
|
|
147
|
+
distillShardIndex?: number | null;
|
|
148
|
+
distillShardCount?: number | null;
|
|
149
|
+
checkpointKey?: string | null;
|
|
150
|
+
resumeFrom?: string | null;
|
|
151
|
+
forceResume?: boolean;
|
|
152
|
+
forceResumeReason?: string | null;
|
|
153
|
+
forceResumeSource?: string | null;
|
|
154
|
+
runtimeProfile?: string | null;
|
|
155
|
+
seed?: number | null;
|
|
156
|
+
kernelPathId?: string | null;
|
|
157
|
+
tokenizerHash?: string | null;
|
|
158
|
+
optimizerStepCount?: number | null;
|
|
159
|
+
buildProvenance?: Record<string, unknown> | null;
|
|
160
|
+
buildId?: string | null;
|
|
161
|
+
buildCommitHash?: string | null;
|
|
162
|
+
buildTimestamp?: string | number | Date | null;
|
|
163
|
+
gpuAdapterInfo?: Record<string, unknown> | null;
|
|
164
|
+
command?: string | null;
|
|
165
|
+
surface?: string | null;
|
|
166
|
+
checkpointOperator?: string | null;
|
|
167
|
+
modelId?: string | null;
|
|
168
|
+
modelUrl?: string | null;
|
|
169
|
+
timestamp?: string | Date | null;
|
|
170
|
+
persistStage1Latents?: boolean;
|
|
171
|
+
checkpointEvery?: number | null;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export declare class TrainingRunner {
|
|
175
|
+
constructor(config: TrainingConfigSchema, options?: TrainingRunnerOptions);
|
|
176
|
+
lastArtifact: UlArtifactFinalizeResult | DistillArtifactFinalizeResult | null;
|
|
177
|
+
lastCheckpoint: {
|
|
178
|
+
key: string;
|
|
179
|
+
defaultKey?: string | null;
|
|
180
|
+
path?: string | null;
|
|
181
|
+
metadata?: Record<string, unknown> | null;
|
|
182
|
+
step: number;
|
|
183
|
+
epoch: number;
|
|
184
|
+
batch: number;
|
|
185
|
+
} | null;
|
|
186
|
+
resumeState: {
|
|
187
|
+
step: number;
|
|
188
|
+
epoch: number;
|
|
189
|
+
batch: number;
|
|
190
|
+
checkpointHash?: string | null;
|
|
191
|
+
resumeAudits?: Array<Record<string, unknown>>;
|
|
192
|
+
resumeAuditCount?: number;
|
|
193
|
+
previousCheckpointHash?: string | null;
|
|
194
|
+
checkpointKey?: string | null;
|
|
195
|
+
} | null;
|
|
196
|
+
run(
|
|
197
|
+
model: {
|
|
198
|
+
forward: (input: Tensor, tape: unknown) => Promise<Tensor>;
|
|
199
|
+
loraParams?: () => Tensor[];
|
|
200
|
+
paramGroups?: () => Record<string, Tensor[]>;
|
|
201
|
+
},
|
|
202
|
+
dataset: TrainingBatch[] | DataLoader<TrainingBatch> | unknown[],
|
|
203
|
+
options?: TrainingRunOptions
|
|
204
|
+
): Promise<TrainingStepMetricsEntry[]>;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export declare function runTraining(
|
|
208
|
+
model: {
|
|
209
|
+
forward: (input: Tensor, tape: unknown) => Promise<Tensor>;
|
|
210
|
+
loraParams?: () => Tensor[];
|
|
211
|
+
paramGroups?: () => Record<string, Tensor[]>;
|
|
212
|
+
},
|
|
213
|
+
dataset: TrainingBatch[] | DataLoader<TrainingBatch> | unknown[],
|
|
214
|
+
config: TrainingConfigSchema,
|
|
215
|
+
options?: TrainingRunOptions & TrainingRunnerOptions
|
|
216
|
+
): Promise<TrainingStepMetricsEntry[]>;
|
|
217
|
+
|
|
218
|
+
export declare function createTrainingCheckpointPayload(
|
|
219
|
+
model: {
|
|
220
|
+
loraParams?: () => Tensor[];
|
|
221
|
+
paramGroups?: () => Record<string, Tensor[]>;
|
|
222
|
+
},
|
|
223
|
+
optimizer: unknown,
|
|
224
|
+
context: {
|
|
225
|
+
step: number;
|
|
226
|
+
epoch: number;
|
|
227
|
+
batch: number;
|
|
228
|
+
config: TrainingConfigSchema;
|
|
229
|
+
}
|
|
230
|
+
): Promise<unknown>;
|
|
231
|
+
|
|
232
|
+
export declare function restoreTrainingCheckpointState(
|
|
233
|
+
model: {
|
|
234
|
+
loraParams?: () => Tensor[];
|
|
235
|
+
paramGroups?: () => Record<string, Tensor[]>;
|
|
236
|
+
},
|
|
237
|
+
optimizer: unknown,
|
|
238
|
+
checkpointRecord: unknown,
|
|
239
|
+
config: TrainingConfigSchema
|
|
240
|
+
): Promise<{
|
|
241
|
+
step: number;
|
|
242
|
+
epoch: number;
|
|
243
|
+
batch: number;
|
|
244
|
+
checkpointHash: string | null;
|
|
245
|
+
previousCheckpointHash: string | null;
|
|
246
|
+
checkpointKey: string | null;
|
|
247
|
+
resumeAudits: Array<Record<string, unknown>>;
|
|
248
|
+
resumeAuditCount: number;
|
|
249
|
+
} | null>;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import type { BackwardRegistrySchema } from '../../config/schema/backward-registry.schema.js';
|
|
2
|
+
import type { TrainingConfigSchema } from '../../config/training-defaults.d.ts';
|
|
3
|
+
import type { Tensor } from '../../gpu/tensor.js';
|
|
4
|
+
import type { TrainingObjective } from './objectives/base.js';
|
|
5
|
+
|
|
6
|
+
export interface TrainingBatch {
|
|
7
|
+
input: Tensor;
|
|
8
|
+
targets: Tensor;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface TrainingOptimizer {
|
|
12
|
+
step: (
|
|
13
|
+
params: Tensor[],
|
|
14
|
+
grads: Map<Tensor, Tensor>,
|
|
15
|
+
config: TrainingConfigSchema,
|
|
16
|
+
context?: {
|
|
17
|
+
trainableGroups?: string[];
|
|
18
|
+
frozenGroups?: string[];
|
|
19
|
+
allGroups?: string[];
|
|
20
|
+
}
|
|
21
|
+
) => Promise<OptimizerMetrics>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface TrainStepOptions {
|
|
25
|
+
registry?: BackwardRegistrySchema;
|
|
26
|
+
crossEntropyLoss?: (
|
|
27
|
+
logits: Tensor,
|
|
28
|
+
targets: Tensor,
|
|
29
|
+
config: TrainingConfigSchema,
|
|
30
|
+
tape: unknown
|
|
31
|
+
) => Promise<Tensor>;
|
|
32
|
+
clipGradients?: (
|
|
33
|
+
grads: Map<Tensor, Tensor>,
|
|
34
|
+
config: TrainingConfigSchema
|
|
35
|
+
) => Promise<ClipMetrics>;
|
|
36
|
+
optimizer?: TrainingOptimizer;
|
|
37
|
+
lossScale?: number;
|
|
38
|
+
applyClip?: boolean;
|
|
39
|
+
applyOptimizer?: boolean;
|
|
40
|
+
trainingObjective?: TrainingObjective;
|
|
41
|
+
stepIndex?: number | null;
|
|
42
|
+
epochIndex?: number | null;
|
|
43
|
+
batchIndex?: number | null;
|
|
44
|
+
stage1ArtifactContext?: Record<string, unknown> | null;
|
|
45
|
+
stageAArtifactContext?: Record<string, unknown> | null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface ClipMetrics {
|
|
49
|
+
clippedGrads: Map<Tensor, Tensor>;
|
|
50
|
+
gradient_norm_unclipped: number;
|
|
51
|
+
gradient_norm_clipped: number;
|
|
52
|
+
clipped_event_count: number;
|
|
53
|
+
total_param_count: number;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface OptimizerMetrics {
|
|
57
|
+
optimizer_ms: number;
|
|
58
|
+
effective_lr?: number | null;
|
|
59
|
+
scheduler_index?: number | null;
|
|
60
|
+
scheduler_phase?: string | null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface TrainStepResult {
|
|
64
|
+
loss: Tensor;
|
|
65
|
+
grads: Map<Tensor, Tensor>;
|
|
66
|
+
forward_ms?: number;
|
|
67
|
+
backward_ms?: number;
|
|
68
|
+
clipMetrics?: ClipMetrics;
|
|
69
|
+
optimizerMetrics?: OptimizerMetrics;
|
|
70
|
+
objectiveName?: string;
|
|
71
|
+
objectiveMetrics?: Record<string, number | string | boolean | null | undefined>;
|
|
72
|
+
paramGroupMetrics?: {
|
|
73
|
+
trainableGroups: string[];
|
|
74
|
+
frozenGroups: string[];
|
|
75
|
+
allGroups: string[];
|
|
76
|
+
trainableParamCount: number;
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export declare function trainStep(
|
|
81
|
+
model: {
|
|
82
|
+
forward: (input: Tensor, tape: unknown) => Promise<Tensor>;
|
|
83
|
+
loraParams?: () => Tensor[];
|
|
84
|
+
paramGroups?: () => Record<string, Tensor[]>;
|
|
85
|
+
},
|
|
86
|
+
batch: TrainingBatch,
|
|
87
|
+
config: TrainingConfigSchema,
|
|
88
|
+
options?: TrainStepOptions
|
|
89
|
+
): Promise<TrainStepResult>;
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
export type TrainingWorkloadKind = 'lora' | 'distill' | 'ul';
|
|
2
|
+
export type TrainingSurfaceSupport = 'node' | 'browser' | 'both';
|
|
3
|
+
export type TrainingSelectionGoal = 'max' | 'min';
|
|
4
|
+
export type TrainingRowOrder = 'dataset_order' | 'seed_hash_sorted_v1';
|
|
5
|
+
export type TrainingEvalKind =
|
|
6
|
+
| 'translation'
|
|
7
|
+
| 'text_generation'
|
|
8
|
+
| 'classification'
|
|
9
|
+
| 'retrieval'
|
|
10
|
+
| 'custom';
|
|
11
|
+
|
|
12
|
+
export type TrainingAgentEvalCategory =
|
|
13
|
+
| 'js_patching'
|
|
14
|
+
| 'wgsl_review'
|
|
15
|
+
| 'manifest_config_review'
|
|
16
|
+
| 'reploid_vfs_status_tool_loop'
|
|
17
|
+
| 'patch_applies'
|
|
18
|
+
| 'no_hallucinated_files_tools';
|
|
19
|
+
|
|
20
|
+
export interface TrainingAgentEvalConfig {
|
|
21
|
+
suiteId: string;
|
|
22
|
+
categories: TrainingAgentEvalCategory[];
|
|
23
|
+
minPassRate: number;
|
|
24
|
+
requirePatchApplies: boolean;
|
|
25
|
+
requireNoHallucinatedFiles: boolean;
|
|
26
|
+
requireNoHallucinatedTools: boolean;
|
|
27
|
+
allowedFiles: string[];
|
|
28
|
+
allowedTools: string[];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface TrainingEvalDataset {
|
|
32
|
+
id: string;
|
|
33
|
+
datasetPath: string;
|
|
34
|
+
evalKind: TrainingEvalKind;
|
|
35
|
+
metrics: string[];
|
|
36
|
+
decodePolicy: {
|
|
37
|
+
maxTokens: number | null;
|
|
38
|
+
stopOnEos: boolean;
|
|
39
|
+
} | null;
|
|
40
|
+
scoreboardColumns: string[];
|
|
41
|
+
quality: {
|
|
42
|
+
baseline: string;
|
|
43
|
+
requireImprovement: boolean;
|
|
44
|
+
minAbsoluteImprovement: number;
|
|
45
|
+
minRelativeImprovement: number;
|
|
46
|
+
} | null;
|
|
47
|
+
agentEval: TrainingAgentEvalConfig | null;
|
|
48
|
+
sourceLangs: string[] | null;
|
|
49
|
+
targetLangs: string[] | null;
|
|
50
|
+
pairAllowlist: string[] | null;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface TrainingOptimizerConfig {
|
|
54
|
+
type: string;
|
|
55
|
+
lr: number;
|
|
56
|
+
beta1: number;
|
|
57
|
+
beta2: number;
|
|
58
|
+
eps: number;
|
|
59
|
+
weightDecay: number;
|
|
60
|
+
scheduler: {
|
|
61
|
+
enabled: boolean;
|
|
62
|
+
type: string;
|
|
63
|
+
warmupSteps: number;
|
|
64
|
+
stepSize: number;
|
|
65
|
+
gamma: number;
|
|
66
|
+
totalSteps: number;
|
|
67
|
+
minLr: number;
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface TrainingWorkloadTrainingConfig {
|
|
72
|
+
optimizer: TrainingOptimizerConfig;
|
|
73
|
+
batchSize: number;
|
|
74
|
+
accumSteps: number;
|
|
75
|
+
steps: number;
|
|
76
|
+
precision: {
|
|
77
|
+
activations: string;
|
|
78
|
+
gradients: string;
|
|
79
|
+
loraParams: string;
|
|
80
|
+
};
|
|
81
|
+
gradientClipping: {
|
|
82
|
+
maxNorm: number;
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface LoRAWorkloadPipelineConfig {
|
|
87
|
+
datasetFormat: string;
|
|
88
|
+
taskType: string;
|
|
89
|
+
baseModelRef: string | null;
|
|
90
|
+
baseModelRevision: string | null;
|
|
91
|
+
maxLength: number | null;
|
|
92
|
+
sequenceLength: number | null;
|
|
93
|
+
joinWith: string | null;
|
|
94
|
+
rowOrder?: TrainingRowOrder;
|
|
95
|
+
adapter: {
|
|
96
|
+
rank: number;
|
|
97
|
+
alpha: number;
|
|
98
|
+
dropout: number;
|
|
99
|
+
targetModules: string[];
|
|
100
|
+
};
|
|
101
|
+
freeze: {
|
|
102
|
+
encoder: boolean;
|
|
103
|
+
prior: boolean;
|
|
104
|
+
decoder: boolean;
|
|
105
|
+
base: boolean;
|
|
106
|
+
lora: boolean;
|
|
107
|
+
};
|
|
108
|
+
export: {
|
|
109
|
+
enabled: boolean;
|
|
110
|
+
atCheckpoints: boolean;
|
|
111
|
+
select: string;
|
|
112
|
+
id: string | null;
|
|
113
|
+
name: string | null;
|
|
114
|
+
format: string;
|
|
115
|
+
} | null;
|
|
116
|
+
activation: {
|
|
117
|
+
enabled: boolean;
|
|
118
|
+
autoActivate: boolean;
|
|
119
|
+
smokePrompt: string | null;
|
|
120
|
+
} | null;
|
|
121
|
+
trainer: {
|
|
122
|
+
modulePath: string;
|
|
123
|
+
exportName: string;
|
|
124
|
+
runnerId: string | null;
|
|
125
|
+
} | null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export interface DistillStagePlanEntry {
|
|
129
|
+
id: string;
|
|
130
|
+
trainingStage: string;
|
|
131
|
+
objective: string;
|
|
132
|
+
steps: number;
|
|
133
|
+
checkpointEvery: number;
|
|
134
|
+
selectionMetric: string;
|
|
135
|
+
selectionGoal: TrainingSelectionGoal;
|
|
136
|
+
evalSchedule: string;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export interface DistillWorkloadPipelineConfig {
|
|
140
|
+
stagePlan: DistillStagePlanEntry[];
|
|
141
|
+
studentGraphMode: string;
|
|
142
|
+
temperature: number;
|
|
143
|
+
alphaKd: number;
|
|
144
|
+
alphaCe: number;
|
|
145
|
+
tripletMargin: number;
|
|
146
|
+
sourceLangs: string[] | null;
|
|
147
|
+
targetLangs: string[] | null;
|
|
148
|
+
pairAllowlist: string[] | null;
|
|
149
|
+
strictPairContract: boolean;
|
|
150
|
+
subsetSpec: Record<string, unknown> | null;
|
|
151
|
+
sftLora: LoRAWorkloadPipelineConfig | null;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export interface TrainingWorkloadPack {
|
|
155
|
+
schemaVersion: number;
|
|
156
|
+
kind: TrainingWorkloadKind;
|
|
157
|
+
id: string;
|
|
158
|
+
description: string;
|
|
159
|
+
claimBoundary: string;
|
|
160
|
+
seed: number;
|
|
161
|
+
baseModelId: string;
|
|
162
|
+
studentModelId: string | null;
|
|
163
|
+
teacherModelId: string | null;
|
|
164
|
+
datasetId: string;
|
|
165
|
+
datasetPath: string | null;
|
|
166
|
+
evalDatasets: TrainingEvalDataset[];
|
|
167
|
+
trainingSchemaVersion: number;
|
|
168
|
+
checkpointEvery: number;
|
|
169
|
+
selectionMetric: string;
|
|
170
|
+
selectionGoal: TrainingSelectionGoal;
|
|
171
|
+
surfaceSupport: TrainingSurfaceSupport;
|
|
172
|
+
training: TrainingWorkloadTrainingConfig;
|
|
173
|
+
pipeline: LoRAWorkloadPipelineConfig | DistillWorkloadPipelineConfig | Record<string, unknown>;
|
|
174
|
+
configHash: string;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export interface LoadedTrainingWorkload {
|
|
178
|
+
absolutePath: string;
|
|
179
|
+
path: string;
|
|
180
|
+
raw: string;
|
|
181
|
+
workloadSha256: string;
|
|
182
|
+
workload: TrainingWorkloadPack;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export declare const TRAINING_WORKLOAD_SCHEMA_VERSION: number;
|
|
186
|
+
export declare const TRAINING_WORKLOAD_KINDS: readonly TrainingWorkloadKind[];
|
|
187
|
+
export declare const TRAINING_WORKLOAD_SURFACE_SUPPORT: readonly TrainingSurfaceSupport[];
|
|
188
|
+
export declare const TRAINING_SELECTION_GOALS: readonly TrainingSelectionGoal[];
|
|
189
|
+
export declare const TRAINING_ROW_ORDERS: readonly TrainingRowOrder[];
|
|
190
|
+
export declare const TRAINING_EVAL_KINDS: readonly TrainingEvalKind[];
|
|
191
|
+
export declare const TRAINING_AGENT_EVAL_CATEGORIES: readonly TrainingAgentEvalCategory[];
|
|
192
|
+
|
|
193
|
+
export declare function normalizeTrainingWorkloadPack(
|
|
194
|
+
payload: Record<string, unknown>,
|
|
195
|
+
context?: { label?: string }
|
|
196
|
+
): TrainingWorkloadPack;
|
|
197
|
+
|
|
198
|
+
export declare function loadTrainingWorkloadPack(
|
|
199
|
+
input: string,
|
|
200
|
+
options?: { registryPath?: string | null }
|
|
201
|
+
): Promise<LoadedTrainingWorkload>;
|
|
202
|
+
|
|
203
|
+
export declare function serializeTrainingWorkloadLock(
|
|
204
|
+
loadedWorkload: LoadedTrainingWorkload
|
|
205
|
+
): string;
|
|
@@ -53,6 +53,7 @@ async function runAdamChunked(device, pipeline, params, grads, moment1, moment2,
|
|
|
53
53
|
view.setFloat32(12, options.beta1, true);
|
|
54
54
|
view.setFloat32(16, options.beta2, true);
|
|
55
55
|
view.setFloat32(20, options.eps, true);
|
|
56
|
+
view.setFloat32(24, options.weightDecay ?? 0, true);
|
|
56
57
|
},
|
|
57
58
|
null,
|
|
58
59
|
device
|
|
@@ -96,7 +97,7 @@ export async function runAdam(
|
|
|
96
97
|
options = {}
|
|
97
98
|
) {
|
|
98
99
|
const device = getDevice();
|
|
99
|
-
const { count, step = 1, lr, beta1, beta2, eps } = options;
|
|
100
|
+
const { count, step = 1, lr, beta1, beta2, eps, weightDecay = 0 } = options;
|
|
100
101
|
|
|
101
102
|
const inferredCount = count ?? tensorElementCount(params);
|
|
102
103
|
const pipeline = await createPipeline('adam', 'default');
|
|
@@ -115,6 +116,7 @@ export async function runAdam(
|
|
|
115
116
|
beta1,
|
|
116
117
|
beta2,
|
|
117
118
|
eps,
|
|
119
|
+
weightDecay,
|
|
118
120
|
},
|
|
119
121
|
inferredCount
|
|
120
122
|
);
|
|
@@ -131,6 +133,7 @@ export async function runAdam(
|
|
|
131
133
|
view.setFloat32(12, beta1, true);
|
|
132
134
|
view.setFloat32(16, beta2, true);
|
|
133
135
|
view.setFloat32(20, eps, true);
|
|
136
|
+
view.setFloat32(24, weightDecay, true);
|
|
134
137
|
},
|
|
135
138
|
null,
|
|
136
139
|
device
|
|
@@ -167,7 +170,7 @@ export async function recordAdam(
|
|
|
167
170
|
options = {}
|
|
168
171
|
) {
|
|
169
172
|
const device = recorder.device;
|
|
170
|
-
const { count, step = 1, lr, beta1, beta2, eps } = options;
|
|
173
|
+
const { count, step = 1, lr, beta1, beta2, eps, weightDecay = 0 } = options;
|
|
171
174
|
|
|
172
175
|
const inferredCount = count ?? tensorElementCount(params);
|
|
173
176
|
const pipeline = await createPipeline('adam', 'default');
|
|
@@ -182,6 +185,7 @@ export async function recordAdam(
|
|
|
182
185
|
view.setFloat32(12, beta1, true);
|
|
183
186
|
view.setFloat32(16, beta2, true);
|
|
184
187
|
view.setFloat32(20, eps, true);
|
|
188
|
+
view.setFloat32(24, weightDecay, true);
|
|
185
189
|
},
|
|
186
190
|
recorder
|
|
187
191
|
);
|
|
@@ -13,7 +13,7 @@ struct Uniforms {
|
|
|
13
13
|
beta1: f32,
|
|
14
14
|
beta2: f32,
|
|
15
15
|
eps: f32,
|
|
16
|
-
|
|
16
|
+
weight_decay: f32,
|
|
17
17
|
_pad1: u32,
|
|
18
18
|
}
|
|
19
19
|
|
|
@@ -44,7 +44,8 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
|
44
44
|
let m_hat = m / (1.0 - pow(beta1, step_f));
|
|
45
45
|
let v_hat = v / (1.0 - pow(beta2, step_f));
|
|
46
46
|
|
|
47
|
-
|
|
47
|
+
let adam_update = m_hat / (sqrt(v_hat) + u.eps);
|
|
48
|
+
params[idx] = params[idx] - u.lr * (adam_update + u.weight_decay * params[idx]);
|
|
48
49
|
moment1[idx] = m;
|
|
49
50
|
moment2[idx] = v;
|
|
50
51
|
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// Frozen Q4_K row-wise weight input gradient.
|
|
2
|
+
//
|
|
3
|
+
// Forward: Y[M,N] = X[M,K] @ W[N,K]^T
|
|
4
|
+
// Backward: dX[M,K] = dY[M,N] @ W[N,K]
|
|
5
|
+
|
|
6
|
+
// The packed base weight remains read-only. Each invocation owns one dX cell
|
|
7
|
+
// and dequantizes only the matching K element from every output row.
|
|
8
|
+
|
|
9
|
+
const QK_K: u32 = 256u;
|
|
10
|
+
const SUBBLOCK_SIZE: u32 = 32u;
|
|
11
|
+
|
|
12
|
+
override TILE_SIZE: u32 = 16u;
|
|
13
|
+
|
|
14
|
+
struct Uniforms {
|
|
15
|
+
M: u32,
|
|
16
|
+
N: u32,
|
|
17
|
+
K: u32,
|
|
18
|
+
alpha: f32,
|
|
19
|
+
transpose_b: u32,
|
|
20
|
+
num_blocks_per_row: u32,
|
|
21
|
+
_pad0: u32,
|
|
22
|
+
_pad1: u32,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
struct Q4KBlock {
|
|
26
|
+
d_dmin: u32,
|
|
27
|
+
scales: array<u32, 3>,
|
|
28
|
+
qs: array<u32, 32>,
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
32
|
+
@group(0) @binding(1) var<storage, read> dY: array<f32>;
|
|
33
|
+
@group(0) @binding(2) var<storage, read> W: array<Q4KBlock>;
|
|
34
|
+
@group(0) @binding(3) var<storage, read_write> dX: array<f32>;
|
|
35
|
+
|
|
36
|
+
fn get_scale_byte(scales: array<u32, 3>, byte_index: u32) -> u32 {
|
|
37
|
+
let word_index = byte_index / 4u;
|
|
38
|
+
let byte_in_word = byte_index % 4u;
|
|
39
|
+
return (scales[word_index] >> (byte_in_word * 8u)) & 0xffu;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
fn get_scale_min(scales: array<u32, 3>, subblock: u32) -> vec2<u32> {
|
|
43
|
+
var scale: u32;
|
|
44
|
+
var minimum: u32;
|
|
45
|
+
if (subblock < 4u) {
|
|
46
|
+
scale = get_scale_byte(scales, subblock) & 63u;
|
|
47
|
+
minimum = get_scale_byte(scales, subblock + 4u) & 63u;
|
|
48
|
+
} else {
|
|
49
|
+
let packed = get_scale_byte(scales, subblock + 4u);
|
|
50
|
+
scale = (packed & 0x0fu) | ((get_scale_byte(scales, subblock - 4u) >> 6u) << 4u);
|
|
51
|
+
minimum = (packed >> 4u) | ((get_scale_byte(scales, subblock) >> 6u) << 4u);
|
|
52
|
+
}
|
|
53
|
+
return vec2<u32>(scale, minimum);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
fn get_q4(qs: array<u32, 32>, element: u32) -> u32 {
|
|
57
|
+
let chunk = element / 64u;
|
|
58
|
+
let chunk_element = element % 64u;
|
|
59
|
+
let upper = chunk_element >= 32u;
|
|
60
|
+
let byte_in_chunk = select(chunk_element, chunk_element - 32u, upper);
|
|
61
|
+
let byte_index = chunk * 32u + byte_in_chunk;
|
|
62
|
+
let byte_value = (qs[byte_index / 4u] >> ((byte_index % 4u) * 8u)) & 0xffu;
|
|
63
|
+
return select(byte_value & 0x0fu, (byte_value >> 4u) & 0x0fu, upper);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
fn dequantize_weight(row: u32, column: u32) -> f32 {
|
|
67
|
+
let block_in_row = column / QK_K;
|
|
68
|
+
let element = column % QK_K;
|
|
69
|
+
let block = W[row * u.num_blocks_per_row + block_in_row];
|
|
70
|
+
let d = unpack2x16float(block.d_dmin).x;
|
|
71
|
+
let dmin = unpack2x16float(block.d_dmin).y;
|
|
72
|
+
let scale_min = get_scale_min(block.scales, element / SUBBLOCK_SIZE);
|
|
73
|
+
return d * f32(scale_min.x) * f32(get_q4(block.qs, element))
|
|
74
|
+
- dmin * f32(scale_min.y);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
@compute @workgroup_size(TILE_SIZE, TILE_SIZE, 1)
|
|
78
|
+
fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
|
|
79
|
+
let row = global_id.x;
|
|
80
|
+
let column = global_id.y;
|
|
81
|
+
if (row >= u.M || column >= u.K || u.transpose_b != 1u) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
var sum = 0.0f;
|
|
86
|
+
for (var output_column = 0u; output_column < u.N; output_column += 1u) {
|
|
87
|
+
sum += dY[row * u.N + output_column] * dequantize_weight(output_column, column);
|
|
88
|
+
}
|
|
89
|
+
dX[row * u.K + column] = sum * u.alpha;
|
|
90
|
+
}
|