doppler-gpu 0.4.10 → 0.4.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +95 -0
  2. package/README.md +40 -1
  3. package/models/catalog.json +637 -32
  4. package/package.json +24 -3
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +135 -13
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +69 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
  18. package/src/config/kernels/kernel-ref-digests.js +2 -1
  19. package/src/config/kernels/registry.json +73 -11
  20. package/src/config/merge.js +24 -0
  21. package/src/config/required-inference-fields-contract-check.js +2 -0
  22. package/src/config/schema/adapter.schema.d.ts +6 -1
  23. package/src/config/schema/adapter.schema.js +5 -0
  24. package/src/config/schema/converter.schema.d.ts +7 -0
  25. package/src/config/schema/converter.schema.js +15 -0
  26. package/src/config/schema/inference.schema.d.ts +8 -0
  27. package/src/config/schema/manifest.schema.d.ts +32 -1
  28. package/src/config/schema/manifest.schema.js +26 -0
  29. package/src/converter/conversion-plan.js +15 -1
  30. package/src/converter/core.js +5 -4
  31. package/src/converter/source-tensor-rules.d.ts +50 -0
  32. package/src/converter/source-tensor-rules.js +180 -0
  33. package/src/converter/source-tokenizer.d.ts +45 -0
  34. package/src/converter/source-tokenizer.js +155 -0
  35. package/src/experimental/adapters/artifact-contract.d.ts +83 -0
  36. package/src/experimental/adapters/artifact-contract.js +254 -0
  37. package/src/experimental/adapters/index.d.ts +25 -0
  38. package/src/experimental/adapters/index.js +12 -0
  39. package/src/experimental/adapters/lora-loader.js +24 -4
  40. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  41. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  42. package/src/experimental/training/artifacts.d.ts +160 -0
  43. package/src/experimental/training/attention-backward.d.ts +64 -0
  44. package/src/experimental/training/attention-forward.d.ts +22 -0
  45. package/src/experimental/training/attention-forward.js +82 -0
  46. package/src/experimental/training/autograd.d.ts +76 -0
  47. package/src/experimental/training/checkpoint.d.ts +36 -0
  48. package/src/experimental/training/clip.d.ts +9 -0
  49. package/src/experimental/training/dataloader.d.ts +8 -0
  50. package/src/experimental/training/distillation/student-fixture.js +1 -0
  51. package/src/experimental/training/export.d.ts +41 -0
  52. package/src/experimental/training/lora-pipeline.d.ts +186 -0
  53. package/src/experimental/training/lora-pipeline.js +5 -0
  54. package/src/experimental/training/lora.d.ts +20 -0
  55. package/src/experimental/training/lora.js +4 -3
  56. package/src/experimental/training/loss-scaling.d.ts +21 -0
  57. package/src/experimental/training/loss.d.ts +10 -0
  58. package/src/experimental/training/native-qwen-lora.d.ts +106 -0
  59. package/src/experimental/training/native-qwen-lora.js +449 -0
  60. package/src/experimental/training/objectives/base.d.ts +58 -0
  61. package/src/experimental/training/optimizer.d.ts +22 -0
  62. package/src/experimental/training/optimizer.js +1 -0
  63. package/src/experimental/training/runner.d.ts +249 -0
  64. package/src/experimental/training/trainer.d.ts +89 -0
  65. package/src/experimental/training/workloads.d.ts +205 -0
  66. package/src/gpu/kernels/backward/adam.d.ts +1 -0
  67. package/src/gpu/kernels/backward/adam.js +6 -2
  68. package/src/gpu/kernels/backward/adam.wgsl +3 -2
  69. package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
  70. package/src/gpu/kernels/backward/utils.js +18 -4
  71. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  72. package/src/inference/pipelines/text/attention/projections.js +161 -24
  73. package/src/inference/pipelines/text/attention/record.js +67 -9
  74. package/src/inference/pipelines/text/attention/run.js +60 -7
  75. package/src/inference/pipelines/text/attention/types.js +18 -9
  76. package/src/inference/pipelines/text/config.d.ts +5 -0
  77. package/src/inference/pipelines/text/config.js +91 -0
  78. package/src/inference/pipelines/text/ffn/dense.js +205 -2
  79. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  80. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  81. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  82. package/src/inference/pipelines/text/generator-steps.js +3 -1
  83. package/src/inference/pipelines/text/generator.d.ts +5 -1
  84. package/src/inference/pipelines/text/generator.js +196 -3
  85. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  86. package/src/inference/pipelines/text/init.d.ts +2 -0
  87. package/src/inference/pipelines/text/init.js +2 -0
  88. package/src/inference/pipelines/text/layer.js +6 -0
  89. package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
  90. package/src/inference/pipelines/text/linear-attention.js +77 -6
  91. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  92. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  93. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  94. package/src/inference/pipelines/text/logits/index.js +85 -11
  95. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  96. package/src/inference/pipelines/text/lora-types.d.ts +6 -1
  97. package/src/inference/pipelines/text/lora-types.js +5 -0
  98. package/src/inference/pipelines/text/ops.js +33 -0
  99. package/src/inference/pipelines/text/types.d.ts +66 -0
  100. package/src/inference/pipelines/text/weights.d.ts +10 -0
  101. package/src/inference/pipelines/text/weights.js +33 -1
  102. package/src/inference/pipelines/text.d.ts +9 -2
  103. package/src/inference/pipelines/text.js +116 -0
  104. package/src/inference/tokenizers/behavior-flags.js +5 -1
  105. package/src/inference/tokenizers/bundled.js +99 -8
  106. package/src/loader/doppler-loader.d.ts +2 -0
  107. package/src/loader/doppler-loader.js +9 -0
  108. package/src/loader/final-weights-loader.d.ts +8 -0
  109. package/src/loader/final-weights-loader.js +42 -0
  110. package/src/loader/layer-loader.d.ts +6 -0
  111. package/src/loader/layer-loader.js +102 -3
  112. package/src/loader/loader-types.d.ts +15 -2
  113. package/src/models/diffusiongemma.js +1 -1
  114. package/src/models/embeddinggemma.js +1 -1
  115. package/src/models/gemma3.js +1 -1
  116. package/src/models/gemma4.js +1 -1
  117. package/src/models/qwen3.js +1 -1
  118. package/src/tooling/hf-registry-utils.js +1 -1
  119. package/src/tooling/node-browser-command-runner.d.ts +16 -0
  120. package/src/tooling/node-browser-command-runner.js +1 -1
  121. package/src/tooling/node-converter.js +33 -1
  122. package/src/tooling/node-webgpu.js +85 -2
  123. package/src/training-browser.d.ts +19 -0
  124. package/src/training-browser.js +13 -0
  125. package/src/training.d.ts +137 -0
  126. package/src/training.js +219 -0
  127. package/src/version.js +1 -1
  128. package/tests/kernels/browser/kernel-suite.js +9 -1
  129. package/tests/kernels/browser/test-page.js +4 -2
  130. package/tests/kernels/reference/adam.js +4 -2
  131. package/NOTICE +0 -5
@@ -0,0 +1,249 @@
1
+ import type { TrainingConfigSchema } from '../../config/training-defaults.d.ts';
2
+ import type { Tensor } from '../../gpu/tensor.js';
3
+ import type { DynamicLossScaler } from './loss-scaling.js';
4
+ import type { TrainingBatch, TrainingOptimizer, ClipMetrics } from './trainer.js';
5
+ import type { DataLoader } from './dataloader.js';
6
+ import type { TrainingObjective } from './objectives/base.js';
7
+ import type { UlArtifactFinalizeResult, DistillArtifactFinalizeResult } from './artifacts.js';
8
+
9
+ export interface TrainingStepMetricsEntry {
10
+ schemaVersion: number;
11
+ step: number;
12
+ epoch: number;
13
+ batch: number;
14
+ total_loss: number;
15
+ step_time_ms: number;
16
+ forward_ms?: number;
17
+ backward_ms?: number;
18
+ optimizer_ms?: number;
19
+ effective_lr?: number | null;
20
+ lr?: number | null;
21
+ seed?: number;
22
+ model_id?: string;
23
+ runtime_profile?: string | null;
24
+ kernel_path?: string | null;
25
+ environment_metadata?: Record<string, unknown>;
26
+ memory_stats?: Record<string, unknown> | null;
27
+ build_provenance?: Record<string, unknown> | null;
28
+ scheduler_index?: number | null;
29
+ scheduler_phase?: string | null;
30
+ gradient_norm_unclipped?: number;
31
+ gradient_norm_clipped?: number;
32
+ clipped_event_count?: number;
33
+ total_param_count?: number;
34
+ trainable_param_count?: number | null;
35
+ trainable_groups?: string[];
36
+ frozen_groups?: string[];
37
+ nan_count?: number;
38
+ inf_count?: number;
39
+ saturation_count?: number;
40
+ telemetry_mode?: 'step' | 'window' | 'epoch';
41
+ telemetry_window_size?: number;
42
+ telemetry_alerts?: string[];
43
+ window_loss_avg?: number | null;
44
+ window_step_time_ms_avg?: number | null;
45
+ ul_stage?: string | null;
46
+ distill_stage?: string | null;
47
+ lambda?: number | null;
48
+ progress_shard_index?: number | null;
49
+ progress_shard_count?: number | null;
50
+ progress_step_in_shard?: number | null;
51
+ progress_steps_in_shard?: number | null;
52
+ progress_global_step?: number | null;
53
+ progress_global_steps?: number | null;
54
+ progress_percent_complete?: number | null;
55
+ progress_elapsed_ms?: number | null;
56
+ progress_eta_ms?: number | null;
57
+ progress_eta_iso?: string | null;
58
+ loss_kd?: number | null;
59
+ loss_triplet?: number | null;
60
+ distill_temperature?: number | null;
61
+ distill_alpha_kd?: number | null;
62
+ distill_alpha_ce?: number | null;
63
+ distill_loss_ce_aux?: number | null;
64
+ distill_loss_total?: number | null;
65
+ distill_triplet_margin?: number | null;
66
+ distill_triplet_active_count?: number | null;
67
+ distill_stage_a_step_count?: number | null;
68
+ distill_stage_a_kd_mean?: number | null;
69
+ objective?: string;
70
+ loss_total?: number | null;
71
+ coeff_ce?: number | null;
72
+ coeff_prior?: number | null;
73
+ coeff_decoder?: number | null;
74
+ coeff_recon?: number | null;
75
+ schedule_step_index?: number | null;
76
+ latent_clean_mean?: number | null;
77
+ latent_clean_std?: number | null;
78
+ latent_noise_mean?: number | null;
79
+ latent_noise_std?: number | null;
80
+ latent_noisy_mean?: number | null;
81
+ latent_noisy_std?: number | null;
82
+ stage1_latent_count?: number | null;
83
+ loss_prior?: number | null;
84
+ loss_decoder?: number | null;
85
+ loss_recon?: number | null;
86
+ latent_bitrate_proxy?: number | null;
87
+ [key: string]: unknown;
88
+ }
89
+
90
+ export interface TrainingRunnerCallbacks {
91
+ onStep?: (entry: TrainingStepMetricsEntry) => Promise<void> | void;
92
+ onEpoch?: (entry: { epoch: number; steps: number; loss: number }) => Promise<void> | void;
93
+ onCheckpoint?: (entry: {
94
+ key: string;
95
+ defaultCheckpointKey: string | null;
96
+ path: string | null;
97
+ metadata: Record<string, unknown> | null;
98
+ payload: unknown;
99
+ step: number;
100
+ epoch: number;
101
+ batch: number;
102
+ }) => Promise<void> | void;
103
+ }
104
+
105
+ export interface TrainingRunnerOptions extends TrainingRunnerCallbacks {
106
+ optimizer?: TrainingOptimizer;
107
+ crossEntropyLoss?: (
108
+ logits: Tensor,
109
+ targets: Tensor,
110
+ config: TrainingConfigSchema,
111
+ tape: unknown
112
+ ) => Promise<Tensor>;
113
+ clipGradients?: (
114
+ grads: Map<Tensor, Tensor>,
115
+ config: TrainingConfigSchema
116
+ ) => Promise<ClipMetrics>;
117
+ lossScaler?: DynamicLossScaler;
118
+ trainingObjective?: TrainingObjective;
119
+ resolveCheckpointKey?: (entry: {
120
+ defaultCheckpointKey: string | null;
121
+ step: number;
122
+ epoch: number;
123
+ batch: number;
124
+ }) => Promise<string> | string;
125
+ }
126
+
127
+ export interface TrainingRunOptions {
128
+ epochs?: number;
129
+ batchSize?: number;
130
+ shuffle?: boolean;
131
+ maxSteps?: number | null;
132
+ logEvery?: number;
133
+ prepareBatch?: (batch: unknown) => Promise<TrainingBatch> | TrainingBatch;
134
+ ulArtifactDir?: string | null;
135
+ distillArtifactDir?: string | null;
136
+ stageAArtifact?: string | null;
137
+ stageAArtifactHash?: string | null;
138
+ teacherModelId?: string | null;
139
+ studentModelId?: string | null;
140
+ distillDatasetId?: string | null;
141
+ distillDatasetPath?: string | null;
142
+ distillLanguagePair?: string | null;
143
+ distillSourceLangs?: string[] | null;
144
+ distillTargetLangs?: string[] | null;
145
+ distillPairAllowlist?: string[] | null;
146
+ strictPairContract?: boolean;
147
+ distillShardIndex?: number | null;
148
+ distillShardCount?: number | null;
149
+ checkpointKey?: string | null;
150
+ resumeFrom?: string | null;
151
+ forceResume?: boolean;
152
+ forceResumeReason?: string | null;
153
+ forceResumeSource?: string | null;
154
+ runtimeProfile?: string | null;
155
+ seed?: number | null;
156
+ kernelPathId?: string | null;
157
+ tokenizerHash?: string | null;
158
+ optimizerStepCount?: number | null;
159
+ buildProvenance?: Record<string, unknown> | null;
160
+ buildId?: string | null;
161
+ buildCommitHash?: string | null;
162
+ buildTimestamp?: string | number | Date | null;
163
+ gpuAdapterInfo?: Record<string, unknown> | null;
164
+ command?: string | null;
165
+ surface?: string | null;
166
+ checkpointOperator?: string | null;
167
+ modelId?: string | null;
168
+ modelUrl?: string | null;
169
+ timestamp?: string | Date | null;
170
+ persistStage1Latents?: boolean;
171
+ checkpointEvery?: number | null;
172
+ }
173
+
174
+ export declare class TrainingRunner {
175
+ constructor(config: TrainingConfigSchema, options?: TrainingRunnerOptions);
176
+ lastArtifact: UlArtifactFinalizeResult | DistillArtifactFinalizeResult | null;
177
+ lastCheckpoint: {
178
+ key: string;
179
+ defaultKey?: string | null;
180
+ path?: string | null;
181
+ metadata?: Record<string, unknown> | null;
182
+ step: number;
183
+ epoch: number;
184
+ batch: number;
185
+ } | null;
186
+ resumeState: {
187
+ step: number;
188
+ epoch: number;
189
+ batch: number;
190
+ checkpointHash?: string | null;
191
+ resumeAudits?: Array<Record<string, unknown>>;
192
+ resumeAuditCount?: number;
193
+ previousCheckpointHash?: string | null;
194
+ checkpointKey?: string | null;
195
+ } | null;
196
+ run(
197
+ model: {
198
+ forward: (input: Tensor, tape: unknown) => Promise<Tensor>;
199
+ loraParams?: () => Tensor[];
200
+ paramGroups?: () => Record<string, Tensor[]>;
201
+ },
202
+ dataset: TrainingBatch[] | DataLoader<TrainingBatch> | unknown[],
203
+ options?: TrainingRunOptions
204
+ ): Promise<TrainingStepMetricsEntry[]>;
205
+ }
206
+
207
+ export declare function runTraining(
208
+ model: {
209
+ forward: (input: Tensor, tape: unknown) => Promise<Tensor>;
210
+ loraParams?: () => Tensor[];
211
+ paramGroups?: () => Record<string, Tensor[]>;
212
+ },
213
+ dataset: TrainingBatch[] | DataLoader<TrainingBatch> | unknown[],
214
+ config: TrainingConfigSchema,
215
+ options?: TrainingRunOptions & TrainingRunnerOptions
216
+ ): Promise<TrainingStepMetricsEntry[]>;
217
+
218
+ export declare function createTrainingCheckpointPayload(
219
+ model: {
220
+ loraParams?: () => Tensor[];
221
+ paramGroups?: () => Record<string, Tensor[]>;
222
+ },
223
+ optimizer: unknown,
224
+ context: {
225
+ step: number;
226
+ epoch: number;
227
+ batch: number;
228
+ config: TrainingConfigSchema;
229
+ }
230
+ ): Promise<unknown>;
231
+
232
+ export declare function restoreTrainingCheckpointState(
233
+ model: {
234
+ loraParams?: () => Tensor[];
235
+ paramGroups?: () => Record<string, Tensor[]>;
236
+ },
237
+ optimizer: unknown,
238
+ checkpointRecord: unknown,
239
+ config: TrainingConfigSchema
240
+ ): Promise<{
241
+ step: number;
242
+ epoch: number;
243
+ batch: number;
244
+ checkpointHash: string | null;
245
+ previousCheckpointHash: string | null;
246
+ checkpointKey: string | null;
247
+ resumeAudits: Array<Record<string, unknown>>;
248
+ resumeAuditCount: number;
249
+ } | null>;
@@ -0,0 +1,89 @@
1
+ import type { BackwardRegistrySchema } from '../../config/schema/backward-registry.schema.js';
2
+ import type { TrainingConfigSchema } from '../../config/training-defaults.d.ts';
3
+ import type { Tensor } from '../../gpu/tensor.js';
4
+ import type { TrainingObjective } from './objectives/base.js';
5
+
6
+ export interface TrainingBatch {
7
+ input: Tensor;
8
+ targets: Tensor;
9
+ }
10
+
11
+ export interface TrainingOptimizer {
12
+ step: (
13
+ params: Tensor[],
14
+ grads: Map<Tensor, Tensor>,
15
+ config: TrainingConfigSchema,
16
+ context?: {
17
+ trainableGroups?: string[];
18
+ frozenGroups?: string[];
19
+ allGroups?: string[];
20
+ }
21
+ ) => Promise<OptimizerMetrics>;
22
+ }
23
+
24
+ export interface TrainStepOptions {
25
+ registry?: BackwardRegistrySchema;
26
+ crossEntropyLoss?: (
27
+ logits: Tensor,
28
+ targets: Tensor,
29
+ config: TrainingConfigSchema,
30
+ tape: unknown
31
+ ) => Promise<Tensor>;
32
+ clipGradients?: (
33
+ grads: Map<Tensor, Tensor>,
34
+ config: TrainingConfigSchema
35
+ ) => Promise<ClipMetrics>;
36
+ optimizer?: TrainingOptimizer;
37
+ lossScale?: number;
38
+ applyClip?: boolean;
39
+ applyOptimizer?: boolean;
40
+ trainingObjective?: TrainingObjective;
41
+ stepIndex?: number | null;
42
+ epochIndex?: number | null;
43
+ batchIndex?: number | null;
44
+ stage1ArtifactContext?: Record<string, unknown> | null;
45
+ stageAArtifactContext?: Record<string, unknown> | null;
46
+ }
47
+
48
+ export interface ClipMetrics {
49
+ clippedGrads: Map<Tensor, Tensor>;
50
+ gradient_norm_unclipped: number;
51
+ gradient_norm_clipped: number;
52
+ clipped_event_count: number;
53
+ total_param_count: number;
54
+ }
55
+
56
+ export interface OptimizerMetrics {
57
+ optimizer_ms: number;
58
+ effective_lr?: number | null;
59
+ scheduler_index?: number | null;
60
+ scheduler_phase?: string | null;
61
+ }
62
+
63
+ export interface TrainStepResult {
64
+ loss: Tensor;
65
+ grads: Map<Tensor, Tensor>;
66
+ forward_ms?: number;
67
+ backward_ms?: number;
68
+ clipMetrics?: ClipMetrics;
69
+ optimizerMetrics?: OptimizerMetrics;
70
+ objectiveName?: string;
71
+ objectiveMetrics?: Record<string, number | string | boolean | null | undefined>;
72
+ paramGroupMetrics?: {
73
+ trainableGroups: string[];
74
+ frozenGroups: string[];
75
+ allGroups: string[];
76
+ trainableParamCount: number;
77
+ };
78
+ }
79
+
80
+ export declare function trainStep(
81
+ model: {
82
+ forward: (input: Tensor, tape: unknown) => Promise<Tensor>;
83
+ loraParams?: () => Tensor[];
84
+ paramGroups?: () => Record<string, Tensor[]>;
85
+ },
86
+ batch: TrainingBatch,
87
+ config: TrainingConfigSchema,
88
+ options?: TrainStepOptions
89
+ ): Promise<TrainStepResult>;
@@ -0,0 +1,205 @@
1
+ export type TrainingWorkloadKind = 'lora' | 'distill' | 'ul';
2
+ export type TrainingSurfaceSupport = 'node' | 'browser' | 'both';
3
+ export type TrainingSelectionGoal = 'max' | 'min';
4
+ export type TrainingRowOrder = 'dataset_order' | 'seed_hash_sorted_v1';
5
+ export type TrainingEvalKind =
6
+ | 'translation'
7
+ | 'text_generation'
8
+ | 'classification'
9
+ | 'retrieval'
10
+ | 'custom';
11
+
12
+ export type TrainingAgentEvalCategory =
13
+ | 'js_patching'
14
+ | 'wgsl_review'
15
+ | 'manifest_config_review'
16
+ | 'reploid_vfs_status_tool_loop'
17
+ | 'patch_applies'
18
+ | 'no_hallucinated_files_tools';
19
+
20
+ export interface TrainingAgentEvalConfig {
21
+ suiteId: string;
22
+ categories: TrainingAgentEvalCategory[];
23
+ minPassRate: number;
24
+ requirePatchApplies: boolean;
25
+ requireNoHallucinatedFiles: boolean;
26
+ requireNoHallucinatedTools: boolean;
27
+ allowedFiles: string[];
28
+ allowedTools: string[];
29
+ }
30
+
31
+ export interface TrainingEvalDataset {
32
+ id: string;
33
+ datasetPath: string;
34
+ evalKind: TrainingEvalKind;
35
+ metrics: string[];
36
+ decodePolicy: {
37
+ maxTokens: number | null;
38
+ stopOnEos: boolean;
39
+ } | null;
40
+ scoreboardColumns: string[];
41
+ quality: {
42
+ baseline: string;
43
+ requireImprovement: boolean;
44
+ minAbsoluteImprovement: number;
45
+ minRelativeImprovement: number;
46
+ } | null;
47
+ agentEval: TrainingAgentEvalConfig | null;
48
+ sourceLangs: string[] | null;
49
+ targetLangs: string[] | null;
50
+ pairAllowlist: string[] | null;
51
+ }
52
+
53
+ export interface TrainingOptimizerConfig {
54
+ type: string;
55
+ lr: number;
56
+ beta1: number;
57
+ beta2: number;
58
+ eps: number;
59
+ weightDecay: number;
60
+ scheduler: {
61
+ enabled: boolean;
62
+ type: string;
63
+ warmupSteps: number;
64
+ stepSize: number;
65
+ gamma: number;
66
+ totalSteps: number;
67
+ minLr: number;
68
+ };
69
+ }
70
+
71
+ export interface TrainingWorkloadTrainingConfig {
72
+ optimizer: TrainingOptimizerConfig;
73
+ batchSize: number;
74
+ accumSteps: number;
75
+ steps: number;
76
+ precision: {
77
+ activations: string;
78
+ gradients: string;
79
+ loraParams: string;
80
+ };
81
+ gradientClipping: {
82
+ maxNorm: number;
83
+ };
84
+ }
85
+
86
+ export interface LoRAWorkloadPipelineConfig {
87
+ datasetFormat: string;
88
+ taskType: string;
89
+ baseModelRef: string | null;
90
+ baseModelRevision: string | null;
91
+ maxLength: number | null;
92
+ sequenceLength: number | null;
93
+ joinWith: string | null;
94
+ rowOrder?: TrainingRowOrder;
95
+ adapter: {
96
+ rank: number;
97
+ alpha: number;
98
+ dropout: number;
99
+ targetModules: string[];
100
+ };
101
+ freeze: {
102
+ encoder: boolean;
103
+ prior: boolean;
104
+ decoder: boolean;
105
+ base: boolean;
106
+ lora: boolean;
107
+ };
108
+ export: {
109
+ enabled: boolean;
110
+ atCheckpoints: boolean;
111
+ select: string;
112
+ id: string | null;
113
+ name: string | null;
114
+ format: string;
115
+ } | null;
116
+ activation: {
117
+ enabled: boolean;
118
+ autoActivate: boolean;
119
+ smokePrompt: string | null;
120
+ } | null;
121
+ trainer: {
122
+ modulePath: string;
123
+ exportName: string;
124
+ runnerId: string | null;
125
+ } | null;
126
+ }
127
+
128
+ export interface DistillStagePlanEntry {
129
+ id: string;
130
+ trainingStage: string;
131
+ objective: string;
132
+ steps: number;
133
+ checkpointEvery: number;
134
+ selectionMetric: string;
135
+ selectionGoal: TrainingSelectionGoal;
136
+ evalSchedule: string;
137
+ }
138
+
139
+ export interface DistillWorkloadPipelineConfig {
140
+ stagePlan: DistillStagePlanEntry[];
141
+ studentGraphMode: string;
142
+ temperature: number;
143
+ alphaKd: number;
144
+ alphaCe: number;
145
+ tripletMargin: number;
146
+ sourceLangs: string[] | null;
147
+ targetLangs: string[] | null;
148
+ pairAllowlist: string[] | null;
149
+ strictPairContract: boolean;
150
+ subsetSpec: Record<string, unknown> | null;
151
+ sftLora: LoRAWorkloadPipelineConfig | null;
152
+ }
153
+
154
+ export interface TrainingWorkloadPack {
155
+ schemaVersion: number;
156
+ kind: TrainingWorkloadKind;
157
+ id: string;
158
+ description: string;
159
+ claimBoundary: string;
160
+ seed: number;
161
+ baseModelId: string;
162
+ studentModelId: string | null;
163
+ teacherModelId: string | null;
164
+ datasetId: string;
165
+ datasetPath: string | null;
166
+ evalDatasets: TrainingEvalDataset[];
167
+ trainingSchemaVersion: number;
168
+ checkpointEvery: number;
169
+ selectionMetric: string;
170
+ selectionGoal: TrainingSelectionGoal;
171
+ surfaceSupport: TrainingSurfaceSupport;
172
+ training: TrainingWorkloadTrainingConfig;
173
+ pipeline: LoRAWorkloadPipelineConfig | DistillWorkloadPipelineConfig | Record<string, unknown>;
174
+ configHash: string;
175
+ }
176
+
177
+ export interface LoadedTrainingWorkload {
178
+ absolutePath: string;
179
+ path: string;
180
+ raw: string;
181
+ workloadSha256: string;
182
+ workload: TrainingWorkloadPack;
183
+ }
184
+
185
+ export declare const TRAINING_WORKLOAD_SCHEMA_VERSION: number;
186
+ export declare const TRAINING_WORKLOAD_KINDS: readonly TrainingWorkloadKind[];
187
+ export declare const TRAINING_WORKLOAD_SURFACE_SUPPORT: readonly TrainingSurfaceSupport[];
188
+ export declare const TRAINING_SELECTION_GOALS: readonly TrainingSelectionGoal[];
189
+ export declare const TRAINING_ROW_ORDERS: readonly TrainingRowOrder[];
190
+ export declare const TRAINING_EVAL_KINDS: readonly TrainingEvalKind[];
191
+ export declare const TRAINING_AGENT_EVAL_CATEGORIES: readonly TrainingAgentEvalCategory[];
192
+
193
+ export declare function normalizeTrainingWorkloadPack(
194
+ payload: Record<string, unknown>,
195
+ context?: { label?: string }
196
+ ): TrainingWorkloadPack;
197
+
198
+ export declare function loadTrainingWorkloadPack(
199
+ input: string,
200
+ options?: { registryPath?: string | null }
201
+ ): Promise<LoadedTrainingWorkload>;
202
+
203
+ export declare function serializeTrainingWorkloadLock(
204
+ loadedWorkload: LoadedTrainingWorkload
205
+ ): string;
@@ -8,6 +8,7 @@ export interface AdamOptions {
8
8
  beta1: number;
9
9
  beta2: number;
10
10
  eps: number;
11
+ weightDecay?: number;
11
12
  }
12
13
 
13
14
  export declare function runAdam(
@@ -53,6 +53,7 @@ async function runAdamChunked(device, pipeline, params, grads, moment1, moment2,
53
53
  view.setFloat32(12, options.beta1, true);
54
54
  view.setFloat32(16, options.beta2, true);
55
55
  view.setFloat32(20, options.eps, true);
56
+ view.setFloat32(24, options.weightDecay ?? 0, true);
56
57
  },
57
58
  null,
58
59
  device
@@ -96,7 +97,7 @@ export async function runAdam(
96
97
  options = {}
97
98
  ) {
98
99
  const device = getDevice();
99
- const { count, step = 1, lr, beta1, beta2, eps } = options;
100
+ const { count, step = 1, lr, beta1, beta2, eps, weightDecay = 0 } = options;
100
101
 
101
102
  const inferredCount = count ?? tensorElementCount(params);
102
103
  const pipeline = await createPipeline('adam', 'default');
@@ -115,6 +116,7 @@ export async function runAdam(
115
116
  beta1,
116
117
  beta2,
117
118
  eps,
119
+ weightDecay,
118
120
  },
119
121
  inferredCount
120
122
  );
@@ -131,6 +133,7 @@ export async function runAdam(
131
133
  view.setFloat32(12, beta1, true);
132
134
  view.setFloat32(16, beta2, true);
133
135
  view.setFloat32(20, eps, true);
136
+ view.setFloat32(24, weightDecay, true);
134
137
  },
135
138
  null,
136
139
  device
@@ -167,7 +170,7 @@ export async function recordAdam(
167
170
  options = {}
168
171
  ) {
169
172
  const device = recorder.device;
170
- const { count, step = 1, lr, beta1, beta2, eps } = options;
173
+ const { count, step = 1, lr, beta1, beta2, eps, weightDecay = 0 } = options;
171
174
 
172
175
  const inferredCount = count ?? tensorElementCount(params);
173
176
  const pipeline = await createPipeline('adam', 'default');
@@ -182,6 +185,7 @@ export async function recordAdam(
182
185
  view.setFloat32(12, beta1, true);
183
186
  view.setFloat32(16, beta2, true);
184
187
  view.setFloat32(20, eps, true);
188
+ view.setFloat32(24, weightDecay, true);
185
189
  },
186
190
  recorder
187
191
  );
@@ -13,7 +13,7 @@ struct Uniforms {
13
13
  beta1: f32,
14
14
  beta2: f32,
15
15
  eps: f32,
16
- _pad0: u32,
16
+ weight_decay: f32,
17
17
  _pad1: u32,
18
18
  }
19
19
 
@@ -44,7 +44,8 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
44
44
  let m_hat = m / (1.0 - pow(beta1, step_f));
45
45
  let v_hat = v / (1.0 - pow(beta2, step_f));
46
46
 
47
- params[idx] = params[idx] - u.lr * m_hat / (sqrt(v_hat) + u.eps);
47
+ let adam_update = m_hat / (sqrt(v_hat) + u.eps);
48
+ params[idx] = params[idx] - u.lr * (adam_update + u.weight_decay * params[idx]);
48
49
  moment1[idx] = m;
49
50
  moment2[idx] = v;
50
51
  }
@@ -0,0 +1,90 @@
1
+ // Frozen Q4_K row-wise weight input gradient.
2
+ //
3
+ // Forward: Y[M,N] = X[M,K] @ W[N,K]^T
4
+ // Backward: dX[M,K] = dY[M,N] @ W[N,K]
5
+
6
+ // The packed base weight remains read-only. Each invocation owns one dX cell
7
+ // and dequantizes only the matching K element from every output row.
8
+
9
+ const QK_K: u32 = 256u;
10
+ const SUBBLOCK_SIZE: u32 = 32u;
11
+
12
+ override TILE_SIZE: u32 = 16u;
13
+
14
+ struct Uniforms {
15
+ M: u32,
16
+ N: u32,
17
+ K: u32,
18
+ alpha: f32,
19
+ transpose_b: u32,
20
+ num_blocks_per_row: u32,
21
+ _pad0: u32,
22
+ _pad1: u32,
23
+ }
24
+
25
+ struct Q4KBlock {
26
+ d_dmin: u32,
27
+ scales: array<u32, 3>,
28
+ qs: array<u32, 32>,
29
+ }
30
+
31
+ @group(0) @binding(0) var<uniform> u: Uniforms;
32
+ @group(0) @binding(1) var<storage, read> dY: array<f32>;
33
+ @group(0) @binding(2) var<storage, read> W: array<Q4KBlock>;
34
+ @group(0) @binding(3) var<storage, read_write> dX: array<f32>;
35
+
36
+ fn get_scale_byte(scales: array<u32, 3>, byte_index: u32) -> u32 {
37
+ let word_index = byte_index / 4u;
38
+ let byte_in_word = byte_index % 4u;
39
+ return (scales[word_index] >> (byte_in_word * 8u)) & 0xffu;
40
+ }
41
+
42
+ fn get_scale_min(scales: array<u32, 3>, subblock: u32) -> vec2<u32> {
43
+ var scale: u32;
44
+ var minimum: u32;
45
+ if (subblock < 4u) {
46
+ scale = get_scale_byte(scales, subblock) & 63u;
47
+ minimum = get_scale_byte(scales, subblock + 4u) & 63u;
48
+ } else {
49
+ let packed = get_scale_byte(scales, subblock + 4u);
50
+ scale = (packed & 0x0fu) | ((get_scale_byte(scales, subblock - 4u) >> 6u) << 4u);
51
+ minimum = (packed >> 4u) | ((get_scale_byte(scales, subblock) >> 6u) << 4u);
52
+ }
53
+ return vec2<u32>(scale, minimum);
54
+ }
55
+
56
+ fn get_q4(qs: array<u32, 32>, element: u32) -> u32 {
57
+ let chunk = element / 64u;
58
+ let chunk_element = element % 64u;
59
+ let upper = chunk_element >= 32u;
60
+ let byte_in_chunk = select(chunk_element, chunk_element - 32u, upper);
61
+ let byte_index = chunk * 32u + byte_in_chunk;
62
+ let byte_value = (qs[byte_index / 4u] >> ((byte_index % 4u) * 8u)) & 0xffu;
63
+ return select(byte_value & 0x0fu, (byte_value >> 4u) & 0x0fu, upper);
64
+ }
65
+
66
+ fn dequantize_weight(row: u32, column: u32) -> f32 {
67
+ let block_in_row = column / QK_K;
68
+ let element = column % QK_K;
69
+ let block = W[row * u.num_blocks_per_row + block_in_row];
70
+ let d = unpack2x16float(block.d_dmin).x;
71
+ let dmin = unpack2x16float(block.d_dmin).y;
72
+ let scale_min = get_scale_min(block.scales, element / SUBBLOCK_SIZE);
73
+ return d * f32(scale_min.x) * f32(get_q4(block.qs, element))
74
+ - dmin * f32(scale_min.y);
75
+ }
76
+
77
+ @compute @workgroup_size(TILE_SIZE, TILE_SIZE, 1)
78
+ fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
79
+ let row = global_id.x;
80
+ let column = global_id.y;
81
+ if (row >= u.M || column >= u.K || u.transpose_b != 1u) {
82
+ return;
83
+ }
84
+
85
+ var sum = 0.0f;
86
+ for (var output_column = 0u; output_column < u.N; output_column += 1u) {
87
+ sum += dY[row * u.N + output_column] * dequantize_weight(output_column, column);
88
+ }
89
+ dX[row * u.K + column] = sum * u.alpha;
90
+ }