doppler-gpu 0.4.11 → 0.4.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +61 -0
  2. package/README.md +13 -1
  3. package/models/catalog.json +29 -29
  4. package/package.json +12 -3
  5. package/src/client/doppler-registry.json +9 -9
  6. package/src/client/runtime/lora.js +5 -0
  7. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
  8. package/src/config/kernels/kernel-ref-digests.js +2 -1
  9. package/src/config/kernels/registry.json +21 -2
  10. package/src/config/schema/adapter.schema.d.ts +6 -1
  11. package/src/config/schema/adapter.schema.js +5 -0
  12. package/src/experimental/adapters/artifact-contract.d.ts +83 -0
  13. package/src/experimental/adapters/artifact-contract.js +254 -0
  14. package/src/experimental/adapters/index.d.ts +25 -0
  15. package/src/experimental/adapters/index.js +12 -0
  16. package/src/experimental/adapters/lora-loader.js +9 -4
  17. package/src/experimental/training/artifacts.d.ts +160 -0
  18. package/src/experimental/training/attention-backward.d.ts +64 -0
  19. package/src/experimental/training/attention-forward.d.ts +22 -0
  20. package/src/experimental/training/attention-forward.js +82 -0
  21. package/src/experimental/training/autograd.d.ts +76 -0
  22. package/src/experimental/training/checkpoint.d.ts +36 -0
  23. package/src/experimental/training/clip.d.ts +9 -0
  24. package/src/experimental/training/dataloader.d.ts +8 -0
  25. package/src/experimental/training/distillation/student-fixture.js +1 -0
  26. package/src/experimental/training/export.d.ts +41 -0
  27. package/src/experimental/training/lora-pipeline.d.ts +186 -0
  28. package/src/experimental/training/lora-pipeline.js +5 -0
  29. package/src/experimental/training/lora.d.ts +20 -0
  30. package/src/experimental/training/lora.js +4 -3
  31. package/src/experimental/training/loss-scaling.d.ts +21 -0
  32. package/src/experimental/training/loss.d.ts +10 -0
  33. package/src/experimental/training/native-qwen-lora.d.ts +106 -0
  34. package/src/experimental/training/native-qwen-lora.js +449 -0
  35. package/src/experimental/training/objectives/base.d.ts +58 -0
  36. package/src/experimental/training/optimizer.d.ts +22 -0
  37. package/src/experimental/training/optimizer.js +1 -0
  38. package/src/experimental/training/runner.d.ts +249 -0
  39. package/src/experimental/training/trainer.d.ts +89 -0
  40. package/src/experimental/training/workloads.d.ts +205 -0
  41. package/src/gpu/kernels/backward/adam.d.ts +1 -0
  42. package/src/gpu/kernels/backward/adam.js +6 -2
  43. package/src/gpu/kernels/backward/adam.wgsl +3 -2
  44. package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
  45. package/src/gpu/kernels/backward/utils.js +18 -4
  46. package/src/inference/pipelines/text/ffn/dense.js +16 -0
  47. package/src/inference/pipelines/text/generator.d.ts +5 -1
  48. package/src/inference/pipelines/text/generator.js +106 -0
  49. package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
  50. package/src/inference/pipelines/text/linear-attention.js +77 -6
  51. package/src/inference/pipelines/text/lora-types.d.ts +6 -1
  52. package/src/inference/pipelines/text/lora-types.js +5 -0
  53. package/src/inference/pipelines/text/ops.js +1 -0
  54. package/src/inference/pipelines/text/types.d.ts +28 -0
  55. package/src/inference/pipelines/text.d.ts +7 -2
  56. package/src/inference/pipelines/text.js +4 -0
  57. package/src/models/diffusiongemma.js +1 -1
  58. package/src/models/embeddinggemma.js +1 -1
  59. package/src/models/gemma3.js +1 -1
  60. package/src/models/gemma4.js +1 -1
  61. package/src/models/qwen3.js +1 -1
  62. package/src/tooling/hf-registry-utils.js +1 -1
  63. package/src/tooling/node-browser-command-runner.d.ts +16 -0
  64. package/src/tooling/node-browser-command-runner.js +1 -1
  65. package/src/training-browser.d.ts +19 -0
  66. package/src/training-browser.js +13 -0
  67. package/src/training.d.ts +137 -0
  68. package/src/training.js +219 -0
  69. package/src/version.js +1 -1
  70. package/tests/kernels/browser/kernel-suite.js +9 -1
  71. package/tests/kernels/browser/test-page.js +4 -2
  72. package/tests/kernels/reference/adam.js +4 -2
  73. package/NOTICE +0 -5
@@ -0,0 +1,254 @@
1
+ export const ADAPTER_ARTIFACT_SCHEMA = 'doppler.adapter-artifact/v1';
2
+ export const ADAPTER_ARTIFACT_LIFECYCLES = Object.freeze([
3
+ 'preserved',
4
+ 'candidate',
5
+ 'qualified',
6
+ 'promoted',
7
+ 'revoked',
8
+ ]);
9
+ export const ADAPTER_ARTIFACT_ACCESS = Object.freeze(['public', 'gated', 'private']);
10
+ export const ADAPTER_ARTIFACT_ORIGIN_PROVIDERS = Object.freeze([
11
+ 'huggingface',
12
+ 'gcs',
13
+ 'https-preservation',
14
+ ]);
15
+
16
+ const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/;
17
+ const HF_REVISION_PATTERN = /^[a-f0-9]{40,64}$/;
18
+ const GCS_GENERATION_PATTERN = /^[1-9][0-9]*$/;
19
+ const GCS_BUCKET_PATTERN = /^[a-z0-9][a-z0-9._-]{1,220}[a-z0-9]$/;
20
+ const HF_REPO_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*\/[A-Za-z0-9][A-Za-z0-9._-]*$/;
21
+ const FORBIDDEN_ARTIFACT_PATH_PATTERN = /[\\?#\u0000-\u001f\u007f]/;
22
+
23
+ const asText = (value) => String(value || '').trim();
24
+ const hasTraversal = (value) => asText(value).split('/').includes('..');
25
+
26
+ const requireText = (errors, value, field) => {
27
+ if (!asText(value)) errors.push({ field, message: `${field} is required`, value });
28
+ };
29
+
30
+ const requireSha256 = (errors, value, field) => {
31
+ if (!SHA256_PATTERN.test(asText(value))) {
32
+ errors.push({ field, message: `${field} must be a sha256: content identity`, value });
33
+ }
34
+ };
35
+
36
+ const validateArtifactPath = (errors, value, field) => {
37
+ const path = asText(value);
38
+ requireText(errors, path, field);
39
+ if (path.startsWith('/') || hasTraversal(path)) {
40
+ errors.push({ field, message: `${field} must be an artifact-relative path`, value });
41
+ }
42
+ if (FORBIDDEN_ARTIFACT_PATH_PATTERN.test(path)) {
43
+ errors.push({ field, message: `${field} contains a forbidden URL delimiter`, value });
44
+ }
45
+ };
46
+
47
+ const encodeArtifactPath = (value) => asText(value).split('/').map(encodeURIComponent).join('/');
48
+
49
+ export function validateAdapterArtifactOrigin(origin, { allowPreservation = false } = {}) {
50
+ const errors = [];
51
+ if (!origin || typeof origin !== 'object' || Array.isArray(origin)) {
52
+ return {
53
+ valid: false,
54
+ errors: [{ field: 'origin', message: 'origin must be an object', value: origin }],
55
+ };
56
+ }
57
+
58
+ if (!ADAPTER_ARTIFACT_ORIGIN_PROVIDERS.includes(origin.provider)) {
59
+ errors.push({ field: 'origin.provider', message: 'origin.provider is unsupported', value: origin.provider });
60
+ } else if (origin.provider === 'huggingface') {
61
+ if (!HF_REPO_PATTERN.test(asText(origin.repoId))) {
62
+ errors.push({ field: 'origin.repoId', message: 'origin.repoId must be an owner/repository identifier', value: origin.repoId });
63
+ }
64
+ if (!HF_REVISION_PATTERN.test(asText(origin.revision))) {
65
+ errors.push({
66
+ field: 'origin.revision',
67
+ message: 'origin.revision must be a full immutable Hugging Face commit hash',
68
+ value: origin.revision,
69
+ });
70
+ }
71
+ validateArtifactPath(errors, origin.path, 'origin.path');
72
+ } else if (origin.provider === 'gcs') {
73
+ if (!GCS_BUCKET_PATTERN.test(asText(origin.bucket))) {
74
+ errors.push({ field: 'origin.bucket', message: 'origin.bucket is invalid', value: origin.bucket });
75
+ }
76
+ validateArtifactPath(errors, origin.object, 'origin.object');
77
+ if (!GCS_GENERATION_PATTERN.test(asText(origin.generation))) {
78
+ errors.push({
79
+ field: 'origin.generation',
80
+ message: 'origin.generation must pin an immutable GCS object generation',
81
+ value: origin.generation,
82
+ });
83
+ }
84
+ } else if (origin.provider === 'https-preservation') {
85
+ if (!allowPreservation) {
86
+ errors.push({
87
+ field: 'origin.provider',
88
+ message: 'https-preservation is allowed only as a preservation mirror',
89
+ value: origin.provider,
90
+ });
91
+ }
92
+ let url = null;
93
+ try {
94
+ url = new URL(asText(origin.url));
95
+ } catch {
96
+ errors.push({ field: 'origin.url', message: 'origin.url must be an absolute HTTPS URL', value: origin.url });
97
+ }
98
+ if (url && (url.protocol !== 'https:' || url.username || url.password || url.search)) {
99
+ errors.push({
100
+ field: 'origin.url',
101
+ message: 'origin.url must be HTTPS, immutable, credential-free, and query-free',
102
+ value: origin.url,
103
+ });
104
+ }
105
+ if (!HF_REVISION_PATTERN.test(asText(origin.revision))) {
106
+ errors.push({
107
+ field: 'origin.revision',
108
+ message: 'preservation origins must declare their full immutable revision',
109
+ value: origin.revision,
110
+ });
111
+ }
112
+ }
113
+
114
+ return { valid: errors.length === 0, errors };
115
+ }
116
+
117
+ export function validateAdapterArtifactRecord(record) {
118
+ const errors = [];
119
+ if (!record || typeof record !== 'object' || Array.isArray(record)) {
120
+ return {
121
+ valid: false,
122
+ errors: [{ field: 'record', message: 'adapter artifact record must be an object', value: record }],
123
+ };
124
+ }
125
+
126
+ if (record.schema !== ADAPTER_ARTIFACT_SCHEMA) {
127
+ errors.push({ field: 'schema', message: `schema must be ${ADAPTER_ARTIFACT_SCHEMA}`, value: record.schema });
128
+ }
129
+ requireText(errors, record.artifactId, 'artifactId');
130
+ if (!ADAPTER_ARTIFACT_LIFECYCLES.includes(record.lifecycle)) {
131
+ errors.push({ field: 'lifecycle', message: 'lifecycle is unsupported', value: record.lifecycle });
132
+ }
133
+ if (!ADAPTER_ARTIFACT_ACCESS.includes(record.access)) {
134
+ errors.push({ field: 'access', message: 'access is unsupported', value: record.access });
135
+ }
136
+
137
+ const weights = record.weights || {};
138
+ requireSha256(errors, weights.sha256, 'weights.sha256');
139
+ if (!Number.isInteger(weights.bytes) || weights.bytes <= 0) {
140
+ errors.push({ field: 'weights.bytes', message: 'weights.bytes must be a positive integer', value: weights.bytes });
141
+ }
142
+ if (weights.format !== 'safetensors') {
143
+ errors.push({ field: 'weights.format', message: 'weights.format must be safetensors', value: weights.format });
144
+ }
145
+
146
+ const manifest = record.adapterManifest || {};
147
+ requireText(errors, manifest.id, 'adapterManifest.id');
148
+ requireText(errors, manifest.baseModel, 'adapterManifest.baseModel');
149
+ if (!Number.isInteger(manifest.rank) || manifest.rank <= 0) {
150
+ errors.push({ field: 'adapterManifest.rank', message: 'adapterManifest.rank must be a positive integer', value: manifest.rank });
151
+ }
152
+ if (!Number.isFinite(manifest.alpha) || manifest.alpha <= 0) {
153
+ errors.push({ field: 'adapterManifest.alpha', message: 'adapterManifest.alpha must be positive', value: manifest.alpha });
154
+ }
155
+ if (!Array.isArray(manifest.targetModules) || manifest.targetModules.length === 0) {
156
+ errors.push({ field: 'adapterManifest.targetModules', message: 'adapterManifest.targetModules is required', value: manifest.targetModules });
157
+ }
158
+ if (asText(manifest.checksumAlgorithm) !== 'sha256') {
159
+ errors.push({ field: 'adapterManifest.checksumAlgorithm', message: 'adapterManifest.checksumAlgorithm must be sha256', value: manifest.checksumAlgorithm });
160
+ }
161
+ const manifestChecksum = asText(manifest.checksum).replace(/^sha256:/, '');
162
+ if (`sha256:${manifestChecksum}` !== asText(weights.sha256)) {
163
+ errors.push({ field: 'adapterManifest.checksum', message: 'adapter manifest checksum must match weights.sha256', value: manifest.checksum });
164
+ }
165
+ if (Number(manifest.weightsSize) !== weights.bytes) {
166
+ errors.push({ field: 'adapterManifest.weightsSize', message: 'adapter manifest weightsSize must match weights.bytes', value: manifest.weightsSize });
167
+ }
168
+
169
+ const trainingBase = record.trainingBase || {};
170
+ requireText(errors, trainingBase.repoId, 'trainingBase.repoId');
171
+ if (!HF_REVISION_PATTERN.test(asText(trainingBase.revision))) {
172
+ errors.push({ field: 'trainingBase.revision', message: 'trainingBase.revision must be a full immutable source revision', value: trainingBase.revision });
173
+ }
174
+
175
+ const runtimeBase = record.runtimeBase || {};
176
+ requireText(errors, runtimeBase.modelId, 'runtimeBase.modelId');
177
+ requireText(errors, runtimeBase.weightPackId, 'runtimeBase.weightPackId');
178
+ requireText(errors, runtimeBase.manifestVariantId, 'runtimeBase.manifestVariantId');
179
+ for (const field of [
180
+ 'modelSha256',
181
+ 'manifestSha256',
182
+ 'tokenizerSha256',
183
+ 'weightPackSha256',
184
+ 'conversionConfigSha256',
185
+ ]) {
186
+ requireSha256(errors, runtimeBase[field], `runtimeBase.${field}`);
187
+ }
188
+ if (manifest.baseModel && runtimeBase.modelId && manifest.baseModel !== runtimeBase.modelId) {
189
+ errors.push({
190
+ field: 'runtimeBase.modelId',
191
+ message: 'runtimeBase.modelId must match adapterManifest.baseModel',
192
+ value: runtimeBase.modelId,
193
+ });
194
+ }
195
+
196
+ if (record.primaryOrigin == null) {
197
+ if (record.lifecycle === 'qualified' || record.lifecycle === 'promoted') {
198
+ errors.push({ field: 'primaryOrigin', message: `${record.lifecycle} artifacts require a primary Hugging Face or GCS origin`, value: record.primaryOrigin });
199
+ }
200
+ } else {
201
+ const originValidation = validateAdapterArtifactOrigin(record.primaryOrigin);
202
+ errors.push(...originValidation.errors.map((error) => ({ ...error, field: `primaryOrigin.${error.field.replace(/^origin\.?/, '')}` })));
203
+ }
204
+
205
+ if (!Array.isArray(record.preservationMirrors)) {
206
+ errors.push({ field: 'preservationMirrors', message: 'preservationMirrors must be an array', value: record.preservationMirrors });
207
+ } else {
208
+ record.preservationMirrors.forEach((origin, index) => {
209
+ const validation = validateAdapterArtifactOrigin(origin, { allowPreservation: true });
210
+ errors.push(...validation.errors.map((error) => ({
211
+ ...error,
212
+ field: `preservationMirrors[${index}].${error.field.replace(/^origin\.?/, '')}`,
213
+ })));
214
+ });
215
+ }
216
+
217
+ if (!Array.isArray(record.evidence) || record.evidence.length === 0) {
218
+ errors.push({ field: 'evidence', message: 'evidence must contain at least one receipt identity', value: record.evidence });
219
+ } else {
220
+ record.evidence.forEach((evidence, index) => {
221
+ requireText(errors, evidence.kind, `evidence[${index}].kind`);
222
+ validateArtifactPath(errors, evidence.path, `evidence[${index}].path`);
223
+ requireSha256(errors, evidence.sha256, `evidence[${index}].sha256`);
224
+ });
225
+ }
226
+
227
+ return { valid: errors.length === 0, errors };
228
+ }
229
+
230
+ export function assertAdapterArtifactRecord(record) {
231
+ const validation = validateAdapterArtifactRecord(record);
232
+ if (!validation.valid) {
233
+ const message = validation.errors.map((error) => `${error.field}: ${error.message}`).join('; ');
234
+ throw new Error(`Adapter artifact contract invalid: ${message}`);
235
+ }
236
+ return record;
237
+ }
238
+
239
+ export function buildImmutableArtifactUrl(origin) {
240
+ const validation = validateAdapterArtifactOrigin(origin);
241
+ if (!validation.valid) {
242
+ const message = validation.errors.map((error) => `${error.field}: ${error.message}`).join('; ');
243
+ throw new Error(`Adapter artifact origin invalid: ${message}`);
244
+ }
245
+ if (origin.provider === 'huggingface') {
246
+ return `https://huggingface.co/${origin.repoId}/resolve/${origin.revision}/${encodeArtifactPath(origin.path)}`;
247
+ }
248
+ return `https://storage.googleapis.com/${origin.bucket}/${encodeArtifactPath(origin.object)}?generation=${origin.generation}`;
249
+ }
250
+
251
+ export function adapterArtifactCacheKey(record) {
252
+ assertAdapterArtifactRecord(record);
253
+ return record.weights.sha256;
254
+ }
@@ -81,6 +81,31 @@ export {
81
81
  createMemoryRegistry,
82
82
  } from './adapter-registry.js';
83
83
 
84
+ export {
85
+ ADAPTER_ARTIFACT_SCHEMA,
86
+ ADAPTER_ARTIFACT_LIFECYCLES,
87
+ ADAPTER_ARTIFACT_ACCESS,
88
+ ADAPTER_ARTIFACT_ORIGIN_PROVIDERS,
89
+ validateAdapterArtifactOrigin,
90
+ validateAdapterArtifactRecord,
91
+ assertAdapterArtifactRecord,
92
+ buildImmutableArtifactUrl,
93
+ adapterArtifactCacheKey,
94
+ } from './artifact-contract.js';
95
+
96
+ export type {
97
+ AdapterArtifactLifecycle,
98
+ AdapterArtifactAccess,
99
+ HuggingFaceArtifactOrigin,
100
+ GcsArtifactOrigin,
101
+ HttpsPreservationOrigin,
102
+ AdapterArtifactOrigin,
103
+ AdapterArtifactMirror,
104
+ AdapterArtifactRecord,
105
+ AdapterArtifactValidationError,
106
+ AdapterArtifactValidationResult,
107
+ } from './artifact-contract.js';
108
+
84
109
  export type {
85
110
  // Types
86
111
  AdapterRegistryEntry,
@@ -40,3 +40,15 @@ export {
40
40
  resetAdapterRegistry,
41
41
  createMemoryRegistry,
42
42
  } from './adapter-registry.js';
43
+
44
+ export {
45
+ ADAPTER_ARTIFACT_SCHEMA,
46
+ ADAPTER_ARTIFACT_LIFECYCLES,
47
+ ADAPTER_ARTIFACT_ACCESS,
48
+ ADAPTER_ARTIFACT_ORIGIN_PROVIDERS,
49
+ validateAdapterArtifactOrigin,
50
+ validateAdapterArtifactRecord,
51
+ assertAdapterArtifactRecord,
52
+ buildImmutableArtifactUrl,
53
+ adapterArtifactCacheKey,
54
+ } from './artifact-contract.js';
@@ -256,10 +256,15 @@ export async function loadLoRAWeights(path, options = {}) {
256
256
  return res.arrayBuffer();
257
257
  };
258
258
 
259
- const adapter = await loadLoRAFromManifest(
260
- manifest,
261
- { ...options, fetchUrl: fetchWithBase }
262
- );
259
+ const manifestLoadOptions = isUrl
260
+ ? {
261
+ ...options,
262
+ readFile: undefined,
263
+ readOPFS: undefined,
264
+ fetchUrl: fetchWithBase,
265
+ }
266
+ : { ...options, fetchUrl: fetchWithBase };
267
+ const adapter = await loadLoRAFromManifest(manifest, manifestLoadOptions);
263
268
 
264
269
  let checksumValid;
265
270
  if (manifest.checksum && !options.skipVerify && Array.isArray(manifest.tensors) && manifest.tensors.length > 0) {
@@ -0,0 +1,160 @@
1
+ import type { TrainingConfigSchema } from '../../config/training-defaults.d.ts';
2
+ import type { DistillTrainingStage } from '../../config/schema/distill-training.schema.js';
3
+ import type { UlTrainingStage } from '../../config/schema/ul-training.schema.js';
4
+
5
+ export interface UlTrainingContract {
6
+ enabled: boolean;
7
+ stage: UlTrainingStage | null;
8
+ artifactDir: string | null;
9
+ stage1Artifact: string | null;
10
+ stage1ArtifactHash: string | null;
11
+ }
12
+
13
+ export interface DistillTrainingContract {
14
+ enabled: boolean;
15
+ stage: DistillTrainingStage | null;
16
+ artifactDir: string | null;
17
+ stageAArtifact: string | null;
18
+ stageAArtifactHash: string | null;
19
+ teacherModelId: string | null;
20
+ studentModelId: string | null;
21
+ datasetId: string | null;
22
+ datasetPath: string | null;
23
+ languagePair: string | null;
24
+ allowHintFallback: boolean;
25
+ }
26
+
27
+ export interface UlArtifactFinalizeResult {
28
+ kind: 'ul';
29
+ stage: UlTrainingStage;
30
+ runDir: string;
31
+ metricsPath: string;
32
+ manifestPath: string;
33
+ manifestHash: string;
34
+ manifestContentHash: string;
35
+ manifestFileHash: string;
36
+ stage1Dependency: {
37
+ path: string;
38
+ hash: string;
39
+ manifestHash: string | null;
40
+ } | null;
41
+ }
42
+
43
+ export interface DistillArtifactFinalizeResult {
44
+ kind: 'distill';
45
+ stage: DistillTrainingStage;
46
+ runDir: string;
47
+ metricsPath: string;
48
+ manifestPath: string;
49
+ manifestHash: string;
50
+ manifestContentHash: string;
51
+ manifestFileHash: string;
52
+ stageADependency: {
53
+ path: string;
54
+ hash: string;
55
+ manifestHash: string | null;
56
+ metricsHash: string;
57
+ metricsSummary: {
58
+ stepCount: number;
59
+ kdCount: number;
60
+ tripletCount: number;
61
+ kdMean: number | null;
62
+ tripletMean: number | null;
63
+ totalLossMean: number | null;
64
+ };
65
+ } | null;
66
+ }
67
+
68
+ export interface UlArtifactSession {
69
+ appendStep(entry: Record<string, unknown>): Promise<void>;
70
+ finalize(stepMetrics: Record<string, unknown>[]): Promise<UlArtifactFinalizeResult>;
71
+ }
72
+
73
+ export interface DistillArtifactSession {
74
+ appendStep(entry: Record<string, unknown>): Promise<void>;
75
+ finalize(stepMetrics: Record<string, unknown>[]): Promise<DistillArtifactFinalizeResult>;
76
+ }
77
+
78
+ export interface CreateUlArtifactSessionOptions {
79
+ config: TrainingConfigSchema;
80
+ stage: UlTrainingStage;
81
+ runOptions?: Record<string, unknown>;
82
+ }
83
+
84
+ export interface CreateDistillArtifactSessionOptions {
85
+ config: TrainingConfigSchema;
86
+ stage: DistillTrainingStage;
87
+ runOptions?: Record<string, unknown>;
88
+ }
89
+
90
+ export interface Stage1ArtifactContext {
91
+ manifestPath: string;
92
+ manifestHash: string;
93
+ ulContractHash: string | null;
94
+ latentDataset: {
95
+ path: string;
96
+ hash: string;
97
+ count: number;
98
+ summary: {
99
+ lambdaMean: number;
100
+ noisyStdMean: number;
101
+ cleanStdMean: number;
102
+ noiseStdMean: number;
103
+ scheduleMaxStep: number;
104
+ vectorCount: number;
105
+ };
106
+ entries: Record<string, unknown>[];
107
+ };
108
+ }
109
+
110
+ export interface StageAArtifactContext {
111
+ manifestPath: string;
112
+ manifestHash: string;
113
+ distillContractHash: string | null;
114
+ metrics: {
115
+ path: string;
116
+ hash: string;
117
+ count: number;
118
+ summary: {
119
+ stepCount: number;
120
+ kdCount: number;
121
+ tripletCount: number;
122
+ kdMean: number | null;
123
+ tripletMean: number | null;
124
+ totalLossMean: number | null;
125
+ };
126
+ entries: Record<string, unknown>[];
127
+ };
128
+ metricsSummary: {
129
+ stepCount: number;
130
+ kdCount: number;
131
+ tripletCount: number;
132
+ kdMean: number | null;
133
+ tripletMean: number | null;
134
+ totalLossMean: number | null;
135
+ };
136
+ }
137
+
138
+ export declare function resolveUlTrainingContract(
139
+ ulConfig: TrainingConfigSchema['training']['ul'] | null | undefined
140
+ ): UlTrainingContract;
141
+
142
+ export declare function resolveDistillTrainingContract(
143
+ distillConfig: TrainingConfigSchema['training']['distill'] | null | undefined
144
+ ): DistillTrainingContract;
145
+
146
+ export declare function createUlArtifactSession(
147
+ options: CreateUlArtifactSessionOptions
148
+ ): Promise<UlArtifactSession | null>;
149
+
150
+ export declare function createDistillArtifactSession(
151
+ options: CreateDistillArtifactSessionOptions
152
+ ): Promise<DistillArtifactSession | null>;
153
+
154
+ export declare function resolveStage1ArtifactContext(
155
+ config: TrainingConfigSchema
156
+ ): Promise<Stage1ArtifactContext | null>;
157
+
158
+ export declare function resolveStageAArtifactContext(
159
+ config: TrainingConfigSchema
160
+ ): Promise<StageAArtifactContext | null>;
@@ -0,0 +1,64 @@
1
+ import type { Tensor } from '../../gpu/tensor.js';
2
+
3
+ export interface AttentionBackwardOptions {
4
+ seqLen: number;
5
+ numHeads: number;
6
+ numKVHeads?: number;
7
+ headDim: number;
8
+ scale?: number;
9
+ causal?: boolean;
10
+ }
11
+
12
+ export interface AttentionGeometry {
13
+ seqLen: number;
14
+ numHeads: number;
15
+ numKVHeads: number;
16
+ headDim: number;
17
+ headsPerKV: number;
18
+ scale: number;
19
+ causal: boolean;
20
+ }
21
+
22
+ export interface AttentionBackwardResult {
23
+ gradQ: Tensor;
24
+ gradK: Tensor;
25
+ gradV: Tensor;
26
+ geometry: AttentionGeometry;
27
+ }
28
+
29
+ export interface AttentionBackwardDataResult {
30
+ dQ: Float32Array;
31
+ dK: Float32Array;
32
+ dV: Float32Array;
33
+ geometry: AttentionGeometry;
34
+ }
35
+
36
+ export declare function computeAttentionSoftmaxData(
37
+ qData: Float32Array,
38
+ kData: Float32Array,
39
+ options: AttentionBackwardOptions
40
+ ): Float32Array;
41
+
42
+ export declare function computeAttentionBackwardData(
43
+ qData: Float32Array,
44
+ kData: Float32Array,
45
+ vData: Float32Array,
46
+ softmaxData: Float32Array,
47
+ gradOutputData: Float32Array,
48
+ options: AttentionBackwardOptions
49
+ ): AttentionBackwardDataResult;
50
+
51
+ export declare function buildAttentionSoftmaxCache(
52
+ q: Tensor,
53
+ k: Tensor,
54
+ options: AttentionBackwardOptions
55
+ ): Promise<Tensor>;
56
+
57
+ export declare function attentionBackwardCpu(
58
+ q: Tensor,
59
+ k: Tensor,
60
+ v: Tensor,
61
+ softmax: Tensor | null,
62
+ gradOutput: Tensor,
63
+ options: AttentionBackwardOptions
64
+ ): Promise<AttentionBackwardResult>;
@@ -0,0 +1,22 @@
1
+ import type { TrainingConfigSchema } from '../../config/training-defaults.d.ts';
2
+ import type { Tensor } from '../../gpu/tensor.js';
3
+
4
+ export declare function recordAttentionForward(
5
+ q: Tensor,
6
+ k: Tensor,
7
+ v: Tensor,
8
+ config: TrainingConfigSchema,
9
+ tape: unknown,
10
+ options?: {
11
+ seqLen?: number;
12
+ numHeads?: number;
13
+ numKVHeads?: number;
14
+ headDim?: number;
15
+ scale?: number;
16
+ causal?: boolean;
17
+ startPos?: number;
18
+ attnSoftcap?: number;
19
+ slidingWindow?: number;
20
+ kvLen?: number;
21
+ }
22
+ ): Promise<{ output: Tensor; softmax: Tensor | null }>;
@@ -0,0 +1,82 @@
1
+ import { runAttention } from '../../gpu/kernels/index.js';
2
+ import { buildAttentionSoftmaxCache } from './attention-backward.js';
3
+ import { OpType } from './autograd.js';
4
+
5
+ export async function recordAttentionForward(
6
+ q,
7
+ k,
8
+ v,
9
+ config,
10
+ tape,
11
+ options = {}
12
+ ) {
13
+ if (!tape || typeof tape.record !== 'function') {
14
+ throw new Error('recordAttentionForward requires an autograd tape');
15
+ }
16
+
17
+ const {
18
+ seqLen,
19
+ numHeads,
20
+ numKVHeads = numHeads,
21
+ headDim,
22
+ scale,
23
+ causal = true,
24
+ startPos = 0,
25
+ attnSoftcap = 0,
26
+ slidingWindow = 0,
27
+ kvLen = seqLen,
28
+ } = options;
29
+
30
+ if (!seqLen || !numHeads || !headDim) {
31
+ throw new Error('recordAttentionForward requires seqLen, numHeads, and headDim');
32
+ }
33
+
34
+ const resolvedScale = scale ?? 1.0 / Math.sqrt(headDim);
35
+ const recomputeForward = Boolean(config?.training?.attention?.recomputeForward);
36
+ const softmax = recomputeForward
37
+ ? null
38
+ : await buildAttentionSoftmaxCache(q, k, {
39
+ seqLen,
40
+ numHeads,
41
+ headDim,
42
+ scale: resolvedScale,
43
+ causal,
44
+ });
45
+
46
+ const attentionOptions = {
47
+ seqLen,
48
+ kvLen,
49
+ numHeads,
50
+ numKVHeads,
51
+ headDim,
52
+ scale: resolvedScale,
53
+ causal,
54
+ startPos,
55
+ attnSoftcap,
56
+ slidingWindow,
57
+ };
58
+
59
+ const output = await tape.record(
60
+ OpType.ATTENTION,
61
+ (inputQ, inputK, inputV) => runAttention(
62
+ inputQ,
63
+ inputK,
64
+ inputV,
65
+ null,
66
+ numHeads,
67
+ headDim,
68
+ attentionOptions
69
+ ),
70
+ [q, k, v, softmax],
71
+ {
72
+ seqLen,
73
+ numHeads,
74
+ headDim,
75
+ scale: resolvedScale,
76
+ causal,
77
+ recomputeForward,
78
+ }
79
+ );
80
+
81
+ return { output, softmax };
82
+ }