doppler-gpu 0.4.8 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/models/catalog.json +21 -7
- package/package.json +8 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
- package/src/config/kernels/backward-registry.json +13 -2
- package/src/config/kernels/kernel-ref-digests.js +8 -4
- package/src/config/kernels/registry.json +1035 -2677
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
- package/src/config/schema/training-metrics.schema.js +4 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
- package/src/experimental/training/attention-backward.js +123 -55
- package/src/experimental/training/autograd.js +101 -5
- package/src/experimental/training/clip.js +6 -1
- package/src/experimental/training/datasets/text-pairs.js +80 -8
- package/src/experimental/training/distillation/student-fixture.js +470 -114
- package/src/experimental/training/lora-pipeline.js +202 -27
- package/src/experimental/training/lora.js +10 -4
- package/src/experimental/training/optimizer.js +9 -1
- package/src/experimental/training/runner.js +18 -4
- package/src/experimental/training/workloads.js +11 -0
- package/src/gpu/kernels/backward/adam.js +6 -4
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
- package/src/gpu/kernels/backward/index.d.ts +3 -0
- package/src/gpu/kernels/backward/index.js +3 -0
- package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
- package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
- package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
- package/src/gpu/kernels/backward/utils.d.ts +12 -2
- package/src/gpu/kernels/backward/utils.js +9 -2
- package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
- package/src/gpu/kernels/index.d.ts +5 -0
- package/src/gpu/kernels/index.js +5 -0
- package/src/gpu/kernels/matmul-selection.js +5 -0
- package/src/gpu/kernels/silu.d.ts +1 -0
- package/src/gpu/kernels/silu.js +19 -4
- package/src/inference/pipelines/text/embed.js +1 -1
- package/src/inference/pipelines/text/execution-v1.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +2 -0
- package/src/inference/pipelines/text/generator.js +29 -0
- package/src/inference/pipelines/text/lora-apply.js +4 -4
- package/src/inference/pipelines/text/moe-cpu.js +1 -0
- package/src/inference/pipelines/text/moe-gpu.js +1 -0
- package/src/inference/tokenizers/bundled.js +105 -32
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/tooling/node-converter.js +20 -5
- package/src/tooling-experimental-exports.shared.d.ts +25 -0
- package/src/tooling-experimental-exports.shared.js +18 -0
- package/src/version.js +1 -1
|
@@ -0,0 +1,573 @@
|
|
|
1
|
+
import { isPlainObject } from '../../utils/plain-object.js';
|
|
2
|
+
import { sha256Hex } from '../../utils/sha256.js';
|
|
3
|
+
import { stableSortObject } from '../../utils/stable-sort-object.js';
|
|
4
|
+
|
|
5
|
+
export const TRAINER_ARTIFACT_BRIDGE_SCHEMA_ID = 'doppler.trainer-artifact-bridge/v1';
|
|
6
|
+
export const TRAINER_ARTIFACT_IMPORT_PLAN_SCHEMA_ID = 'doppler.trainer-artifact-import-plan/v1';
|
|
7
|
+
export const TRAINER_ARTIFACT_PARITY_EVIDENCE_SCHEMA_ID = 'doppler.trainer-artifact-parity-evidence/v1';
|
|
8
|
+
export const TRAINER_ARTIFACT_PARITY_RECEIPT_SCHEMA_ID = 'doppler.trainer-artifact-parity-receipt/v1';
|
|
9
|
+
|
|
10
|
+
export const TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT = 'full_checkpoint';
|
|
11
|
+
export const TRAINER_ARTIFACT_KIND_PEFT_ADAPTER = 'peft_adapter';
|
|
12
|
+
|
|
13
|
+
export const TRANSLATION_FULL_CHECKPOINT_PARITY_CHECKS = Object.freeze([
|
|
14
|
+
'source_artifact_byte_identity',
|
|
15
|
+
'tokenizer_and_prompt_identity',
|
|
16
|
+
'architecture_and_conversion_lineage',
|
|
17
|
+
'evaluation_input_identity',
|
|
18
|
+
'first_token_logits',
|
|
19
|
+
'selected_tokens',
|
|
20
|
+
'completions',
|
|
21
|
+
'exact_browser_artifact_capability',
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
export const COLUMBO_QWEN_ADAPTER_PARITY_CHECKS = Object.freeze([
|
|
25
|
+
'source_base_vs_converted_base',
|
|
26
|
+
'reference_converted_base_vs_doppler_base',
|
|
27
|
+
'source_adapter_vs_converted_adapter',
|
|
28
|
+
'reference_converted_adapter_vs_doppler_adapter',
|
|
29
|
+
'browser_capability_evaluation',
|
|
30
|
+
]);
|
|
31
|
+
|
|
32
|
+
const SHA256_PATTERN = /^[0-9a-f]{64}$/;
|
|
33
|
+
const FILE_ROLES_BY_KIND = Object.freeze({
|
|
34
|
+
[TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT]: Object.freeze([
|
|
35
|
+
'weights',
|
|
36
|
+
'config',
|
|
37
|
+
'generation_config',
|
|
38
|
+
]),
|
|
39
|
+
[TRAINER_ARTIFACT_KIND_PEFT_ADAPTER]: Object.freeze([
|
|
40
|
+
'adapter_weights',
|
|
41
|
+
'adapter_config',
|
|
42
|
+
'doppler_adapter_manifest',
|
|
43
|
+
'runtime_adapter_manifest',
|
|
44
|
+
'training_report',
|
|
45
|
+
]),
|
|
46
|
+
});
|
|
47
|
+
const PARITY_CHECKS_BY_PROFILE = Object.freeze({
|
|
48
|
+
translation_full_checkpoint: TRANSLATION_FULL_CHECKPOINT_PARITY_CHECKS,
|
|
49
|
+
columbo_qwen_adapter: COLUMBO_QWEN_ADAPTER_PARITY_CHECKS,
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
function stableJson(value) {
|
|
53
|
+
return JSON.stringify(stableSortObject(value));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function hashStableJson(value) {
|
|
57
|
+
return sha256Hex(stableJson(value));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function requireText(value, label) {
|
|
61
|
+
const text = typeof value === 'string' ? value.trim() : '';
|
|
62
|
+
if (!text) {
|
|
63
|
+
throw new Error(`trainer artifact bridge: ${label} is required.`);
|
|
64
|
+
}
|
|
65
|
+
return text;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function requireSha256(value, label) {
|
|
69
|
+
const digest = requireText(value, label).replace(/^sha256:/, '').toLowerCase();
|
|
70
|
+
if (!SHA256_PATTERN.test(digest)) {
|
|
71
|
+
throw new Error(`trainer artifact bridge: ${label} must be a SHA-256 digest.`);
|
|
72
|
+
}
|
|
73
|
+
return digest;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function requirePositiveInteger(value, label) {
|
|
77
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
78
|
+
throw new Error(`trainer artifact bridge: ${label} must be a positive integer.`);
|
|
79
|
+
}
|
|
80
|
+
return value;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function assertRepoRelativePath(value, label, allowEmpty = false) {
|
|
84
|
+
const path = typeof value === 'string' ? value.trim().replaceAll('\\', '/') : '';
|
|
85
|
+
if (!path && allowEmpty) return '';
|
|
86
|
+
if (!path || path.startsWith('/') || path.split('/').includes('..')) {
|
|
87
|
+
throw new Error(`trainer artifact bridge: ${label} must be a repository-relative path.`);
|
|
88
|
+
}
|
|
89
|
+
return path.replace(/^\.\//, '').replace(/\/$/, '');
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function normalizeFileIdentity(file, label) {
|
|
93
|
+
if (!isPlainObject(file)) {
|
|
94
|
+
throw new Error(`trainer artifact bridge: ${label} must be an object.`);
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
id: requireText(file.id, `${label}.id`),
|
|
98
|
+
role: requireText(file.role, `${label}.role`),
|
|
99
|
+
repository: requireText(file.repository, `${label}.repository`),
|
|
100
|
+
rootPath: assertRepoRelativePath(file.rootPath, `${label}.rootPath`, true),
|
|
101
|
+
path: assertRepoRelativePath(file.path, `${label}.path`),
|
|
102
|
+
sha256: requireSha256(file.sha256, `${label}.sha256`),
|
|
103
|
+
bytes: requirePositiveInteger(file.bytes, `${label}.bytes`),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function normalizeFileList(files, label) {
|
|
108
|
+
if (!Array.isArray(files) || files.length === 0) {
|
|
109
|
+
throw new Error(`trainer artifact bridge: ${label} must be a non-empty array.`);
|
|
110
|
+
}
|
|
111
|
+
const normalized = files.map((file, index) => normalizeFileIdentity(file, `${label}[${index}]`));
|
|
112
|
+
const ids = new Set();
|
|
113
|
+
for (const file of normalized) {
|
|
114
|
+
if (ids.has(file.id)) {
|
|
115
|
+
throw new Error(`trainer artifact bridge: duplicate file id "${file.id}".`);
|
|
116
|
+
}
|
|
117
|
+
ids.add(file.id);
|
|
118
|
+
}
|
|
119
|
+
return normalized;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function normalizeArchitecture(architecture) {
|
|
123
|
+
if (!isPlainObject(architecture)) {
|
|
124
|
+
throw new Error('trainer artifact bridge: baseModel.architecture must be an object.');
|
|
125
|
+
}
|
|
126
|
+
const architectures = Array.isArray(architecture.architectures)
|
|
127
|
+
? architecture.architectures.map((entry, index) => requireText(entry, `baseModel.architecture.architectures[${index}]`))
|
|
128
|
+
: [];
|
|
129
|
+
if (architectures.length === 0) {
|
|
130
|
+
throw new Error('trainer artifact bridge: baseModel.architecture.architectures must be non-empty.');
|
|
131
|
+
}
|
|
132
|
+
return {
|
|
133
|
+
architectures,
|
|
134
|
+
modelType: requireText(architecture.modelType, 'baseModel.architecture.modelType'),
|
|
135
|
+
hiddenSize: requirePositiveInteger(architecture.hiddenSize, 'baseModel.architecture.hiddenSize'),
|
|
136
|
+
intermediateSize: requirePositiveInteger(
|
|
137
|
+
architecture.intermediateSize,
|
|
138
|
+
'baseModel.architecture.intermediateSize'
|
|
139
|
+
),
|
|
140
|
+
layers: requirePositiveInteger(architecture.layers, 'baseModel.architecture.layers'),
|
|
141
|
+
attentionHeads: requirePositiveInteger(
|
|
142
|
+
architecture.attentionHeads,
|
|
143
|
+
'baseModel.architecture.attentionHeads'
|
|
144
|
+
),
|
|
145
|
+
keyValueHeads: requirePositiveInteger(
|
|
146
|
+
architecture.keyValueHeads,
|
|
147
|
+
'baseModel.architecture.keyValueHeads'
|
|
148
|
+
),
|
|
149
|
+
headDim: requirePositiveInteger(architecture.headDim, 'baseModel.architecture.headDim'),
|
|
150
|
+
vocabularySize: requirePositiveInteger(
|
|
151
|
+
architecture.vocabularySize,
|
|
152
|
+
'baseModel.architecture.vocabularySize'
|
|
153
|
+
),
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function normalizeSelection(selection) {
|
|
158
|
+
if (!isPlainObject(selection)) {
|
|
159
|
+
throw new Error('trainer artifact bridge: selection must be an object.');
|
|
160
|
+
}
|
|
161
|
+
const status = requireText(selection.status, 'selection.status');
|
|
162
|
+
if (status !== 'not_selected' && status !== 'selected') {
|
|
163
|
+
throw new Error('trainer artifact bridge: selection.status must be "not_selected" or "selected".');
|
|
164
|
+
}
|
|
165
|
+
const receipt = selection.receipt == null ? null : requireText(selection.receipt, 'selection.receipt');
|
|
166
|
+
if (status === 'not_selected' && receipt !== null) {
|
|
167
|
+
throw new Error('trainer artifact bridge: an unselected artifact cannot carry a selection receipt.');
|
|
168
|
+
}
|
|
169
|
+
if (status === 'selected' && receipt === null) {
|
|
170
|
+
throw new Error('trainer artifact bridge: a selected artifact requires a selection receipt.');
|
|
171
|
+
}
|
|
172
|
+
return {
|
|
173
|
+
authority: requireText(selection.authority, 'selection.authority'),
|
|
174
|
+
status,
|
|
175
|
+
receipt,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function normalizeParity(parity, artifactKind) {
|
|
180
|
+
if (!isPlainObject(parity)) {
|
|
181
|
+
throw new Error('trainer artifact bridge: parity must be an object.');
|
|
182
|
+
}
|
|
183
|
+
const profile = requireText(parity.profile, 'parity.profile');
|
|
184
|
+
const canonicalChecks = PARITY_CHECKS_BY_PROFILE[profile];
|
|
185
|
+
if (!canonicalChecks) {
|
|
186
|
+
throw new Error(`trainer artifact bridge: unsupported parity profile "${profile}".`);
|
|
187
|
+
}
|
|
188
|
+
const expectedProfile = artifactKind === TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT
|
|
189
|
+
? 'translation_full_checkpoint'
|
|
190
|
+
: 'columbo_qwen_adapter';
|
|
191
|
+
if (profile !== expectedProfile) {
|
|
192
|
+
throw new Error(
|
|
193
|
+
`trainer artifact bridge: artifact kind "${artifactKind}" requires parity profile "${expectedProfile}".`
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
const requiredChecks = Array.isArray(parity.requiredChecks) ? parity.requiredChecks : [];
|
|
197
|
+
if (stableJson(requiredChecks) !== stableJson(canonicalChecks)) {
|
|
198
|
+
throw new Error(`trainer artifact bridge: parity.requiredChecks must match profile "${profile}".`);
|
|
199
|
+
}
|
|
200
|
+
return { profile, requiredChecks: [...canonicalChecks] };
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function assertArtifactRoles(artifact) {
|
|
204
|
+
const expectedRoles = FILE_ROLES_BY_KIND[artifact.kind];
|
|
205
|
+
const roles = new Set(artifact.files.map((file) => file.role));
|
|
206
|
+
for (const role of expectedRoles) {
|
|
207
|
+
if (!roles.has(role)) {
|
|
208
|
+
throw new Error(`trainer artifact bridge: artifact file role "${role}" is required.`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function normalizeDescriptor(descriptor) {
|
|
214
|
+
if (!isPlainObject(descriptor)) {
|
|
215
|
+
throw new Error('trainer artifact bridge: descriptor must be an object.');
|
|
216
|
+
}
|
|
217
|
+
if (descriptor.schema !== TRAINER_ARTIFACT_BRIDGE_SCHEMA_ID) {
|
|
218
|
+
throw new Error(`trainer artifact bridge: schema must be "${TRAINER_ARTIFACT_BRIDGE_SCHEMA_ID}".`);
|
|
219
|
+
}
|
|
220
|
+
const kind = requireText(descriptor.artifact?.kind, 'artifact.kind');
|
|
221
|
+
if (!FILE_ROLES_BY_KIND[kind]) {
|
|
222
|
+
throw new Error(`trainer artifact bridge: unsupported artifact kind "${kind}".`);
|
|
223
|
+
}
|
|
224
|
+
const role = requireText(descriptor.artifact?.role, 'artifact.role');
|
|
225
|
+
if (
|
|
226
|
+
role !== 'diagnostic_baseline'
|
|
227
|
+
&& role !== 'diagnostic_candidate'
|
|
228
|
+
&& role !== 'selected_candidate'
|
|
229
|
+
) {
|
|
230
|
+
throw new Error('trainer artifact bridge: artifact.role is unsupported.');
|
|
231
|
+
}
|
|
232
|
+
const artifact = {
|
|
233
|
+
kind,
|
|
234
|
+
role,
|
|
235
|
+
format: requireText(descriptor.artifact?.format, 'artifact.format'),
|
|
236
|
+
repository: requireText(descriptor.artifact?.repository, 'artifact.repository'),
|
|
237
|
+
rootPath: assertRepoRelativePath(descriptor.artifact?.rootPath, 'artifact.rootPath', true),
|
|
238
|
+
files: normalizeFileList(descriptor.artifact?.files, 'artifact.files'),
|
|
239
|
+
};
|
|
240
|
+
assertArtifactRoles(artifact);
|
|
241
|
+
|
|
242
|
+
const tokenizerFiles = normalizeFileList(
|
|
243
|
+
descriptor.baseModel?.tokenizer?.files,
|
|
244
|
+
'baseModel.tokenizer.files'
|
|
245
|
+
);
|
|
246
|
+
const promptContract = normalizeFileIdentity(
|
|
247
|
+
descriptor.baseModel?.tokenizer?.promptContract,
|
|
248
|
+
'baseModel.tokenizer.promptContract'
|
|
249
|
+
);
|
|
250
|
+
const selection = normalizeSelection(descriptor.selection);
|
|
251
|
+
if (
|
|
252
|
+
(role === 'diagnostic_baseline' || role === 'diagnostic_candidate')
|
|
253
|
+
&& selection.status !== 'not_selected'
|
|
254
|
+
) {
|
|
255
|
+
throw new Error('trainer artifact bridge: a diagnostic artifact cannot be selected.');
|
|
256
|
+
}
|
|
257
|
+
if (role === 'selected_candidate' && selection.status !== 'selected') {
|
|
258
|
+
throw new Error('trainer artifact bridge: a selected candidate requires selected status.');
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
const conversionConfig = descriptor.conversion?.config == null
|
|
262
|
+
? null
|
|
263
|
+
: normalizeFileIdentity(descriptor.conversion.config, 'conversion.config');
|
|
264
|
+
const evaluationFiles = normalizeFileList(descriptor.evaluation?.files, 'evaluation.files');
|
|
265
|
+
const normalized = {
|
|
266
|
+
schema: TRAINER_ARTIFACT_BRIDGE_SCHEMA_ID,
|
|
267
|
+
bridgeId: requireText(descriptor.bridgeId, 'bridgeId'),
|
|
268
|
+
sourceContractId: requireText(descriptor.sourceContractId, 'sourceContractId'),
|
|
269
|
+
artifact,
|
|
270
|
+
baseModel: {
|
|
271
|
+
modelId: requireText(descriptor.baseModel?.modelId, 'baseModel.modelId'),
|
|
272
|
+
checkpointSha256: requireSha256(
|
|
273
|
+
descriptor.baseModel?.checkpointSha256,
|
|
274
|
+
'baseModel.checkpointSha256'
|
|
275
|
+
),
|
|
276
|
+
tokenizer: { files: tokenizerFiles, promptContract },
|
|
277
|
+
architecture: normalizeArchitecture(descriptor.baseModel?.architecture),
|
|
278
|
+
},
|
|
279
|
+
conversion: {
|
|
280
|
+
owner: requireText(descriptor.conversion?.owner, 'conversion.owner'),
|
|
281
|
+
sourceArtifactSha256: requireSha256(
|
|
282
|
+
descriptor.conversion?.sourceArtifactSha256,
|
|
283
|
+
'conversion.sourceArtifactSha256'
|
|
284
|
+
),
|
|
285
|
+
config: conversionConfig,
|
|
286
|
+
runtimeArtifact: isPlainObject(descriptor.conversion?.runtimeArtifact)
|
|
287
|
+
? stableSortObject(descriptor.conversion.runtimeArtifact)
|
|
288
|
+
: null,
|
|
289
|
+
},
|
|
290
|
+
evaluation: {
|
|
291
|
+
populationRole: requireText(descriptor.evaluation?.populationRole, 'evaluation.populationRole'),
|
|
292
|
+
files: evaluationFiles,
|
|
293
|
+
},
|
|
294
|
+
selection,
|
|
295
|
+
parity: normalizeParity(descriptor.parity, kind),
|
|
296
|
+
claimBoundary: requireText(descriptor.claimBoundary, 'claimBoundary'),
|
|
297
|
+
};
|
|
298
|
+
if (normalized.conversion.sourceArtifactSha256 !== normalized.baseModel.checkpointSha256) {
|
|
299
|
+
throw new Error('trainer artifact bridge: conversion source hash must match the base checkpoint hash.');
|
|
300
|
+
}
|
|
301
|
+
return normalized;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function toFileIdentity(id, role, repository, rootPath, identity) {
|
|
305
|
+
return {
|
|
306
|
+
id,
|
|
307
|
+
role,
|
|
308
|
+
repository,
|
|
309
|
+
rootPath,
|
|
310
|
+
path: identity.path,
|
|
311
|
+
sha256: identity.sha256,
|
|
312
|
+
bytes: identity.bytes,
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
function splitQualifiedPath(value, fallbackRepository) {
|
|
317
|
+
const text = requireText(value, 'qualified file path');
|
|
318
|
+
const delimiter = text.indexOf(':');
|
|
319
|
+
if (delimiter < 0) {
|
|
320
|
+
return { repository: fallbackRepository, path: text };
|
|
321
|
+
}
|
|
322
|
+
return {
|
|
323
|
+
repository: text.slice(0, delimiter),
|
|
324
|
+
path: text.slice(delimiter + 1),
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
export function normalizeGammaTrainerArtifactHandoff(contract) {
|
|
329
|
+
if (!isPlainObject(contract)) {
|
|
330
|
+
throw new Error('trainer artifact bridge: Gamma handoff contract must be an object.');
|
|
331
|
+
}
|
|
332
|
+
const checkpoint = contract.sourceCheckpoint;
|
|
333
|
+
const repository = requireText(checkpoint?.repository, 'sourceCheckpoint.repository');
|
|
334
|
+
const checkpointRoot = assertRepoRelativePath(checkpoint?.path, 'sourceCheckpoint.path');
|
|
335
|
+
const conversionConfig = contract.conversionLineage?.conversionConfig;
|
|
336
|
+
const artifactFiles = [
|
|
337
|
+
toFileIdentity('checkpoint.weights', 'weights', repository, checkpointRoot, checkpoint.weights),
|
|
338
|
+
toFileIdentity('checkpoint.config', 'config', repository, checkpointRoot, checkpoint.config),
|
|
339
|
+
toFileIdentity(
|
|
340
|
+
'checkpoint.generation_config',
|
|
341
|
+
'generation_config',
|
|
342
|
+
repository,
|
|
343
|
+
checkpointRoot,
|
|
344
|
+
checkpoint.generationConfig
|
|
345
|
+
),
|
|
346
|
+
];
|
|
347
|
+
const tokenizerFiles = contract.tokenizer.files.map((identity, index) => toFileIdentity(
|
|
348
|
+
`tokenizer.${index}`,
|
|
349
|
+
'tokenizer_asset',
|
|
350
|
+
repository,
|
|
351
|
+
checkpointRoot,
|
|
352
|
+
identity
|
|
353
|
+
));
|
|
354
|
+
const promptContract = toFileIdentity(
|
|
355
|
+
'tokenizer.prompt_contract',
|
|
356
|
+
'prompt_contract',
|
|
357
|
+
repository,
|
|
358
|
+
checkpointRoot,
|
|
359
|
+
contract.tokenizer.promptContract
|
|
360
|
+
);
|
|
361
|
+
const evaluationFiles = [
|
|
362
|
+
['evaluation.population', 'population', contract.evaluationInputs.population],
|
|
363
|
+
['evaluation.summary', 'summary', contract.evaluationInputs.summary],
|
|
364
|
+
['evaluation.predictions', 'predictions', contract.evaluationInputs.predictions],
|
|
365
|
+
].map(([id, role, identity]) => {
|
|
366
|
+
const qualified = splitQualifiedPath(identity.path, repository);
|
|
367
|
+
return toFileIdentity(id, role, qualified.repository, '', { ...identity, path: qualified.path });
|
|
368
|
+
});
|
|
369
|
+
const selectionStatus = contract.selection?.status === 'gamma_selected' ? 'selected' : 'not_selected';
|
|
370
|
+
|
|
371
|
+
return normalizeDescriptor({
|
|
372
|
+
schema: TRAINER_ARTIFACT_BRIDGE_SCHEMA_ID,
|
|
373
|
+
bridgeId: `bridge.${contract.contractId}`,
|
|
374
|
+
sourceContractId: contract.contractId,
|
|
375
|
+
artifact: {
|
|
376
|
+
kind: TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT,
|
|
377
|
+
role: contract.artifactRole,
|
|
378
|
+
format: checkpoint.format,
|
|
379
|
+
repository,
|
|
380
|
+
rootPath: checkpointRoot,
|
|
381
|
+
files: artifactFiles,
|
|
382
|
+
},
|
|
383
|
+
baseModel: {
|
|
384
|
+
modelId: contract.conversionLineage.diagnosticHostedArtifact.modelId,
|
|
385
|
+
checkpointSha256: checkpoint.weights.sha256,
|
|
386
|
+
tokenizer: { files: tokenizerFiles, promptContract },
|
|
387
|
+
architecture: contract.architecture,
|
|
388
|
+
},
|
|
389
|
+
conversion: {
|
|
390
|
+
owner: contract.authorities.conversionAndParity,
|
|
391
|
+
sourceArtifactSha256: contract.conversionLineage.sourceWeightsSha256,
|
|
392
|
+
config: toFileIdentity(
|
|
393
|
+
'conversion.config',
|
|
394
|
+
'conversion_config',
|
|
395
|
+
'clocksmith/doppler',
|
|
396
|
+
'',
|
|
397
|
+
conversionConfig
|
|
398
|
+
),
|
|
399
|
+
runtimeArtifact: contract.conversionLineage.diagnosticHostedArtifact,
|
|
400
|
+
},
|
|
401
|
+
evaluation: {
|
|
402
|
+
populationRole: contract.evaluationInputs.populationRole,
|
|
403
|
+
files: evaluationFiles,
|
|
404
|
+
},
|
|
405
|
+
selection: {
|
|
406
|
+
authority: contract.authorities.bf16Selection,
|
|
407
|
+
status: selectionStatus,
|
|
408
|
+
receipt: contract.selection.receipt,
|
|
409
|
+
},
|
|
410
|
+
parity: {
|
|
411
|
+
profile: 'translation_full_checkpoint',
|
|
412
|
+
requiredChecks: contract.parity.requiredChecks,
|
|
413
|
+
},
|
|
414
|
+
claimBoundary: contract.claimBoundary,
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
export function validateTrainerArtifactBridgeDescriptor(descriptor) {
|
|
419
|
+
try {
|
|
420
|
+
return { valid: true, descriptor: normalizeDescriptor(descriptor), errors: [] };
|
|
421
|
+
} catch (error) {
|
|
422
|
+
return {
|
|
423
|
+
valid: false,
|
|
424
|
+
descriptor: null,
|
|
425
|
+
errors: [error instanceof Error ? error.message : String(error)],
|
|
426
|
+
};
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
export function assertTrainerArtifactCandidateEntry(descriptor) {
|
|
431
|
+
const normalized = normalizeDescriptor(descriptor);
|
|
432
|
+
if (normalized.artifact.role !== 'selected_candidate') {
|
|
433
|
+
throw new Error('trainer artifact bridge: candidate entry denied; artifact is not a selected candidate.');
|
|
434
|
+
}
|
|
435
|
+
if (normalized.selection.status !== 'selected' || !normalized.selection.receipt) {
|
|
436
|
+
throw new Error('trainer artifact bridge: candidate entry denied; selection receipt is required.');
|
|
437
|
+
}
|
|
438
|
+
const expectedAuthority = normalized.artifact.kind === TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT
|
|
439
|
+
? 'clocksmith/gamma'
|
|
440
|
+
: 'clocksmith/columbo';
|
|
441
|
+
if (normalized.selection.authority !== expectedAuthority) {
|
|
442
|
+
throw new Error(
|
|
443
|
+
`trainer artifact bridge: candidate entry denied; selection authority must be "${expectedAuthority}".`
|
|
444
|
+
);
|
|
445
|
+
}
|
|
446
|
+
return normalized;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
export function buildTrainerArtifactImportPlan(descriptor, verificationReceipt = null) {
|
|
450
|
+
const normalized = normalizeDescriptor(descriptor);
|
|
451
|
+
if (verificationReceipt && verificationReceipt.ok !== true) {
|
|
452
|
+
throw new Error('trainer artifact bridge: import plan requires a passing verification receipt.');
|
|
453
|
+
}
|
|
454
|
+
const candidateCompetitionAllowed = normalized.artifact.role === 'selected_candidate'
|
|
455
|
+
&& normalized.selection.status === 'selected'
|
|
456
|
+
&& Boolean(normalized.selection.receipt);
|
|
457
|
+
const requiredRoles = FILE_ROLES_BY_KIND[normalized.artifact.kind];
|
|
458
|
+
const filesByRole = Object.fromEntries(
|
|
459
|
+
normalized.artifact.files.map((file) => [file.role, file.id])
|
|
460
|
+
);
|
|
461
|
+
const plan = {
|
|
462
|
+
schema: TRAINER_ARTIFACT_IMPORT_PLAN_SCHEMA_ID,
|
|
463
|
+
bridgeId: normalized.bridgeId,
|
|
464
|
+
artifactKind: normalized.artifact.kind,
|
|
465
|
+
entrypoint: normalized.artifact.kind === TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT
|
|
466
|
+
? 'resolveNodeSourceRuntimeBundle'
|
|
467
|
+
: 'loadLoRAWeights',
|
|
468
|
+
source: {
|
|
469
|
+
repository: normalized.artifact.repository,
|
|
470
|
+
rootPath: normalized.artifact.rootPath,
|
|
471
|
+
format: normalized.artifact.format,
|
|
472
|
+
requiredRoles,
|
|
473
|
+
filesByRole,
|
|
474
|
+
},
|
|
475
|
+
baseModel: normalized.baseModel,
|
|
476
|
+
conversion: normalized.conversion,
|
|
477
|
+
verificationReceiptHash: verificationReceipt?.receiptHash ?? null,
|
|
478
|
+
admission: {
|
|
479
|
+
identityVerificationRequired: true,
|
|
480
|
+
parityExecutionAllowed: true,
|
|
481
|
+
candidateCompetitionAllowed,
|
|
482
|
+
promotionAllowed: false,
|
|
483
|
+
selectionAuthority: normalized.selection.authority,
|
|
484
|
+
selectionReceipt: normalized.selection.receipt,
|
|
485
|
+
},
|
|
486
|
+
};
|
|
487
|
+
return { ...plan, planHash: hashStableJson(plan) };
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
export function buildTrainerArtifactParityTemplate(descriptor, verificationReceipt) {
|
|
491
|
+
const normalized = normalizeDescriptor(descriptor);
|
|
492
|
+
if (!verificationReceipt || verificationReceipt.ok !== true) {
|
|
493
|
+
throw new Error('trainer artifact bridge: parity template requires a passing verification receipt.');
|
|
494
|
+
}
|
|
495
|
+
const identityCheckIds = new Set([
|
|
496
|
+
'source_artifact_byte_identity',
|
|
497
|
+
'tokenizer_and_prompt_identity',
|
|
498
|
+
'architecture_and_conversion_lineage',
|
|
499
|
+
'evaluation_input_identity',
|
|
500
|
+
]);
|
|
501
|
+
return {
|
|
502
|
+
schema: TRAINER_ARTIFACT_PARITY_EVIDENCE_SCHEMA_ID,
|
|
503
|
+
bridgeId: normalized.bridgeId,
|
|
504
|
+
profile: normalized.parity.profile,
|
|
505
|
+
artifactIdentitySha256: verificationReceipt.artifactIdentitySha256,
|
|
506
|
+
checks: normalized.parity.requiredChecks.map((id) => ({
|
|
507
|
+
id,
|
|
508
|
+
status: identityCheckIds.has(id) ? 'pass' : 'pending',
|
|
509
|
+
evidenceHash: identityCheckIds.has(id) ? verificationReceipt.receiptHash : null,
|
|
510
|
+
artifactIdentitySha256: verificationReceipt.artifactIdentitySha256,
|
|
511
|
+
upstreamDecision: identityCheckIds.has(id) ? 'pass' : null,
|
|
512
|
+
})),
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
export function verifyTrainerArtifactParityEvidence(descriptor, verificationReceipt, evidence) {
|
|
517
|
+
const normalized = normalizeDescriptor(descriptor);
|
|
518
|
+
if (!verificationReceipt || verificationReceipt.ok !== true) {
|
|
519
|
+
throw new Error('trainer artifact bridge: parity verification requires a passing identity receipt.');
|
|
520
|
+
}
|
|
521
|
+
const blockers = [];
|
|
522
|
+
if (!isPlainObject(evidence) || evidence.schema !== TRAINER_ARTIFACT_PARITY_EVIDENCE_SCHEMA_ID) {
|
|
523
|
+
blockers.push('invalid_parity_evidence_schema');
|
|
524
|
+
}
|
|
525
|
+
if (evidence?.bridgeId !== normalized.bridgeId) blockers.push('parity_bridge_id_mismatch');
|
|
526
|
+
if (evidence?.profile !== normalized.parity.profile) blockers.push('parity_profile_mismatch');
|
|
527
|
+
if (evidence?.artifactIdentitySha256 !== verificationReceipt.artifactIdentitySha256) {
|
|
528
|
+
blockers.push('parity_artifact_identity_mismatch');
|
|
529
|
+
}
|
|
530
|
+
const checks = Array.isArray(evidence?.checks) ? evidence.checks : [];
|
|
531
|
+
const checkById = new Map();
|
|
532
|
+
for (const check of checks) {
|
|
533
|
+
const id = typeof check?.id === 'string' ? check.id : '';
|
|
534
|
+
if (!id) {
|
|
535
|
+
blockers.push('parity_check_missing_id');
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
538
|
+
if (checkById.has(id)) blockers.push(`duplicate_parity_check:${id}`);
|
|
539
|
+
checkById.set(id, check);
|
|
540
|
+
}
|
|
541
|
+
for (const id of normalized.parity.requiredChecks) {
|
|
542
|
+
const check = checkById.get(id);
|
|
543
|
+
if (!check) {
|
|
544
|
+
blockers.push(`missing_parity_check:${id}`);
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
if (check.status !== 'pass') blockers.push(`parity_check_not_passed:${id}`);
|
|
548
|
+
if (check.upstreamDecision !== 'pass') blockers.push(`parity_upstream_not_passed:${id}`);
|
|
549
|
+
if (check.artifactIdentitySha256 !== verificationReceipt.artifactIdentitySha256) {
|
|
550
|
+
blockers.push(`parity_check_identity_mismatch:${id}`);
|
|
551
|
+
}
|
|
552
|
+
const evidenceHash = typeof check.evidenceHash === 'string'
|
|
553
|
+
? check.evidenceHash.replace(/^sha256:/, '').toLowerCase()
|
|
554
|
+
: '';
|
|
555
|
+
if (!SHA256_PATTERN.test(evidenceHash)) blockers.push(`invalid_parity_evidence_hash:${id}`);
|
|
556
|
+
}
|
|
557
|
+
for (const id of checkById.keys()) {
|
|
558
|
+
if (!normalized.parity.requiredChecks.includes(id)) blockers.push(`unexpected_parity_check:${id}`);
|
|
559
|
+
}
|
|
560
|
+
const core = {
|
|
561
|
+
schema: TRAINER_ARTIFACT_PARITY_RECEIPT_SCHEMA_ID,
|
|
562
|
+
bridgeId: normalized.bridgeId,
|
|
563
|
+
profile: normalized.parity.profile,
|
|
564
|
+
artifactIdentitySha256: verificationReceipt.artifactIdentitySha256,
|
|
565
|
+
identityReceiptHash: verificationReceipt.receiptHash,
|
|
566
|
+
decision: blockers.length === 0 ? 'pass' : 'block',
|
|
567
|
+
blockers: [...new Set(blockers)].sort(),
|
|
568
|
+
checkEvidenceHashes: Object.fromEntries(
|
|
569
|
+
normalized.parity.requiredChecks.map((id) => [id, checkById.get(id)?.evidenceHash ?? null])
|
|
570
|
+
),
|
|
571
|
+
};
|
|
572
|
+
return { ...core, receiptHash: hashStableJson(core) };
|
|
573
|
+
}
|