doppler-gpu 0.4.10 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +40 -1
- package/models/catalog.json +637 -32
- package/package.json +24 -3
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +135 -13
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +69 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +73 -11
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/adapter.schema.d.ts +6 -1
- package/src/config/schema/adapter.schema.js +5 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/artifact-contract.d.ts +83 -0
- package/src/experimental/adapters/artifact-contract.js +254 -0
- package/src/experimental/adapters/index.d.ts +25 -0
- package/src/experimental/adapters/index.js +12 -0
- package/src/experimental/adapters/lora-loader.js +24 -4
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/experimental/training/artifacts.d.ts +160 -0
- package/src/experimental/training/attention-backward.d.ts +64 -0
- package/src/experimental/training/attention-forward.d.ts +22 -0
- package/src/experimental/training/attention-forward.js +82 -0
- package/src/experimental/training/autograd.d.ts +76 -0
- package/src/experimental/training/checkpoint.d.ts +36 -0
- package/src/experimental/training/clip.d.ts +9 -0
- package/src/experimental/training/dataloader.d.ts +8 -0
- package/src/experimental/training/distillation/student-fixture.js +1 -0
- package/src/experimental/training/export.d.ts +41 -0
- package/src/experimental/training/lora-pipeline.d.ts +186 -0
- package/src/experimental/training/lora-pipeline.js +5 -0
- package/src/experimental/training/lora.d.ts +20 -0
- package/src/experimental/training/lora.js +4 -3
- package/src/experimental/training/loss-scaling.d.ts +21 -0
- package/src/experimental/training/loss.d.ts +10 -0
- package/src/experimental/training/native-qwen-lora.d.ts +106 -0
- package/src/experimental/training/native-qwen-lora.js +449 -0
- package/src/experimental/training/objectives/base.d.ts +58 -0
- package/src/experimental/training/optimizer.d.ts +22 -0
- package/src/experimental/training/optimizer.js +1 -0
- package/src/experimental/training/runner.d.ts +249 -0
- package/src/experimental/training/trainer.d.ts +89 -0
- package/src/experimental/training/workloads.d.ts +205 -0
- package/src/gpu/kernels/backward/adam.d.ts +1 -0
- package/src/gpu/kernels/backward/adam.js +6 -2
- package/src/gpu/kernels/backward/adam.wgsl +3 -2
- package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
- package/src/gpu/kernels/backward/utils.js +18 -4
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +205 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.d.ts +5 -1
- package/src/inference/pipelines/text/generator.js +196 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
- package/src/inference/pipelines/text/linear-attention.js +77 -6
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/lora-types.d.ts +6 -1
- package/src/inference/pipelines/text/lora-types.js +5 -0
- package/src/inference/pipelines/text/ops.js +33 -0
- package/src/inference/pipelines/text/types.d.ts +66 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +9 -2
- package/src/inference/pipelines/text.js +116 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/models/diffusiongemma.js +1 -1
- package/src/models/embeddinggemma.js +1 -1
- package/src/models/gemma3.js +1 -1
- package/src/models/gemma4.js +1 -1
- package/src/models/qwen3.js +1 -1
- package/src/tooling/hf-registry-utils.js +1 -1
- package/src/tooling/node-browser-command-runner.d.ts +16 -0
- package/src/tooling/node-browser-command-runner.js +1 -1
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/training-browser.d.ts +19 -0
- package/src/training-browser.js +13 -0
- package/src/training.d.ts +137 -0
- package/src/training.js +219 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +9 -1
- package/tests/kernels/browser/test-page.js +4 -2
- package/tests/kernels/reference/adam.js +4 -2
- package/NOTICE +0 -5
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
function assertPlainObject(value, label) {
|
|
2
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
3
|
+
throw new Error(`${label} must be an object.`);
|
|
4
|
+
}
|
|
5
|
+
return value;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
function requireToken(value, label) {
|
|
9
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
10
|
+
throw new Error(`${label} must be a non-empty string.`);
|
|
11
|
+
}
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function validateRelativeFile(value) {
|
|
16
|
+
const file = requireToken(value, 'sourceTokenizer.vocabFile');
|
|
17
|
+
if (file.startsWith('/') || file.includes('\\') || file.split('/').includes('..')) {
|
|
18
|
+
throw new Error('sourceTokenizer.vocabFile must stay inside the source model directory.');
|
|
19
|
+
}
|
|
20
|
+
return file;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function parseVocab(text) {
|
|
24
|
+
if (typeof text !== 'string') {
|
|
25
|
+
throw new Error('Source vocabulary must be text.');
|
|
26
|
+
}
|
|
27
|
+
const tokens = text.replace(/\r/gu, '').split('\n');
|
|
28
|
+
if (tokens.at(-1) === '') tokens.pop();
|
|
29
|
+
if (tokens.length === 0 || tokens.some((token) => token.length === 0)) {
|
|
30
|
+
throw new Error('Source vocabulary must contain one non-empty token per line.');
|
|
31
|
+
}
|
|
32
|
+
const vocab = {};
|
|
33
|
+
for (let id = 0; id < tokens.length; id += 1) {
|
|
34
|
+
const token = tokens[id];
|
|
35
|
+
if (Object.prototype.hasOwnProperty.call(vocab, token)) {
|
|
36
|
+
throw new Error(`Source vocabulary contains duplicate token "${token}".`);
|
|
37
|
+
}
|
|
38
|
+
vocab[token] = id;
|
|
39
|
+
}
|
|
40
|
+
return { tokens, vocab };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function validateSourceTokenizerPolicy(policy) {
|
|
44
|
+
if (policy == null) return null;
|
|
45
|
+
const value = assertPlainObject(policy, 'sourceTokenizer');
|
|
46
|
+
if (value.kind !== 'character_vocab' && value.kind !== 'greedy_vocab') {
|
|
47
|
+
throw new Error(`Unsupported sourceTokenizer.kind "${String(value.kind)}".`);
|
|
48
|
+
}
|
|
49
|
+
const specialTokens = assertPlainObject(value.specialTokens, 'sourceTokenizer.specialTokens');
|
|
50
|
+
const normalized = {
|
|
51
|
+
kind: value.kind,
|
|
52
|
+
vocabFile: validateRelativeFile(value.vocabFile),
|
|
53
|
+
specialTokens: {
|
|
54
|
+
pad: requireToken(specialTokens.pad, 'sourceTokenizer.specialTokens.pad'),
|
|
55
|
+
bos: requireToken(specialTokens.bos, 'sourceTokenizer.specialTokens.bos'),
|
|
56
|
+
eos: value.kind === 'character_vocab'
|
|
57
|
+
? requireToken(specialTokens.eos, 'sourceTokenizer.specialTokens.eos')
|
|
58
|
+
: specialTokens.eos == null
|
|
59
|
+
? null
|
|
60
|
+
: requireToken(specialTokens.eos, 'sourceTokenizer.specialTokens.eos'),
|
|
61
|
+
unk: requireToken(specialTokens.unk, 'sourceTokenizer.specialTokens.unk'),
|
|
62
|
+
mask: specialTokens.mask == null
|
|
63
|
+
? null
|
|
64
|
+
: requireToken(specialTokens.mask, 'sourceTokenizer.specialTokens.mask'),
|
|
65
|
+
},
|
|
66
|
+
addBosToken: value.addBosToken === true,
|
|
67
|
+
addEosToken: value.addEosToken === true,
|
|
68
|
+
};
|
|
69
|
+
if (typeof value.addBosToken !== 'boolean' || typeof value.addEosToken !== 'boolean') {
|
|
70
|
+
throw new Error('sourceTokenizer.addBosToken and addEosToken must be booleans.');
|
|
71
|
+
}
|
|
72
|
+
return normalized;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function buildSourceTokenizerJson(policy, vocabText) {
|
|
76
|
+
const normalized = validateSourceTokenizerPolicy(policy);
|
|
77
|
+
if (!normalized) throw new Error('sourceTokenizer policy is required.');
|
|
78
|
+
const { tokens, vocab } = parseVocab(vocabText);
|
|
79
|
+
for (const [role, token] of Object.entries(normalized.specialTokens)) {
|
|
80
|
+
if (token != null && !Object.prototype.hasOwnProperty.call(vocab, token)) {
|
|
81
|
+
throw new Error(`Source vocabulary is missing ${role} token "${token}".`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const uniqueSpecialTokens = [...new Set(Object.values(normalized.specialTokens).filter(Boolean))];
|
|
85
|
+
const characterVocab = normalized.kind === 'character_vocab';
|
|
86
|
+
return {
|
|
87
|
+
version: '1.0',
|
|
88
|
+
truncation: null,
|
|
89
|
+
padding: null,
|
|
90
|
+
added_tokens: uniqueSpecialTokens.map((token) => ({
|
|
91
|
+
id: vocab[token],
|
|
92
|
+
content: token,
|
|
93
|
+
single_word: false,
|
|
94
|
+
lstrip: false,
|
|
95
|
+
rstrip: false,
|
|
96
|
+
normalized: false,
|
|
97
|
+
special: true,
|
|
98
|
+
})),
|
|
99
|
+
normalizer: null,
|
|
100
|
+
pre_tokenizer: characterVocab
|
|
101
|
+
? {
|
|
102
|
+
type: 'Split',
|
|
103
|
+
pattern: { String: '' },
|
|
104
|
+
behavior: 'Removed',
|
|
105
|
+
invert: false,
|
|
106
|
+
}
|
|
107
|
+
: null,
|
|
108
|
+
post_processor: null,
|
|
109
|
+
decoder: characterVocab
|
|
110
|
+
? {
|
|
111
|
+
type: 'WordPiece',
|
|
112
|
+
prefix: '##',
|
|
113
|
+
cleanup: false,
|
|
114
|
+
}
|
|
115
|
+
: null,
|
|
116
|
+
model: characterVocab
|
|
117
|
+
? {
|
|
118
|
+
type: 'WordPiece',
|
|
119
|
+
dropout: null,
|
|
120
|
+
unk_token: normalized.specialTokens.unk,
|
|
121
|
+
continuing_subword_prefix: '##',
|
|
122
|
+
max_input_chars_per_word: 100,
|
|
123
|
+
vocab,
|
|
124
|
+
}
|
|
125
|
+
: {
|
|
126
|
+
type: 'BPE',
|
|
127
|
+
dropout: null,
|
|
128
|
+
unk_token: normalized.specialTokens.unk,
|
|
129
|
+
continuing_subword_prefix: null,
|
|
130
|
+
end_of_word_suffix: null,
|
|
131
|
+
fuse_unk: false,
|
|
132
|
+
byte_fallback: false,
|
|
133
|
+
vocab,
|
|
134
|
+
merges: [],
|
|
135
|
+
},
|
|
136
|
+
special_tokens_map: {
|
|
137
|
+
pad_token: normalized.specialTokens.pad,
|
|
138
|
+
bos_token: normalized.specialTokens.bos,
|
|
139
|
+
...(normalized.specialTokens.eos ? { eos_token: normalized.specialTokens.eos } : {}),
|
|
140
|
+
unk_token: normalized.specialTokens.unk,
|
|
141
|
+
...(normalized.specialTokens.mask ? { mask_token: normalized.specialTokens.mask } : {}),
|
|
142
|
+
},
|
|
143
|
+
add_bos_token: normalized.addBosToken,
|
|
144
|
+
add_eos_token: normalized.addEosToken,
|
|
145
|
+
source_vocab_size: tokens.length,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function buildCharacterTokenizerJson(policy, vocabText) {
|
|
150
|
+
const normalized = validateSourceTokenizerPolicy(policy);
|
|
151
|
+
if (normalized?.kind !== 'character_vocab') {
|
|
152
|
+
throw new Error('buildCharacterTokenizerJson requires sourceTokenizer.kind="character_vocab".');
|
|
153
|
+
}
|
|
154
|
+
return buildSourceTokenizerJson(normalized, vocabText);
|
|
155
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import type { AdapterManifest } from './adapter-manifest.js';
|
|
2
|
+
|
|
3
|
+
export type AdapterArtifactLifecycle = 'preserved' | 'candidate' | 'qualified' | 'promoted' | 'revoked';
|
|
4
|
+
export type AdapterArtifactAccess = 'public' | 'gated' | 'private';
|
|
5
|
+
|
|
6
|
+
export interface HuggingFaceArtifactOrigin {
|
|
7
|
+
provider: 'huggingface';
|
|
8
|
+
repoId: string;
|
|
9
|
+
revision: string;
|
|
10
|
+
path: string;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface GcsArtifactOrigin {
|
|
14
|
+
provider: 'gcs';
|
|
15
|
+
bucket: string;
|
|
16
|
+
object: string;
|
|
17
|
+
generation: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface HttpsPreservationOrigin {
|
|
21
|
+
provider: 'https-preservation';
|
|
22
|
+
url: string;
|
|
23
|
+
revision: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export type AdapterArtifactOrigin = HuggingFaceArtifactOrigin | GcsArtifactOrigin;
|
|
27
|
+
export type AdapterArtifactMirror = AdapterArtifactOrigin | HttpsPreservationOrigin;
|
|
28
|
+
|
|
29
|
+
export interface AdapterArtifactRecord {
|
|
30
|
+
schema: 'doppler.adapter-artifact/v1';
|
|
31
|
+
artifactId: string;
|
|
32
|
+
lifecycle: AdapterArtifactLifecycle;
|
|
33
|
+
access: AdapterArtifactAccess;
|
|
34
|
+
weights: {
|
|
35
|
+
sha256: string;
|
|
36
|
+
bytes: number;
|
|
37
|
+
format: 'safetensors';
|
|
38
|
+
};
|
|
39
|
+
adapterManifest: AdapterManifest;
|
|
40
|
+
trainingBase: {
|
|
41
|
+
repoId: string;
|
|
42
|
+
revision: string;
|
|
43
|
+
};
|
|
44
|
+
runtimeBase: {
|
|
45
|
+
modelId: string;
|
|
46
|
+
modelSha256: string;
|
|
47
|
+
manifestSha256: string;
|
|
48
|
+
tokenizerSha256: string;
|
|
49
|
+
weightPackId: string;
|
|
50
|
+
weightPackSha256: string;
|
|
51
|
+
manifestVariantId: string;
|
|
52
|
+
conversionConfigSha256: string;
|
|
53
|
+
};
|
|
54
|
+
primaryOrigin: AdapterArtifactOrigin | null;
|
|
55
|
+
preservationMirrors: AdapterArtifactMirror[];
|
|
56
|
+
evidence: Array<{ kind: string; path: string; sha256: string }>;
|
|
57
|
+
claimBoundary?: string;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface AdapterArtifactValidationError {
|
|
61
|
+
field: string;
|
|
62
|
+
message: string;
|
|
63
|
+
value?: unknown;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface AdapterArtifactValidationResult {
|
|
67
|
+
valid: boolean;
|
|
68
|
+
errors: AdapterArtifactValidationError[];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export declare const ADAPTER_ARTIFACT_SCHEMA: 'doppler.adapter-artifact/v1';
|
|
72
|
+
export declare const ADAPTER_ARTIFACT_LIFECYCLES: readonly AdapterArtifactLifecycle[];
|
|
73
|
+
export declare const ADAPTER_ARTIFACT_ACCESS: readonly AdapterArtifactAccess[];
|
|
74
|
+
export declare const ADAPTER_ARTIFACT_ORIGIN_PROVIDERS: readonly ['huggingface', 'gcs', 'https-preservation'];
|
|
75
|
+
|
|
76
|
+
export declare function validateAdapterArtifactOrigin(
|
|
77
|
+
origin: unknown,
|
|
78
|
+
options?: { allowPreservation?: boolean }
|
|
79
|
+
): AdapterArtifactValidationResult;
|
|
80
|
+
export declare function validateAdapterArtifactRecord(record: unknown): AdapterArtifactValidationResult;
|
|
81
|
+
export declare function assertAdapterArtifactRecord<T extends AdapterArtifactRecord>(record: T): T;
|
|
82
|
+
export declare function buildImmutableArtifactUrl(origin: AdapterArtifactOrigin): string;
|
|
83
|
+
export declare function adapterArtifactCacheKey(record: AdapterArtifactRecord): string;
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
export const ADAPTER_ARTIFACT_SCHEMA = 'doppler.adapter-artifact/v1';
|
|
2
|
+
export const ADAPTER_ARTIFACT_LIFECYCLES = Object.freeze([
|
|
3
|
+
'preserved',
|
|
4
|
+
'candidate',
|
|
5
|
+
'qualified',
|
|
6
|
+
'promoted',
|
|
7
|
+
'revoked',
|
|
8
|
+
]);
|
|
9
|
+
export const ADAPTER_ARTIFACT_ACCESS = Object.freeze(['public', 'gated', 'private']);
|
|
10
|
+
export const ADAPTER_ARTIFACT_ORIGIN_PROVIDERS = Object.freeze([
|
|
11
|
+
'huggingface',
|
|
12
|
+
'gcs',
|
|
13
|
+
'https-preservation',
|
|
14
|
+
]);
|
|
15
|
+
|
|
16
|
+
const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/;
|
|
17
|
+
const HF_REVISION_PATTERN = /^[a-f0-9]{40,64}$/;
|
|
18
|
+
const GCS_GENERATION_PATTERN = /^[1-9][0-9]*$/;
|
|
19
|
+
const GCS_BUCKET_PATTERN = /^[a-z0-9][a-z0-9._-]{1,220}[a-z0-9]$/;
|
|
20
|
+
const HF_REPO_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*\/[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
21
|
+
const FORBIDDEN_ARTIFACT_PATH_PATTERN = /[\\?#\u0000-\u001f\u007f]/;
|
|
22
|
+
|
|
23
|
+
const asText = (value) => String(value || '').trim();
|
|
24
|
+
const hasTraversal = (value) => asText(value).split('/').includes('..');
|
|
25
|
+
|
|
26
|
+
const requireText = (errors, value, field) => {
|
|
27
|
+
if (!asText(value)) errors.push({ field, message: `${field} is required`, value });
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
const requireSha256 = (errors, value, field) => {
|
|
31
|
+
if (!SHA256_PATTERN.test(asText(value))) {
|
|
32
|
+
errors.push({ field, message: `${field} must be a sha256: content identity`, value });
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
const validateArtifactPath = (errors, value, field) => {
|
|
37
|
+
const path = asText(value);
|
|
38
|
+
requireText(errors, path, field);
|
|
39
|
+
if (path.startsWith('/') || hasTraversal(path)) {
|
|
40
|
+
errors.push({ field, message: `${field} must be an artifact-relative path`, value });
|
|
41
|
+
}
|
|
42
|
+
if (FORBIDDEN_ARTIFACT_PATH_PATTERN.test(path)) {
|
|
43
|
+
errors.push({ field, message: `${field} contains a forbidden URL delimiter`, value });
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const encodeArtifactPath = (value) => asText(value).split('/').map(encodeURIComponent).join('/');
|
|
48
|
+
|
|
49
|
+
export function validateAdapterArtifactOrigin(origin, { allowPreservation = false } = {}) {
|
|
50
|
+
const errors = [];
|
|
51
|
+
if (!origin || typeof origin !== 'object' || Array.isArray(origin)) {
|
|
52
|
+
return {
|
|
53
|
+
valid: false,
|
|
54
|
+
errors: [{ field: 'origin', message: 'origin must be an object', value: origin }],
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (!ADAPTER_ARTIFACT_ORIGIN_PROVIDERS.includes(origin.provider)) {
|
|
59
|
+
errors.push({ field: 'origin.provider', message: 'origin.provider is unsupported', value: origin.provider });
|
|
60
|
+
} else if (origin.provider === 'huggingface') {
|
|
61
|
+
if (!HF_REPO_PATTERN.test(asText(origin.repoId))) {
|
|
62
|
+
errors.push({ field: 'origin.repoId', message: 'origin.repoId must be an owner/repository identifier', value: origin.repoId });
|
|
63
|
+
}
|
|
64
|
+
if (!HF_REVISION_PATTERN.test(asText(origin.revision))) {
|
|
65
|
+
errors.push({
|
|
66
|
+
field: 'origin.revision',
|
|
67
|
+
message: 'origin.revision must be a full immutable Hugging Face commit hash',
|
|
68
|
+
value: origin.revision,
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
validateArtifactPath(errors, origin.path, 'origin.path');
|
|
72
|
+
} else if (origin.provider === 'gcs') {
|
|
73
|
+
if (!GCS_BUCKET_PATTERN.test(asText(origin.bucket))) {
|
|
74
|
+
errors.push({ field: 'origin.bucket', message: 'origin.bucket is invalid', value: origin.bucket });
|
|
75
|
+
}
|
|
76
|
+
validateArtifactPath(errors, origin.object, 'origin.object');
|
|
77
|
+
if (!GCS_GENERATION_PATTERN.test(asText(origin.generation))) {
|
|
78
|
+
errors.push({
|
|
79
|
+
field: 'origin.generation',
|
|
80
|
+
message: 'origin.generation must pin an immutable GCS object generation',
|
|
81
|
+
value: origin.generation,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
} else if (origin.provider === 'https-preservation') {
|
|
85
|
+
if (!allowPreservation) {
|
|
86
|
+
errors.push({
|
|
87
|
+
field: 'origin.provider',
|
|
88
|
+
message: 'https-preservation is allowed only as a preservation mirror',
|
|
89
|
+
value: origin.provider,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
let url = null;
|
|
93
|
+
try {
|
|
94
|
+
url = new URL(asText(origin.url));
|
|
95
|
+
} catch {
|
|
96
|
+
errors.push({ field: 'origin.url', message: 'origin.url must be an absolute HTTPS URL', value: origin.url });
|
|
97
|
+
}
|
|
98
|
+
if (url && (url.protocol !== 'https:' || url.username || url.password || url.search)) {
|
|
99
|
+
errors.push({
|
|
100
|
+
field: 'origin.url',
|
|
101
|
+
message: 'origin.url must be HTTPS, immutable, credential-free, and query-free',
|
|
102
|
+
value: origin.url,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
if (!HF_REVISION_PATTERN.test(asText(origin.revision))) {
|
|
106
|
+
errors.push({
|
|
107
|
+
field: 'origin.revision',
|
|
108
|
+
message: 'preservation origins must declare their full immutable revision',
|
|
109
|
+
value: origin.revision,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return { valid: errors.length === 0, errors };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function validateAdapterArtifactRecord(record) {
|
|
118
|
+
const errors = [];
|
|
119
|
+
if (!record || typeof record !== 'object' || Array.isArray(record)) {
|
|
120
|
+
return {
|
|
121
|
+
valid: false,
|
|
122
|
+
errors: [{ field: 'record', message: 'adapter artifact record must be an object', value: record }],
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (record.schema !== ADAPTER_ARTIFACT_SCHEMA) {
|
|
127
|
+
errors.push({ field: 'schema', message: `schema must be ${ADAPTER_ARTIFACT_SCHEMA}`, value: record.schema });
|
|
128
|
+
}
|
|
129
|
+
requireText(errors, record.artifactId, 'artifactId');
|
|
130
|
+
if (!ADAPTER_ARTIFACT_LIFECYCLES.includes(record.lifecycle)) {
|
|
131
|
+
errors.push({ field: 'lifecycle', message: 'lifecycle is unsupported', value: record.lifecycle });
|
|
132
|
+
}
|
|
133
|
+
if (!ADAPTER_ARTIFACT_ACCESS.includes(record.access)) {
|
|
134
|
+
errors.push({ field: 'access', message: 'access is unsupported', value: record.access });
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const weights = record.weights || {};
|
|
138
|
+
requireSha256(errors, weights.sha256, 'weights.sha256');
|
|
139
|
+
if (!Number.isInteger(weights.bytes) || weights.bytes <= 0) {
|
|
140
|
+
errors.push({ field: 'weights.bytes', message: 'weights.bytes must be a positive integer', value: weights.bytes });
|
|
141
|
+
}
|
|
142
|
+
if (weights.format !== 'safetensors') {
|
|
143
|
+
errors.push({ field: 'weights.format', message: 'weights.format must be safetensors', value: weights.format });
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const manifest = record.adapterManifest || {};
|
|
147
|
+
requireText(errors, manifest.id, 'adapterManifest.id');
|
|
148
|
+
requireText(errors, manifest.baseModel, 'adapterManifest.baseModel');
|
|
149
|
+
if (!Number.isInteger(manifest.rank) || manifest.rank <= 0) {
|
|
150
|
+
errors.push({ field: 'adapterManifest.rank', message: 'adapterManifest.rank must be a positive integer', value: manifest.rank });
|
|
151
|
+
}
|
|
152
|
+
if (!Number.isFinite(manifest.alpha) || manifest.alpha <= 0) {
|
|
153
|
+
errors.push({ field: 'adapterManifest.alpha', message: 'adapterManifest.alpha must be positive', value: manifest.alpha });
|
|
154
|
+
}
|
|
155
|
+
if (!Array.isArray(manifest.targetModules) || manifest.targetModules.length === 0) {
|
|
156
|
+
errors.push({ field: 'adapterManifest.targetModules', message: 'adapterManifest.targetModules is required', value: manifest.targetModules });
|
|
157
|
+
}
|
|
158
|
+
if (asText(manifest.checksumAlgorithm) !== 'sha256') {
|
|
159
|
+
errors.push({ field: 'adapterManifest.checksumAlgorithm', message: 'adapterManifest.checksumAlgorithm must be sha256', value: manifest.checksumAlgorithm });
|
|
160
|
+
}
|
|
161
|
+
const manifestChecksum = asText(manifest.checksum).replace(/^sha256:/, '');
|
|
162
|
+
if (`sha256:${manifestChecksum}` !== asText(weights.sha256)) {
|
|
163
|
+
errors.push({ field: 'adapterManifest.checksum', message: 'adapter manifest checksum must match weights.sha256', value: manifest.checksum });
|
|
164
|
+
}
|
|
165
|
+
if (Number(manifest.weightsSize) !== weights.bytes) {
|
|
166
|
+
errors.push({ field: 'adapterManifest.weightsSize', message: 'adapter manifest weightsSize must match weights.bytes', value: manifest.weightsSize });
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const trainingBase = record.trainingBase || {};
|
|
170
|
+
requireText(errors, trainingBase.repoId, 'trainingBase.repoId');
|
|
171
|
+
if (!HF_REVISION_PATTERN.test(asText(trainingBase.revision))) {
|
|
172
|
+
errors.push({ field: 'trainingBase.revision', message: 'trainingBase.revision must be a full immutable source revision', value: trainingBase.revision });
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const runtimeBase = record.runtimeBase || {};
|
|
176
|
+
requireText(errors, runtimeBase.modelId, 'runtimeBase.modelId');
|
|
177
|
+
requireText(errors, runtimeBase.weightPackId, 'runtimeBase.weightPackId');
|
|
178
|
+
requireText(errors, runtimeBase.manifestVariantId, 'runtimeBase.manifestVariantId');
|
|
179
|
+
for (const field of [
|
|
180
|
+
'modelSha256',
|
|
181
|
+
'manifestSha256',
|
|
182
|
+
'tokenizerSha256',
|
|
183
|
+
'weightPackSha256',
|
|
184
|
+
'conversionConfigSha256',
|
|
185
|
+
]) {
|
|
186
|
+
requireSha256(errors, runtimeBase[field], `runtimeBase.${field}`);
|
|
187
|
+
}
|
|
188
|
+
if (manifest.baseModel && runtimeBase.modelId && manifest.baseModel !== runtimeBase.modelId) {
|
|
189
|
+
errors.push({
|
|
190
|
+
field: 'runtimeBase.modelId',
|
|
191
|
+
message: 'runtimeBase.modelId must match adapterManifest.baseModel',
|
|
192
|
+
value: runtimeBase.modelId,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (record.primaryOrigin == null) {
|
|
197
|
+
if (record.lifecycle === 'qualified' || record.lifecycle === 'promoted') {
|
|
198
|
+
errors.push({ field: 'primaryOrigin', message: `${record.lifecycle} artifacts require a primary Hugging Face or GCS origin`, value: record.primaryOrigin });
|
|
199
|
+
}
|
|
200
|
+
} else {
|
|
201
|
+
const originValidation = validateAdapterArtifactOrigin(record.primaryOrigin);
|
|
202
|
+
errors.push(...originValidation.errors.map((error) => ({ ...error, field: `primaryOrigin.${error.field.replace(/^origin\.?/, '')}` })));
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
if (!Array.isArray(record.preservationMirrors)) {
|
|
206
|
+
errors.push({ field: 'preservationMirrors', message: 'preservationMirrors must be an array', value: record.preservationMirrors });
|
|
207
|
+
} else {
|
|
208
|
+
record.preservationMirrors.forEach((origin, index) => {
|
|
209
|
+
const validation = validateAdapterArtifactOrigin(origin, { allowPreservation: true });
|
|
210
|
+
errors.push(...validation.errors.map((error) => ({
|
|
211
|
+
...error,
|
|
212
|
+
field: `preservationMirrors[${index}].${error.field.replace(/^origin\.?/, '')}`,
|
|
213
|
+
})));
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if (!Array.isArray(record.evidence) || record.evidence.length === 0) {
|
|
218
|
+
errors.push({ field: 'evidence', message: 'evidence must contain at least one receipt identity', value: record.evidence });
|
|
219
|
+
} else {
|
|
220
|
+
record.evidence.forEach((evidence, index) => {
|
|
221
|
+
requireText(errors, evidence.kind, `evidence[${index}].kind`);
|
|
222
|
+
validateArtifactPath(errors, evidence.path, `evidence[${index}].path`);
|
|
223
|
+
requireSha256(errors, evidence.sha256, `evidence[${index}].sha256`);
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return { valid: errors.length === 0, errors };
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export function assertAdapterArtifactRecord(record) {
|
|
231
|
+
const validation = validateAdapterArtifactRecord(record);
|
|
232
|
+
if (!validation.valid) {
|
|
233
|
+
const message = validation.errors.map((error) => `${error.field}: ${error.message}`).join('; ');
|
|
234
|
+
throw new Error(`Adapter artifact contract invalid: ${message}`);
|
|
235
|
+
}
|
|
236
|
+
return record;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
export function buildImmutableArtifactUrl(origin) {
|
|
240
|
+
const validation = validateAdapterArtifactOrigin(origin);
|
|
241
|
+
if (!validation.valid) {
|
|
242
|
+
const message = validation.errors.map((error) => `${error.field}: ${error.message}`).join('; ');
|
|
243
|
+
throw new Error(`Adapter artifact origin invalid: ${message}`);
|
|
244
|
+
}
|
|
245
|
+
if (origin.provider === 'huggingface') {
|
|
246
|
+
return `https://huggingface.co/${origin.repoId}/resolve/${origin.revision}/${encodeArtifactPath(origin.path)}`;
|
|
247
|
+
}
|
|
248
|
+
return `https://storage.googleapis.com/${origin.bucket}/${encodeArtifactPath(origin.object)}?generation=${origin.generation}`;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export function adapterArtifactCacheKey(record) {
|
|
252
|
+
assertAdapterArtifactRecord(record);
|
|
253
|
+
return record.weights.sha256;
|
|
254
|
+
}
|
|
@@ -81,6 +81,31 @@ export {
|
|
|
81
81
|
createMemoryRegistry,
|
|
82
82
|
} from './adapter-registry.js';
|
|
83
83
|
|
|
84
|
+
export {
|
|
85
|
+
ADAPTER_ARTIFACT_SCHEMA,
|
|
86
|
+
ADAPTER_ARTIFACT_LIFECYCLES,
|
|
87
|
+
ADAPTER_ARTIFACT_ACCESS,
|
|
88
|
+
ADAPTER_ARTIFACT_ORIGIN_PROVIDERS,
|
|
89
|
+
validateAdapterArtifactOrigin,
|
|
90
|
+
validateAdapterArtifactRecord,
|
|
91
|
+
assertAdapterArtifactRecord,
|
|
92
|
+
buildImmutableArtifactUrl,
|
|
93
|
+
adapterArtifactCacheKey,
|
|
94
|
+
} from './artifact-contract.js';
|
|
95
|
+
|
|
96
|
+
export type {
|
|
97
|
+
AdapterArtifactLifecycle,
|
|
98
|
+
AdapterArtifactAccess,
|
|
99
|
+
HuggingFaceArtifactOrigin,
|
|
100
|
+
GcsArtifactOrigin,
|
|
101
|
+
HttpsPreservationOrigin,
|
|
102
|
+
AdapterArtifactOrigin,
|
|
103
|
+
AdapterArtifactMirror,
|
|
104
|
+
AdapterArtifactRecord,
|
|
105
|
+
AdapterArtifactValidationError,
|
|
106
|
+
AdapterArtifactValidationResult,
|
|
107
|
+
} from './artifact-contract.js';
|
|
108
|
+
|
|
84
109
|
export type {
|
|
85
110
|
// Types
|
|
86
111
|
AdapterRegistryEntry,
|
|
@@ -40,3 +40,15 @@ export {
|
|
|
40
40
|
resetAdapterRegistry,
|
|
41
41
|
createMemoryRegistry,
|
|
42
42
|
} from './adapter-registry.js';
|
|
43
|
+
|
|
44
|
+
export {
|
|
45
|
+
ADAPTER_ARTIFACT_SCHEMA,
|
|
46
|
+
ADAPTER_ARTIFACT_LIFECYCLES,
|
|
47
|
+
ADAPTER_ARTIFACT_ACCESS,
|
|
48
|
+
ADAPTER_ARTIFACT_ORIGIN_PROVIDERS,
|
|
49
|
+
validateAdapterArtifactOrigin,
|
|
50
|
+
validateAdapterArtifactRecord,
|
|
51
|
+
assertAdapterArtifactRecord,
|
|
52
|
+
buildImmutableArtifactUrl,
|
|
53
|
+
adapterArtifactCacheKey,
|
|
54
|
+
} from './artifact-contract.js';
|
|
@@ -149,8 +149,15 @@ async function loadExternalWeights(manifest, options = {}) {
|
|
|
149
149
|
}
|
|
150
150
|
|
|
151
151
|
function assertCompleteAdapterLayers(adapter) {
|
|
152
|
+
const targetModules = new Set(adapter.targetModules || []);
|
|
153
|
+
const loadedModules = new Set();
|
|
152
154
|
for (const [layerIndex, layer] of adapter.layers.entries()) {
|
|
153
155
|
for (const [moduleName, weights] of Object.entries(layer)) {
|
|
156
|
+
if (!targetModules.has(moduleName)) {
|
|
157
|
+
throw new Error(
|
|
158
|
+
`LoRA adapter layer ${layerIndex} contains module ${moduleName} outside targetModules.`
|
|
159
|
+
);
|
|
160
|
+
}
|
|
154
161
|
const hasA = weights?.a instanceof Float32Array && weights.a.length > 0;
|
|
155
162
|
const hasB = weights?.b instanceof Float32Array && weights.b.length > 0;
|
|
156
163
|
if (!hasA || !hasB) {
|
|
@@ -158,6 +165,14 @@ function assertCompleteAdapterLayers(adapter) {
|
|
|
158
165
|
`LoRA adapter layer ${layerIndex} module ${moduleName} is incomplete; both lora_a and lora_b tensors are required.`
|
|
159
166
|
);
|
|
160
167
|
}
|
|
168
|
+
loadedModules.add(moduleName);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
for (const moduleName of targetModules) {
|
|
172
|
+
if (!loadedModules.has(moduleName)) {
|
|
173
|
+
throw new Error(
|
|
174
|
+
`LoRA adapter targetModules declares ${moduleName}, but no complete tensors were loaded for it.`
|
|
175
|
+
);
|
|
161
176
|
}
|
|
162
177
|
}
|
|
163
178
|
}
|
|
@@ -241,10 +256,15 @@ export async function loadLoRAWeights(path, options = {}) {
|
|
|
241
256
|
return res.arrayBuffer();
|
|
242
257
|
};
|
|
243
258
|
|
|
244
|
-
const
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
259
|
+
const manifestLoadOptions = isUrl
|
|
260
|
+
? {
|
|
261
|
+
...options,
|
|
262
|
+
readFile: undefined,
|
|
263
|
+
readOPFS: undefined,
|
|
264
|
+
fetchUrl: fetchWithBase,
|
|
265
|
+
}
|
|
266
|
+
: { ...options, fetchUrl: fetchWithBase };
|
|
267
|
+
const adapter = await loadLoRAFromManifest(manifest, manifestLoadOptions);
|
|
248
268
|
|
|
249
269
|
let checksumValid;
|
|
250
270
|
if (manifest.checksum && !options.skipVerify && Array.isArray(manifest.tensors) && manifest.tensors.length > 0) {
|
|
@@ -7,6 +7,7 @@ export declare const TRAINER_ARTIFACT_KIND_PEFT_ADAPTER: 'peft_adapter';
|
|
|
7
7
|
|
|
8
8
|
export declare const TRANSLATION_FULL_CHECKPOINT_PARITY_CHECKS: readonly string[];
|
|
9
9
|
export declare const COLUMBO_QWEN_ADAPTER_PARITY_CHECKS: readonly string[];
|
|
10
|
+
export declare const TINKER_PEFT_BROWSER_ADAPTER_PARITY_CHECKS: readonly string[];
|
|
10
11
|
|
|
11
12
|
export interface TrainerArtifactFileIdentity {
|
|
12
13
|
id: string;
|
|
@@ -67,7 +68,7 @@ export interface TrainerArtifactBridgeDescriptor {
|
|
|
67
68
|
receipt: string | null;
|
|
68
69
|
};
|
|
69
70
|
parity: {
|
|
70
|
-
profile: 'translation_full_checkpoint' | 'columbo_qwen_adapter';
|
|
71
|
+
profile: 'translation_full_checkpoint' | 'columbo_qwen_adapter' | 'tinker_peft_browser_adapter';
|
|
71
72
|
requiredChecks: string[];
|
|
72
73
|
};
|
|
73
74
|
claimBoundary: string;
|