doppler-gpu 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +27 -0
- package/models/catalog.json +612 -7
- package/package.json +13 -1
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +126 -4
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +64 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/kernels/registry.json +52 -9
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/lora-loader.js +15 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +189 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.js +90 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/ops.js +32 -0
- package/src/inference/pipelines/text/types.d.ts +38 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +4 -2
- package/src/inference/pipelines/text.js +112 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/version.js +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "doppler-gpu",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.11",
|
|
4
4
|
"description": "Browser-native WebGPU inference engine for local intent and inference loops",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -138,7 +138,16 @@
|
|
|
138
138
|
"product:readiness:report": "node tools/render-product-readiness-report.js",
|
|
139
139
|
"claims:evidence:check": "node tools/check-claim-evidence-contract.js",
|
|
140
140
|
"policy:schemas:check": "node tools/check-policy-schema-registry.js",
|
|
141
|
+
"verify:amplify-120m": "node tools/run-registry-verify.js amplify-120m",
|
|
142
|
+
"verify:amplify-120m-f16-af32": "node tools/run-registry-verify.js amplify-120m-f16-af32",
|
|
143
|
+
"verify:biohub-esmc-300m": "node tools/run-registry-verify.js biohub-esmc-300m",
|
|
144
|
+
"verify:chandar-lab-amplify_120m": "node tools/run-registry-verify.js chandar-lab-amplify_120m",
|
|
141
145
|
"verify:embeddinggemma-300m": "node tools/run-registry-verify.js embeddinggemma-300m",
|
|
146
|
+
"verify:esm2-35m": "node tools/run-registry-verify.js esm2-35m",
|
|
147
|
+
"verify:esm2-t12-35m-ur50d-f32-af32": "node tools/run-registry-verify.js esm2-t12-35m-ur50d-f32-af32",
|
|
148
|
+
"verify:esmc-300m": "node tools/run-registry-verify.js esmc-300m",
|
|
149
|
+
"verify:esmc-300m-f32-af32": "node tools/run-registry-verify.js esmc-300m-f32-af32",
|
|
150
|
+
"verify:facebook-esm2_t12_35m_ur50d": "node tools/run-registry-verify.js facebook-esm2_t12_35m_ur50d",
|
|
142
151
|
"verify:gemma-3-1b-it-q4k-ehf16-af32": "node tools/run-registry-verify.js gemma-3-1b-it-q4k-ehf16-af32",
|
|
143
152
|
"verify:gemma-3-1b-it-wq4k-ef16-hf16": "node tools/run-registry-verify.js gemma-3-1b-it-wq4k-ef16-hf16",
|
|
144
153
|
"verify:gemma-3-270m-it-q4k-ehf16-af32": "node tools/run-registry-verify.js gemma-3-270m-it-q4k-ehf16-af32",
|
|
@@ -163,6 +172,9 @@
|
|
|
163
172
|
"verify:google-gemma-3-270m-it": "node tools/run-registry-verify.js google-gemma-3-270m-it",
|
|
164
173
|
"verify:google-gemma-4-e2b-it": "node tools/run-registry-verify.js google-gemma-4-e2b-it",
|
|
165
174
|
"verify:google-translategemma-4b-it": "node tools/run-registry-verify.js google-translategemma-4b-it",
|
|
175
|
+
"verify:instadeepai-nucleotide-transformer-v2-50m-multi-species": "node tools/run-registry-verify.js instadeepai-nucleotide-transformer-v2-50m-multi-species",
|
|
176
|
+
"verify:nucleotide-transformer-v2-50m": "node tools/run-registry-verify.js nucleotide-transformer-v2-50m",
|
|
177
|
+
"verify:nucleotide-transformer-v2-50m-f32-af32": "node tools/run-registry-verify.js nucleotide-transformer-v2-50m-f32-af32",
|
|
166
178
|
"verify:qwen-3-5-0-8b": "node tools/run-registry-verify.js qwen-3-5-0-8b",
|
|
167
179
|
"verify:qwen-3-5-0-8b-q4k-ehaf16": "node tools/run-registry-verify.js qwen-3-5-0-8b-q4k-ehaf16",
|
|
168
180
|
"verify:qwen-3-5-0-8b-wq4k-ef16-hf16-f16": "node tools/run-registry-verify.js qwen-3-5-0-8b-wq4k-ef16-hf16-f16",
|
|
@@ -25,6 +25,8 @@ function normalizeEntry(entry) {
|
|
|
25
25
|
const runtimePromotionState = typeof entry.runtimePromotionState === 'string' ? entry.runtimePromotionState.trim() : '';
|
|
26
26
|
const vendorBenchmark = cloneOptionalObject(entry.vendorBenchmark);
|
|
27
27
|
const benchmarkEvidence = cloneOptionalObject(entry.benchmarkEvidence);
|
|
28
|
+
const classification = cloneOptionalObject(entry.classification);
|
|
29
|
+
const typeCluster = cloneOptionalObject(entry.typeCluster);
|
|
28
30
|
if (
|
|
29
31
|
!sourceCheckpointId
|
|
30
32
|
|| !weightPackId
|
|
@@ -49,6 +51,8 @@ function normalizeEntry(entry) {
|
|
|
49
51
|
modes: Array.isArray(entry.modes)
|
|
50
52
|
? entry.modes.filter((mode) => typeof mode === 'string' && mode.trim().length > 0)
|
|
51
53
|
: [],
|
|
54
|
+
...(classification ? { classification } : {}),
|
|
55
|
+
...(typeCluster ? { typeCluster } : {}),
|
|
52
56
|
hf: entry.hf && typeof entry.hf === 'object'
|
|
53
57
|
? {
|
|
54
58
|
repoId: typeof entry.hf.repoId === 'string' ? entry.hf.repoId.trim() : '',
|
|
@@ -91,6 +95,8 @@ export async function listQuickstartModels() {
|
|
|
91
95
|
weightsRefAllowed: entry.weightsRefAllowed,
|
|
92
96
|
aliases: [...entry.aliases],
|
|
93
97
|
modes: [...entry.modes],
|
|
98
|
+
...(entry.classification ? { classification: cloneOptionalObject(entry.classification) } : {}),
|
|
99
|
+
...(entry.typeCluster ? { typeCluster: cloneOptionalObject(entry.typeCluster) } : {}),
|
|
94
100
|
...(entry.vendorBenchmark ? { vendorBenchmark: cloneOptionalObject(entry.vendorBenchmark) } : {}),
|
|
95
101
|
...(entry.benchmarkEvidence ? { benchmarkEvidence: cloneOptionalObject(entry.benchmarkEvidence) } : {}),
|
|
96
102
|
}));
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 1,
|
|
3
3
|
"source": "models/catalog.json",
|
|
4
|
+
"taxonomySource": "models/model-type-taxonomy.json",
|
|
4
5
|
"models": [
|
|
5
6
|
{
|
|
6
7
|
"modelId": "gemma-3-270m-it-q4k-ehf16-af32",
|
|
@@ -18,9 +19,25 @@
|
|
|
18
19
|
"gemma-3-270m-it-wq4k-ef16-hf16-f32"
|
|
19
20
|
],
|
|
20
21
|
"modes": [
|
|
21
|
-
"text"
|
|
22
|
-
"vision"
|
|
22
|
+
"text"
|
|
23
23
|
],
|
|
24
|
+
"classification": {
|
|
25
|
+
"domain": "language",
|
|
26
|
+
"tasks": [
|
|
27
|
+
"generation"
|
|
28
|
+
],
|
|
29
|
+
"architectureRole": "autoregressive-decoder",
|
|
30
|
+
"inputs": [
|
|
31
|
+
"text"
|
|
32
|
+
],
|
|
33
|
+
"outputs": [
|
|
34
|
+
"text"
|
|
35
|
+
]
|
|
36
|
+
},
|
|
37
|
+
"typeCluster": {
|
|
38
|
+
"id": "text-generators",
|
|
39
|
+
"label": "Text generators"
|
|
40
|
+
},
|
|
24
41
|
"hf": {
|
|
25
42
|
"repoId": "Clocksmith/rdrr",
|
|
26
43
|
"revision": "7e194b4a6824a8b71cbd0eaa511d9f2c7e1c0129",
|
|
@@ -56,6 +73,23 @@
|
|
|
56
73
|
"modes": [
|
|
57
74
|
"embedding"
|
|
58
75
|
],
|
|
76
|
+
"classification": {
|
|
77
|
+
"domain": "language",
|
|
78
|
+
"tasks": [
|
|
79
|
+
"embedding"
|
|
80
|
+
],
|
|
81
|
+
"architectureRole": "decoder-embedding-encoder",
|
|
82
|
+
"inputs": [
|
|
83
|
+
"text"
|
|
84
|
+
],
|
|
85
|
+
"outputs": [
|
|
86
|
+
"pooled-embedding"
|
|
87
|
+
]
|
|
88
|
+
},
|
|
89
|
+
"typeCluster": {
|
|
90
|
+
"id": "language-embedders",
|
|
91
|
+
"label": "Language embedders"
|
|
92
|
+
},
|
|
59
93
|
"hf": {
|
|
60
94
|
"repoId": "Clocksmith/rdrr",
|
|
61
95
|
"revision": "95f0b29ec73dea70394c6bcfa8407bc6796df6c9",
|
|
@@ -82,9 +116,25 @@
|
|
|
82
116
|
"gemma-3-1b-it-wq4k-ef16-hf16"
|
|
83
117
|
],
|
|
84
118
|
"modes": [
|
|
85
|
-
"text"
|
|
86
|
-
"vision"
|
|
119
|
+
"text"
|
|
87
120
|
],
|
|
121
|
+
"classification": {
|
|
122
|
+
"domain": "language",
|
|
123
|
+
"tasks": [
|
|
124
|
+
"generation"
|
|
125
|
+
],
|
|
126
|
+
"architectureRole": "autoregressive-decoder",
|
|
127
|
+
"inputs": [
|
|
128
|
+
"text"
|
|
129
|
+
],
|
|
130
|
+
"outputs": [
|
|
131
|
+
"text"
|
|
132
|
+
]
|
|
133
|
+
},
|
|
134
|
+
"typeCluster": {
|
|
135
|
+
"id": "text-generators",
|
|
136
|
+
"label": "Text generators"
|
|
137
|
+
},
|
|
88
138
|
"hf": {
|
|
89
139
|
"repoId": "Clocksmith/rdrr",
|
|
90
140
|
"revision": "7c3d30e300bcb02cbd68fb0db3eee64fbf738f99",
|
|
@@ -113,6 +163,25 @@
|
|
|
113
163
|
"text",
|
|
114
164
|
"vision"
|
|
115
165
|
],
|
|
166
|
+
"classification": {
|
|
167
|
+
"domain": "language",
|
|
168
|
+
"tasks": [
|
|
169
|
+
"generation"
|
|
170
|
+
],
|
|
171
|
+
"architectureRole": "autoregressive-decoder",
|
|
172
|
+
"inputs": [
|
|
173
|
+
"text",
|
|
174
|
+
"image",
|
|
175
|
+
"audio"
|
|
176
|
+
],
|
|
177
|
+
"outputs": [
|
|
178
|
+
"text"
|
|
179
|
+
]
|
|
180
|
+
},
|
|
181
|
+
"typeCluster": {
|
|
182
|
+
"id": "multimodal-generators",
|
|
183
|
+
"label": "Multimodal generators"
|
|
184
|
+
},
|
|
116
185
|
"hf": {
|
|
117
186
|
"repoId": "Clocksmith/rdrr",
|
|
118
187
|
"revision": "2070777c77047d54e6eae105f6dcb1891cf6f21a",
|
|
@@ -141,6 +210,25 @@
|
|
|
141
210
|
"text",
|
|
142
211
|
"vision"
|
|
143
212
|
],
|
|
213
|
+
"classification": {
|
|
214
|
+
"domain": "language",
|
|
215
|
+
"tasks": [
|
|
216
|
+
"generation"
|
|
217
|
+
],
|
|
218
|
+
"architectureRole": "autoregressive-decoder",
|
|
219
|
+
"inputs": [
|
|
220
|
+
"text",
|
|
221
|
+
"image",
|
|
222
|
+
"audio"
|
|
223
|
+
],
|
|
224
|
+
"outputs": [
|
|
225
|
+
"text"
|
|
226
|
+
]
|
|
227
|
+
},
|
|
228
|
+
"typeCluster": {
|
|
229
|
+
"id": "multimodal-generators",
|
|
230
|
+
"label": "Multimodal generators"
|
|
231
|
+
},
|
|
144
232
|
"hf": {
|
|
145
233
|
"repoId": "Clocksmith/rdrr",
|
|
146
234
|
"revision": "16256bf16dc8f92b8fe5105b07628cef91587f0a",
|
|
@@ -170,6 +258,23 @@
|
|
|
170
258
|
"modes": [
|
|
171
259
|
"text"
|
|
172
260
|
],
|
|
261
|
+
"classification": {
|
|
262
|
+
"domain": "language",
|
|
263
|
+
"tasks": [
|
|
264
|
+
"generation"
|
|
265
|
+
],
|
|
266
|
+
"architectureRole": "autoregressive-decoder",
|
|
267
|
+
"inputs": [
|
|
268
|
+
"text"
|
|
269
|
+
],
|
|
270
|
+
"outputs": [
|
|
271
|
+
"text"
|
|
272
|
+
]
|
|
273
|
+
},
|
|
274
|
+
"typeCluster": {
|
|
275
|
+
"id": "text-generators",
|
|
276
|
+
"label": "Text generators"
|
|
277
|
+
},
|
|
173
278
|
"hf": {
|
|
174
279
|
"repoId": "Clocksmith/rdrr",
|
|
175
280
|
"revision": "f58f1d0b58641c84e7ea50d13fea0dd4dc91389a",
|
|
@@ -199,6 +304,23 @@
|
|
|
199
304
|
"modes": [
|
|
200
305
|
"text"
|
|
201
306
|
],
|
|
307
|
+
"classification": {
|
|
308
|
+
"domain": "language",
|
|
309
|
+
"tasks": [
|
|
310
|
+
"generation"
|
|
311
|
+
],
|
|
312
|
+
"architectureRole": "autoregressive-decoder",
|
|
313
|
+
"inputs": [
|
|
314
|
+
"text"
|
|
315
|
+
],
|
|
316
|
+
"outputs": [
|
|
317
|
+
"text"
|
|
318
|
+
]
|
|
319
|
+
},
|
|
320
|
+
"typeCluster": {
|
|
321
|
+
"id": "text-generators",
|
|
322
|
+
"label": "Text generators"
|
|
323
|
+
},
|
|
202
324
|
"hf": {
|
|
203
325
|
"repoId": "Clocksmith/rdrr",
|
|
204
326
|
"revision": "977d145bf2478a7fb542e6aca65030585620ca60",
|
|
@@ -11,6 +11,16 @@ export interface LoRALoadOptions {
|
|
|
11
11
|
fetchUrl?: (url: string) => Promise<ArrayBuffer | Uint8Array>;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
+
export declare function assertLoRABaseModelForPipeline(
|
|
15
|
+
pipeline: InferencePipeline | null | undefined,
|
|
16
|
+
adapter: { baseModel?: string } | null | undefined
|
|
17
|
+
): void;
|
|
18
|
+
|
|
19
|
+
export declare function assertLoRATargetsForPipeline(
|
|
20
|
+
pipeline: InferencePipeline | null | undefined,
|
|
21
|
+
adapter: { layers?: Map<number, Record<string, unknown>> } | null | undefined
|
|
22
|
+
): void;
|
|
23
|
+
|
|
14
24
|
export declare function loadLoRAAdapterForPipeline(
|
|
15
25
|
pipeline: InferencePipeline | null | undefined,
|
|
16
26
|
adapter: LoRAManifest | RDRRManifest | string,
|
|
@@ -4,6 +4,17 @@ import { fetchArrayBuffer, readOPFSFile, writeOPFSFile } from './storage.js';
|
|
|
4
4
|
|
|
5
5
|
let loraModulePromise = null;
|
|
6
6
|
|
|
7
|
+
const LORA_TARGET_WEIGHT_KEYS = Object.freeze({
|
|
8
|
+
q_proj: 'qProj',
|
|
9
|
+
k_proj: 'kProj',
|
|
10
|
+
v_proj: 'vProj',
|
|
11
|
+
o_proj: 'oProj',
|
|
12
|
+
gate_proj: 'gate',
|
|
13
|
+
up_proj: 'up',
|
|
14
|
+
down_proj: 'down',
|
|
15
|
+
gate_up_proj: 'gateUp',
|
|
16
|
+
});
|
|
17
|
+
|
|
7
18
|
async function getExperimentalLoRAModule() {
|
|
8
19
|
loraModulePromise ??= import('../../experimental/adapters/lora-loader.js');
|
|
9
20
|
return loraModulePromise;
|
|
@@ -23,6 +34,57 @@ function getActiveLoRAName(pipeline) {
|
|
|
23
34
|
return active ? active.name : null;
|
|
24
35
|
}
|
|
25
36
|
|
|
37
|
+
function getPipelineModelId(pipeline) {
|
|
38
|
+
return String(
|
|
39
|
+
pipeline?.manifest?.modelId ||
|
|
40
|
+
pipeline?.dopplerLoader?.manifest?.modelId ||
|
|
41
|
+
pipeline?.dopplerLoader?.modelId ||
|
|
42
|
+
''
|
|
43
|
+
).trim();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function assertLoRABaseModelForPipeline(pipeline, lora) {
|
|
47
|
+
const modelId = getPipelineModelId(pipeline);
|
|
48
|
+
if (!modelId) {
|
|
49
|
+
throw new Error('LoRA activation requires a loaded pipeline with an exact manifest modelId.');
|
|
50
|
+
}
|
|
51
|
+
const baseModel = String(lora?.baseModel || '').trim();
|
|
52
|
+
if (!baseModel) {
|
|
53
|
+
throw new Error('LoRA activation requires the adapter to declare baseModel.');
|
|
54
|
+
}
|
|
55
|
+
if (baseModel !== modelId) {
|
|
56
|
+
throw new Error(
|
|
57
|
+
`LoRA adapter targets base model "${baseModel}", but the loaded model is "${modelId}".`
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function assertLoRATargetsForPipeline(pipeline, lora) {
|
|
63
|
+
if (!(pipeline?.weights instanceof Map)) {
|
|
64
|
+
throw new Error('LoRA activation requires loaded per-layer model weights.');
|
|
65
|
+
}
|
|
66
|
+
if (!(lora?.layers instanceof Map) || lora.layers.size === 0) {
|
|
67
|
+
throw new Error('LoRA activation requires at least one loaded adapter layer.');
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
for (const [layerIndex, modules] of lora.layers.entries()) {
|
|
71
|
+
const layerWeights = pipeline.weights.get(`layer_${layerIndex}`);
|
|
72
|
+
if (!layerWeights) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
`LoRA adapter targets layer ${layerIndex}, which is absent from the loaded model.`
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
for (const moduleName of Object.keys(modules)) {
|
|
78
|
+
const weightKey = LORA_TARGET_WEIGHT_KEYS[moduleName];
|
|
79
|
+
if (!weightKey || !layerWeights[weightKey]) {
|
|
80
|
+
throw new Error(
|
|
81
|
+
`LoRA adapter targets ${moduleName} at layer ${layerIndex}, but the loaded model has no compatible weight.`
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
26
88
|
export async function loadLoRAAdapterForPipeline(pipeline, adapter, loadOptions = {}) {
|
|
27
89
|
if (!pipeline) {
|
|
28
90
|
throw new Error('No model loaded. Call load() first.');
|
|
@@ -42,6 +104,8 @@ export async function loadLoRAAdapterForPipeline(pipeline, adapter, loadOptions
|
|
|
42
104
|
lora = await loadLoRAFromManifest(adapter, options);
|
|
43
105
|
}
|
|
44
106
|
|
|
107
|
+
assertLoRABaseModelForPipeline(pipeline, lora);
|
|
108
|
+
assertLoRATargetsForPipeline(pipeline, lora);
|
|
45
109
|
pipeline.setLoRAAdapter(lora);
|
|
46
110
|
log.info('doppler', `LoRA adapter loaded: ${lora.name}`);
|
|
47
111
|
}
|
|
@@ -13,7 +13,6 @@ import { requestPersistence, getStorageReport } from '../../storage/quota.js';
|
|
|
13
13
|
import { initDevice, getKernelCapabilities, getDeviceLimits, destroyDevice, getDevice } from '../../gpu/device.js';
|
|
14
14
|
import { prepareKernelRuntime } from '../../gpu/kernel-runtime.js';
|
|
15
15
|
import { createPipeline } from '../../inference/pipelines/text.js';
|
|
16
|
-
import { getDopplerLoader } from '../../loader/doppler-loader.js';
|
|
17
16
|
import { log } from '../../debug/index.js';
|
|
18
17
|
import { DopplerCapabilities } from './types.js';
|
|
19
18
|
import { GB, HEADER_READ_SIZE } from '../../config/schema/index.js';
|
|
@@ -25,22 +24,20 @@ import {
|
|
|
25
24
|
synthesizeStoredSourceArtifactManifest,
|
|
26
25
|
verifyStoredSourceArtifact,
|
|
27
26
|
} from '../../storage/source-artifact-store.js';
|
|
27
|
+
import {
|
|
28
|
+
activateLoRAFromTrainingOutputForPipeline,
|
|
29
|
+
getActiveLoRAForPipeline,
|
|
30
|
+
} from './lora.js';
|
|
28
31
|
|
|
29
32
|
let pipeline = null;
|
|
30
33
|
let currentModelId = null;
|
|
31
34
|
let bridgeModulePromise = null;
|
|
32
|
-
let loraModulePromise = null;
|
|
33
35
|
|
|
34
36
|
async function getExperimentalBridgeModule() {
|
|
35
37
|
bridgeModulePromise ??= import('../../experimental/bridge/index.js');
|
|
36
38
|
return bridgeModulePromise;
|
|
37
39
|
}
|
|
38
40
|
|
|
39
|
-
async function getExperimentalLoRAModule() {
|
|
40
|
-
loraModulePromise ??= import('../../experimental/adapters/lora-loader.js');
|
|
41
|
-
return loraModulePromise;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
41
|
function manifestsDiffer(localManifest, remoteManifest) {
|
|
45
42
|
if (!localManifest || !remoteManifest) return true;
|
|
46
43
|
if (localManifest.modelId !== remoteManifest.modelId) return true;
|
|
@@ -582,168 +579,10 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
582
579
|
}
|
|
583
580
|
}
|
|
584
581
|
|
|
585
|
-
function createLoRALoadOptions(overrides = {}) {
|
|
586
|
-
return {
|
|
587
|
-
readOPFS: readOPFSFile,
|
|
588
|
-
writeOPFS: writeOPFSFile,
|
|
589
|
-
fetchUrl: fetchArrayBuffer,
|
|
590
|
-
...overrides,
|
|
591
|
-
};
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
async function loadLoRAAdapter(adapter, loadOptions = {}) {
|
|
595
|
-
if (!pipeline) {
|
|
596
|
-
throw new Error('No model loaded. Call loadModel() first.');
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
const options = createLoRALoadOptions(loadOptions);
|
|
600
|
-
|
|
601
|
-
let lora;
|
|
602
|
-
if (typeof adapter === 'string') {
|
|
603
|
-
const { loadLoRAFromUrl } = await getExperimentalLoRAModule();
|
|
604
|
-
lora = await loadLoRAFromUrl(adapter, options);
|
|
605
|
-
} else if (adapter.adapterType === 'lora' || adapter.modelType === 'lora') {
|
|
606
|
-
const loader = pipeline.dopplerLoader || getDopplerLoader();
|
|
607
|
-
await loader.init();
|
|
608
|
-
lora = await loader.loadLoRAWeights(adapter);
|
|
609
|
-
} else {
|
|
610
|
-
const { loadLoRAFromManifest } = await getExperimentalLoRAModule();
|
|
611
|
-
lora = await loadLoRAFromManifest(adapter, options);
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
pipeline.setLoRAAdapter(lora);
|
|
615
|
-
log.info('DopplerProvider', `LoRA adapter loaded: ${lora.name}`);
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
async function readLocalJson(path) {
|
|
619
|
-
const { readFile } = await import('node:fs/promises');
|
|
620
|
-
const raw = await readFile(path, 'utf8');
|
|
621
|
-
return JSON.parse(raw);
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
async function createLocalFileLoadOptions(path) {
|
|
625
|
-
const { readFile } = await import('node:fs/promises');
|
|
626
|
-
const { dirname, isAbsolute, join } = await import('node:path');
|
|
627
|
-
const basePath = dirname(path);
|
|
628
|
-
return {
|
|
629
|
-
basePath,
|
|
630
|
-
resolvePath(filePath) {
|
|
631
|
-
return isAbsolute(filePath) ? filePath : join(basePath, filePath);
|
|
632
|
-
},
|
|
633
|
-
async readFile(filePath) {
|
|
634
|
-
const data = await readFile(filePath);
|
|
635
|
-
return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
|
|
636
|
-
},
|
|
637
|
-
};
|
|
638
|
-
}
|
|
639
|
-
|
|
640
582
|
export async function activateLoRAFromTrainingOutput(trainingOutput) {
|
|
641
|
-
|
|
642
|
-
return {
|
|
643
|
-
activated: false,
|
|
644
|
-
adapterName: null,
|
|
645
|
-
source: null,
|
|
646
|
-
reason: 'no_model_loaded',
|
|
647
|
-
};
|
|
648
|
-
}
|
|
649
|
-
|
|
650
|
-
const output = trainingOutput && typeof trainingOutput === 'object'
|
|
651
|
-
? trainingOutput
|
|
652
|
-
: null;
|
|
653
|
-
if (!output && typeof trainingOutput !== 'string') {
|
|
654
|
-
return {
|
|
655
|
-
activated: false,
|
|
656
|
-
adapterName: getActiveLoRA(),
|
|
657
|
-
source: null,
|
|
658
|
-
reason: 'no_adapter_candidate',
|
|
659
|
-
};
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
if (typeof trainingOutput === 'string') {
|
|
663
|
-
await loadLoRAAdapter(trainingOutput);
|
|
664
|
-
return {
|
|
665
|
-
activated: true,
|
|
666
|
-
adapterName: getActiveLoRA(),
|
|
667
|
-
source: 'adapter-string',
|
|
668
|
-
reason: null,
|
|
669
|
-
};
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
if (output.adapterManifest && typeof output.adapterManifest === 'object') {
|
|
673
|
-
await loadLoRAAdapter(output.adapterManifest);
|
|
674
|
-
return {
|
|
675
|
-
activated: true,
|
|
676
|
-
adapterName: getActiveLoRA(),
|
|
677
|
-
source: 'adapterManifest',
|
|
678
|
-
reason: null,
|
|
679
|
-
};
|
|
680
|
-
}
|
|
681
|
-
|
|
682
|
-
if (typeof output.adapterManifestJson === 'string' && output.adapterManifestJson.trim()) {
|
|
683
|
-
const manifest = JSON.parse(output.adapterManifestJson);
|
|
684
|
-
await loadLoRAAdapter(manifest);
|
|
685
|
-
return {
|
|
686
|
-
activated: true,
|
|
687
|
-
adapterName: getActiveLoRA(),
|
|
688
|
-
source: 'adapterManifestJson',
|
|
689
|
-
reason: null,
|
|
690
|
-
};
|
|
691
|
-
}
|
|
692
|
-
|
|
693
|
-
if (typeof output.adapterManifestUrl === 'string' && output.adapterManifestUrl.trim()) {
|
|
694
|
-
await loadLoRAAdapter(output.adapterManifestUrl.trim());
|
|
695
|
-
return {
|
|
696
|
-
activated: true,
|
|
697
|
-
adapterName: getActiveLoRA(),
|
|
698
|
-
source: 'adapterManifestUrl',
|
|
699
|
-
reason: null,
|
|
700
|
-
};
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
if (typeof output.adapterManifestPath === 'string' && output.adapterManifestPath.trim()) {
|
|
704
|
-
const path = output.adapterManifestPath.trim();
|
|
705
|
-
if (path.startsWith('http://') || path.startsWith('https://')) {
|
|
706
|
-
await loadLoRAAdapter(path);
|
|
707
|
-
return {
|
|
708
|
-
activated: true,
|
|
709
|
-
adapterName: getActiveLoRA(),
|
|
710
|
-
source: 'adapterManifestPath:url',
|
|
711
|
-
reason: null,
|
|
712
|
-
};
|
|
713
|
-
}
|
|
714
|
-
const isNode = typeof process !== 'undefined' && !!process.versions?.node;
|
|
715
|
-
if (!isNode) {
|
|
716
|
-
throw new Error('adapterManifestPath local files require Node runtime.');
|
|
717
|
-
}
|
|
718
|
-
const manifest = await readLocalJson(path);
|
|
719
|
-
await loadLoRAAdapter(manifest, await createLocalFileLoadOptions(path));
|
|
720
|
-
return {
|
|
721
|
-
activated: true,
|
|
722
|
-
adapterName: getActiveLoRA(),
|
|
723
|
-
source: 'adapterManifestPath:file',
|
|
724
|
-
reason: null,
|
|
725
|
-
};
|
|
726
|
-
}
|
|
727
|
-
|
|
728
|
-
if (output.adapter != null) {
|
|
729
|
-
await loadLoRAAdapter(output.adapter);
|
|
730
|
-
return {
|
|
731
|
-
activated: true,
|
|
732
|
-
adapterName: getActiveLoRA(),
|
|
733
|
-
source: 'adapter',
|
|
734
|
-
reason: null,
|
|
735
|
-
};
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
return {
|
|
739
|
-
activated: false,
|
|
740
|
-
adapterName: getActiveLoRA(),
|
|
741
|
-
source: null,
|
|
742
|
-
reason: 'no_adapter_candidate',
|
|
743
|
-
};
|
|
583
|
+
return activateLoRAFromTrainingOutputForPipeline(pipeline, trainingOutput);
|
|
744
584
|
}
|
|
745
585
|
|
|
746
586
|
export function getActiveLoRA() {
|
|
747
|
-
|
|
748
|
-
return active ? active.name : null;
|
|
587
|
+
return getActiveLoRAForPipeline(pipeline);
|
|
749
588
|
}
|
|
@@ -2,7 +2,7 @@ import type { InferencePipeline, KVCacheSnapshot, PromptInput } from '../../infe
|
|
|
2
2
|
import type { ChatMessage } from '../../inference/pipelines/text/chat-format.js';
|
|
3
3
|
import type { GenerateOptions } from '../../generation/index.js';
|
|
4
4
|
import type { RDRRManifest } from '../../formats/rdrr/index.js';
|
|
5
|
-
import type { LogitsStepResult, PrefillResult } from '../../inference/pipelines/text/types.d.ts';
|
|
5
|
+
import type { LogitsStepResult, PrefillResult, SequenceEncodeOptions, SequenceEncodeResult } from '../../inference/pipelines/text/types.d.ts';
|
|
6
6
|
import type { LoRAManifest } from './types.js';
|
|
7
7
|
import type { LoRALoadOptions } from './lora.js';
|
|
8
8
|
|
|
@@ -22,6 +22,7 @@ export interface DopplerModelHandle {
|
|
|
22
22
|
generateText(prompt: string, options?: DopplerGenerateOptions): Promise<string>;
|
|
23
23
|
chat(messages: ChatMessage[], options?: DopplerGenerateOptions): AsyncGenerator<string, void, void>;
|
|
24
24
|
chatText(messages: ChatMessage[], options?: DopplerGenerateOptions): Promise<DopplerChatResponse>;
|
|
25
|
+
encodeSequence(sequence: string, options?: SequenceEncodeOptions): Promise<SequenceEncodeResult>;
|
|
25
26
|
resetGenerationState(): void;
|
|
26
27
|
loadLoRA(adapter: LoRAManifest | RDRRManifest | string, loadOptions?: LoRALoadOptions): Promise<void>;
|
|
27
28
|
activateLoRAFromTrainingOutput(
|
|
@@ -50,6 +51,7 @@ export interface DopplerModelHandle {
|
|
|
50
51
|
readonly manifestHash: string | null;
|
|
51
52
|
readonly manifest: unknown;
|
|
52
53
|
readonly deviceInfo: Record<string, unknown> | null;
|
|
54
|
+
readonly supportsSequence: boolean;
|
|
53
55
|
readonly advanced: {
|
|
54
56
|
tokenizeText(text: string): number[];
|
|
55
57
|
prefillKV(prompt: string, options?: DopplerGenerateOptions): Promise<KVCacheSnapshot>;
|
|
@@ -93,6 +93,9 @@ export function createModelHandle(pipeline, resolved) {
|
|
|
93
93
|
async embedBatch(prompts, options = {}) {
|
|
94
94
|
return pipeline.embedBatch(prompts, options);
|
|
95
95
|
},
|
|
96
|
+
async encodeSequence(sequence, options = {}) {
|
|
97
|
+
return pipeline.encodeSequence(sequence, options);
|
|
98
|
+
},
|
|
96
99
|
async embedImage(args = {}) {
|
|
97
100
|
return pipeline.embedImage(args);
|
|
98
101
|
},
|
|
@@ -112,6 +115,9 @@ export function createModelHandle(pipeline, resolved) {
|
|
|
112
115
|
return pipeline.manifest?.modelType === 'embedding'
|
|
113
116
|
|| pipeline.manifest?.inference?.supportsEmbedding === true;
|
|
114
117
|
},
|
|
118
|
+
get supportsSequence() {
|
|
119
|
+
return pipeline.manifest?.inference?.supportsSequence === true;
|
|
120
|
+
},
|
|
115
121
|
get supportsTranscription() {
|
|
116
122
|
return pipeline.manifest?.inference?.supportsTranscription === true
|
|
117
123
|
&& pipeline.audioCapable === true;
|
|
@@ -136,6 +136,10 @@ export function wrapPipelineAsHandle(pipeline, resolved = {}) {
|
|
|
136
136
|
return manifest?.modelType === 'embedding'
|
|
137
137
|
|| manifest?.inference?.supportsEmbedding === true;
|
|
138
138
|
},
|
|
139
|
+
get supportsSequence() {
|
|
140
|
+
const manifest = pipeline.manifest || resolved.manifest || null;
|
|
141
|
+
return manifest?.inference?.supportsSequence === true;
|
|
142
|
+
},
|
|
139
143
|
get supportsTranscription() {
|
|
140
144
|
const manifest = pipeline.manifest || resolved.manifest || null;
|
|
141
145
|
return manifest?.inference?.supportsTranscription === true
|
|
@@ -155,6 +159,9 @@ export function wrapPipelineAsHandle(pipeline, resolved = {}) {
|
|
|
155
159
|
async embedBatch(prompts, options = {}) {
|
|
156
160
|
return pipeline.embedBatch(prompts, options);
|
|
157
161
|
},
|
|
162
|
+
async encodeSequence(sequence, options = {}) {
|
|
163
|
+
return pipeline.encodeSequence(sequence, options);
|
|
164
|
+
},
|
|
158
165
|
async embedImage(args = {}) {
|
|
159
166
|
return pipeline.embedImage(args);
|
|
160
167
|
},
|