dsh-plugin-local-infer 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -3
- package/docs/zh/README.md +14 -3
- package/dsh/client.js +1214 -242
- package/package.json +16 -10
- package/packages/core/lib/adapter.d.ts +119 -6
- package/packages/core/lib/deps.d.ts +4 -3
- package/packages/core/lib/index.d.ts +2 -1
- package/packages/core/lib/index.js +1 -0
- package/packages/core/lib/launch-command.d.ts +13 -0
- package/packages/core/lib/launch-command.js +64 -0
- package/packages/core/lib/models.d.ts +2 -1
- package/packages/core/lib/models.js +10 -1
- package/packages/core/lib/registrar.d.ts +5 -5
- package/packages/core/lib/registrar.js +3 -3
- package/packages/core/lib/status.d.ts +9 -1
- package/packages/core/lib/status.js +6 -1
- package/packages/core/package.json +1 -1
- package/packages/providers/lib/acquire.d.ts +127 -0
- package/packages/providers/lib/acquire.js +346 -0
- package/packages/providers/lib/adapters.d.ts +19 -172
- package/packages/providers/lib/adapters.js +114 -659
- package/packages/providers/lib/deps.d.ts +26 -2
- package/packages/providers/lib/deps.js +37 -4
- package/packages/providers/lib/engines/hyperqwenAdapter.d.ts +38 -0
- package/packages/providers/lib/engines/hyperqwenAdapter.js +105 -0
- package/packages/providers/lib/engines/infernix.d.ts +170 -0
- package/packages/providers/lib/engines/infernix.js +671 -0
- package/packages/providers/lib/engines/infernixAdapter.d.ts +53 -0
- package/packages/providers/lib/engines/infernixAdapter.js +202 -0
- package/packages/providers/lib/engines/llamacppAdapter.d.ts +51 -0
- package/packages/providers/lib/engines/llamacppAdapter.js +232 -0
- package/packages/providers/lib/{ninfer.d.ts → engines/ninfer.d.ts} +7 -0
- package/packages/providers/lib/{ninfer.js → engines/ninfer.js} +8 -1
- package/packages/providers/lib/engines/ninferAdapter.d.ts +82 -0
- package/packages/providers/lib/engines/ninferAdapter.js +324 -0
- package/packages/providers/lib/engines/strata.d.ts +53 -0
- package/packages/providers/lib/engines/strata.js +197 -0
- package/packages/providers/lib/engines/strataAdapter.d.ts +98 -0
- package/packages/providers/lib/engines/strataAdapter.js +427 -0
- package/packages/providers/lib/engines.d.ts +5 -3
- package/packages/providers/lib/engines.js +16 -8
- package/packages/providers/lib/guidelines.js +16 -0
- package/packages/providers/lib/index.d.ts +21 -10
- package/packages/providers/lib/index.js +23 -9
- package/packages/providers/lib/model-defaults.d.ts +8 -5
- package/packages/providers/lib/model-defaults.js +47 -5
- package/packages/providers/lib/platform.d.ts +31 -0
- package/packages/providers/lib/platform.js +64 -0
- package/packages/providers/package.json +1 -1
- package/packages/service/lib/index.d.ts +74 -22
- package/packages/service/lib/index.js +354 -44
- package/packages/service/lib/types.d.ts +48 -5
- package/packages/service/package.json +1 -1
- package/packages/tool/package.json +1 -1
- package/packages/ui/lib/config.d.ts +79 -0
- package/packages/ui/lib/config.js +235 -0
- package/packages/ui/lib/deps.d.ts +3 -3
- package/packages/ui/lib/ops/engine.d.ts +108 -0
- package/packages/ui/lib/ops/engine.js +493 -0
- package/packages/ui/lib/ops/hf.d.ts +27 -0
- package/packages/ui/lib/ops/hf.js +60 -0
- package/packages/ui/lib/ops/logs.d.ts +23 -0
- package/packages/ui/lib/ops/logs.js +58 -0
- package/packages/ui/lib/ops/model.d.ts +79 -0
- package/packages/ui/lib/ops/model.js +130 -0
- package/packages/ui/lib/request.d.ts +106 -0
- package/packages/ui/lib/request.js +80 -0
- package/packages/ui/lib/route.d.ts +96 -357
- package/packages/ui/lib/route.js +123 -1415
- package/packages/ui/lib/routes/chat-template.d.ts +14 -0
- package/packages/ui/lib/routes/chat-template.js +37 -0
- package/packages/ui/lib/routes/deps.d.ts +16 -0
- package/packages/ui/lib/routes/deps.js +27 -0
- package/packages/ui/lib/routes/download.d.ts +16 -0
- package/packages/ui/lib/routes/download.js +48 -0
- package/packages/ui/lib/routes/get.d.ts +18 -0
- package/packages/ui/lib/routes/get.js +57 -0
- package/packages/ui/lib/routes/install.d.ts +16 -0
- package/packages/ui/lib/routes/install.js +36 -0
- package/packages/ui/lib/routes/launch.d.ts +18 -0
- package/packages/ui/lib/routes/launch.js +50 -0
- package/packages/ui/lib/routes/provider-delete.d.ts +15 -0
- package/packages/ui/lib/routes/provider-delete.js +37 -0
- package/packages/ui/lib/routes/register.d.ts +16 -0
- package/packages/ui/lib/routes/register.js +28 -0
- package/packages/ui/lib/routes/save.d.ts +18 -0
- package/packages/ui/lib/routes/save.js +139 -0
- package/packages/ui/lib/routes/stop.d.ts +16 -0
- package/packages/ui/lib/routes/stop.js +29 -0
- package/packages/ui/lib/routes/summary.d.ts +15 -0
- package/packages/ui/lib/routes/summary.js +26 -0
- package/packages/ui/lib/routes/unregister.d.ts +14 -0
- package/packages/ui/lib/routes/unregister.js +26 -0
- package/packages/ui/lib/viewmodels/deps.d.ts +31 -0
- package/packages/ui/lib/viewmodels/deps.js +89 -0
- package/packages/ui/lib/viewmodels/depsHelpers.d.ts +28 -0
- package/packages/ui/lib/viewmodels/depsHelpers.js +33 -0
- package/packages/ui/lib/viewmodels/summary.d.ts +140 -0
- package/packages/ui/lib/viewmodels/summary.js +225 -0
- package/packages/ui/lib/viewmodels/summaryHelpers.d.ts +30 -0
- package/packages/ui/lib/viewmodels/summaryHelpers.js +42 -0
- package/packages/ui/package.json +1 -1
- package/packages/providers/lib/strata.d.ts +0 -15
- package/packages/providers/lib/strata.js +0 -14
- /package/packages/providers/lib/{hyperqwen.d.ts → engines/hyperqwen.d.ts} +0 -0
- /package/packages/providers/lib/{hyperqwen.js → engines/hyperqwen.js} +0 -0
- /package/packages/providers/lib/{llamacpp.d.ts → engines/llamacpp.d.ts} +0 -0
- /package/packages/providers/lib/{llamacpp.js → engines/llamacpp.js} +0 -0
|
@@ -1,58 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The
|
|
3
|
-
*
|
|
4
|
-
* `LaunchPlan
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
2
|
+
* The shared engine-adapter helpers: the spec-to-argv and spec-to-env
|
|
3
|
+
* translators the concrete adapters (the `engines/*Adapter.ts` files) compose
|
|
4
|
+
* into a `LaunchPlan`, the card's Custom Arguments translators, the
|
|
5
|
+
* `LifecycleHost` the deploy, health-check, and stop steps drive, and the
|
|
6
|
+
* health poll. The node implementation (`nodeLifecycleHost`) is what the
|
|
7
|
+
* service wires in on a live host.
|
|
8
8
|
* @module dsh-plugin-local-infer/providers/adapters
|
|
9
9
|
*/
|
|
10
10
|
import { exec, spawn as nodeSpawn } from 'node:child_process';
|
|
11
|
-
import { basename, dirname, join } from 'node:path';
|
|
12
|
-
import { EngineAdapter } from '../../core/lib/index.js';
|
|
13
|
-
import { NINFER_PARAMETERS } from "./ninfer.js";
|
|
14
|
-
import { HYPERQWEN_PARAMETERS } from "./hyperqwen.js";
|
|
15
|
-
import { LLAMACPP_MODEL_ARTIFACTS, LLAMACPP_PARAMETERS, llamaCppModelKey, } from "./llamacpp.js";
|
|
16
|
-
import { STRATA_PARAMETERS } from "./strata.js";
|
|
17
|
-
/**
|
|
18
|
-
* The NInfer model artifacts, keyed by the model registry id. The 27B entry
|
|
19
|
-
* is the default the DEFAULT_MODELS list names; the 35B-A3B entries are the
|
|
20
|
-
* Qwen3.8 measured artifact (`qwen3-35b`) and the Qwen3.6 default
|
|
21
|
-
* (`qwen36-35b`).
|
|
22
|
-
*/
|
|
23
|
-
export const NINFER_MODEL_ARTIFACTS = {
|
|
24
|
-
'qwen3-27b': {
|
|
25
|
-
file: 'qwen3_8_27b_nvfp4.ninfer',
|
|
26
|
-
repo: 'neroued/Qwen3.8-27B-nvfp4-NInfer',
|
|
27
|
-
modelId: 'Qwen3.8-27B',
|
|
28
|
-
},
|
|
29
|
-
// The Swift 1.5 ablated 27B the local launcher bootstraps: the nvfp4-full
|
|
30
|
-
// DFlash2 artifact the qwen-ninfer script serves. It reuses the 27B model id
|
|
31
|
-
// the engine advertises, so a chat turn routes to the same alias.
|
|
32
|
-
'qwen3-27b-swift15': {
|
|
33
|
-
file: 'qwen3_8_27b_swift15abl_nvfp4full-dflash2.ninfer',
|
|
34
|
-
repo: 'kaushikvira/Qwen3.8-27B-swift15-uncensored-ajgazin-nvfp4full-dflash2-NInfer-v3',
|
|
35
|
-
modelId: 'Qwen3.8-27B',
|
|
36
|
-
},
|
|
37
|
-
'qwen3-35b': {
|
|
38
|
-
file: 'qwen3_8_35b_a3b.ninfer',
|
|
39
|
-
repo: 'Neroued/Qwen3.8-35B-A3B-NInfer',
|
|
40
|
-
modelId: 'Qwen3.8-35B-A3B',
|
|
41
|
-
},
|
|
42
|
-
'qwen36-35b': {
|
|
43
|
-
file: 'qwen3_6_35b_a3b.ninfer',
|
|
44
|
-
repo: 'neroued/Qwen3.6-35B-A3B-NInfer',
|
|
45
|
-
modelId: 'Qwen3.6-35B-A3B',
|
|
46
|
-
},
|
|
47
|
-
};
|
|
48
|
-
/**
|
|
49
|
-
* artifactFor resolves the artifact a model id deploys. An id no entry names
|
|
50
|
-
* falls back to the 27B artifact, so a bare config always renders a plan.
|
|
51
|
-
*/
|
|
52
|
-
function artifactFor(modelId) {
|
|
53
|
-
const artifact = modelId !== undefined ? NINFER_MODEL_ARTIFACTS[modelId] : undefined;
|
|
54
|
-
return artifact ?? NINFER_MODEL_ARTIFACTS['qwen3-27b'];
|
|
55
|
-
}
|
|
56
11
|
/**
|
|
57
12
|
* flagsFromSpecs turns a spec surface and a config into the CLI flag list: each
|
|
58
13
|
* spec's `toArgv` runs over the config value (falling back to the spec
|
|
@@ -169,6 +124,35 @@ export function createNodeLifecycleHost(debug) {
|
|
|
169
124
|
});
|
|
170
125
|
});
|
|
171
126
|
},
|
|
127
|
+
async runStream(command, onOutput) {
|
|
128
|
+
log?.(`run: ${command}`);
|
|
129
|
+
return new Promise(resolve => {
|
|
130
|
+
// No overall timeout: a full CUDA build legitimately runs long, and
|
|
131
|
+
// the card prices a stalled step with its own output-idle watchdog.
|
|
132
|
+
const child = nodeSpawn(command, { shell: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
133
|
+
let output = '';
|
|
134
|
+
const append = (chunk) => {
|
|
135
|
+
const text = chunk.toString('utf8');
|
|
136
|
+
onOutput(text);
|
|
137
|
+
output += text;
|
|
138
|
+
if (output.length > 2048) {
|
|
139
|
+
output = output.slice(-1024);
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
child.stdout.on('data', append);
|
|
143
|
+
child.stderr.on('data', append);
|
|
144
|
+
child.on('error', error => {
|
|
145
|
+
resolve({ ok: false, detail: error.message });
|
|
146
|
+
});
|
|
147
|
+
child.on('close', code => {
|
|
148
|
+
if (code === 0) {
|
|
149
|
+
resolve({ ok: true });
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
resolve({ ok: false, detail: `exit ${code}: ${output.trim().slice(-1024)}` });
|
|
153
|
+
});
|
|
154
|
+
});
|
|
155
|
+
},
|
|
172
156
|
spawn(argv, env, cwd) {
|
|
173
157
|
log?.(`spawn: ${argv.join(' ')}${cwd !== undefined ? ` (cwd: ${cwd})` : ''}`);
|
|
174
158
|
const child = nodeSpawn(argv[0] ?? '', argv.slice(1), {
|
|
@@ -190,6 +174,51 @@ export function createNodeLifecycleHost(debug) {
|
|
|
190
174
|
};
|
|
191
175
|
child.stdout?.on('data', append);
|
|
192
176
|
child.stderr?.on('data', append);
|
|
177
|
+
// An EventEmitter with no 'error' listener turns the event into a thrown
|
|
178
|
+
// uncaughtException, which takes the DSH host down with it. A spawn that
|
|
179
|
+
// cannot start (a missing engine binary, an ENOENT) and a stream that
|
|
180
|
+
// breaks on a crashing child both emit 'error' here, so every path gets
|
|
181
|
+
// a listener and the failure lands in the logs instead of the crash.
|
|
182
|
+
child.stdout?.on('error', (error) => {
|
|
183
|
+
logs = (logs + `stdout error: ${error.message}\n`).slice(-65536);
|
|
184
|
+
});
|
|
185
|
+
child.stderr?.on('error', (error) => {
|
|
186
|
+
logs = (logs + `stderr error: ${error.message}\n`).slice(-65536);
|
|
187
|
+
});
|
|
188
|
+
// The one-shot exit contract: 'error' names a child that never ran,
|
|
189
|
+
// 'exit' carries the code or signal, and 'close' follows once the pipes
|
|
190
|
+
// drain. The first settled source wins, so the handle's onExit listener
|
|
191
|
+
// (crash, pkill, or clean stop) fires exactly once with the detail.
|
|
192
|
+
const listeners = new Set();
|
|
193
|
+
let exitInfo = null;
|
|
194
|
+
const settle = (info) => {
|
|
195
|
+
if (exitInfo !== null)
|
|
196
|
+
return;
|
|
197
|
+
exitInfo = info;
|
|
198
|
+
for (const listener of listeners) {
|
|
199
|
+
try {
|
|
200
|
+
listener(info);
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
// A listener that throws must not re-raise into the child's event
|
|
204
|
+
// pipeline and take the host down the way an unlistened 'error' did.
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
listeners.clear();
|
|
208
|
+
};
|
|
209
|
+
let exitSignal = null;
|
|
210
|
+
child.on('error', (error) => {
|
|
211
|
+
logs = (logs + `spawn error: ${error.message}\n`).slice(-65536);
|
|
212
|
+
// The error fires before the stdio drains; settle immediately so a
|
|
213
|
+
// launch of a missing binary reports rather than hangs the watcher.
|
|
214
|
+
settle({ code: null, signal: null, error: error.message });
|
|
215
|
+
});
|
|
216
|
+
child.on('exit', (_code, signal) => {
|
|
217
|
+
exitSignal = signal;
|
|
218
|
+
});
|
|
219
|
+
child.on('close', (code, signal) => {
|
|
220
|
+
settle({ code, signal: signal ?? exitSignal });
|
|
221
|
+
});
|
|
193
222
|
return {
|
|
194
223
|
pid: child.pid ?? -1,
|
|
195
224
|
kill: () => {
|
|
@@ -198,6 +227,16 @@ export function createNodeLifecycleHost(debug) {
|
|
|
198
227
|
get logs() {
|
|
199
228
|
return logs;
|
|
200
229
|
},
|
|
230
|
+
onExit: (listener) => {
|
|
231
|
+
// A listener registered after the child already ended fires
|
|
232
|
+
// immediately, so a late attach (the service wiring after deploy)
|
|
233
|
+
// never misses the crash that landed in between.
|
|
234
|
+
if (exitInfo !== null) {
|
|
235
|
+
listener(exitInfo);
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
listeners.add(listener);
|
|
239
|
+
},
|
|
201
240
|
};
|
|
202
241
|
},
|
|
203
242
|
async fetch(url) {
|
|
@@ -219,6 +258,24 @@ export function createNodeLifecycleHost(debug) {
|
|
|
219
258
|
* in debug mode.
|
|
220
259
|
*/
|
|
221
260
|
export const nodeLifecycleHost = createNodeLifecycleHost();
|
|
261
|
+
/**
|
|
262
|
+
* trackChild stores a spawned handle in the adapter's children map and ties
|
|
263
|
+
* the entry's lifetime to the child's: when the child ends (a crash, a pkill,
|
|
264
|
+
* or the adapter's own stop kill) the entry is dropped, so a later stop never
|
|
265
|
+
* signals a pid a recycled process has taken over. A handle whose host cannot
|
|
266
|
+
* watch (onExit absent) stays tracked until stop removes it.
|
|
267
|
+
*/
|
|
268
|
+
export function trackChild(children, key, handle) {
|
|
269
|
+
children.set(key, handle);
|
|
270
|
+
handle.onExit?.(() => {
|
|
271
|
+
// Compare identity before deleting: a newer launch that already replaced
|
|
272
|
+
// this key's entry keeps its handle, and only the ended child's entry
|
|
273
|
+
// goes.
|
|
274
|
+
if (children.get(key) === handle) {
|
|
275
|
+
children.delete(key);
|
|
276
|
+
}
|
|
277
|
+
});
|
|
278
|
+
}
|
|
222
279
|
/**
|
|
223
280
|
* pollHealth polls a plan's health URL until a 200 or the attempt cap, the
|
|
224
281
|
* same loop the qwen-ninfer launcher runs. A poll interval of 0 skips the
|
|
@@ -243,616 +300,14 @@ export async function pollHealth(host, plan, options) {
|
|
|
243
300
|
detail: `health check did not return 200 within ${maxAttempts} attempts`,
|
|
244
301
|
};
|
|
245
302
|
}
|
|
246
|
-
|
|
303
|
+
/**
|
|
304
|
+
* delay waits `ms` milliseconds. The health polls use it between attempts; a
|
|
305
|
+
* poll interval of 0 skips the wait, which the tests use to run the loop
|
|
306
|
+
* without a real sleep.
|
|
307
|
+
*/
|
|
308
|
+
export function delay(ms) {
|
|
247
309
|
return new Promise(resolve => {
|
|
248
310
|
setTimeout(resolve, ms);
|
|
249
311
|
});
|
|
250
312
|
}
|
|
251
|
-
/**
|
|
252
|
-
* The NInfer docker image the adapter pulls and runs. The engineFork platform
|
|
253
|
-
* config overrides the git source for a local build; the image stays the
|
|
254
|
-
* upstream ghcr artifact.
|
|
255
|
-
*/
|
|
256
|
-
export const NINFER_DOCKER_IMAGE = 'ghcr.io/iamwavecut/ninfer-all';
|
|
257
|
-
/**
|
|
258
|
-
* NInferAdapter is the canonical both-capable engine. `renderLaunch` renders a
|
|
259
|
-
* local `ninfer-serve` process or a `docker run` of the ghcr image from the
|
|
260
|
-
* same spec surface; `deploy` builds the engine in the engine directory (or
|
|
261
|
-
* pulls the image), downloads the model into the model directory, and starts
|
|
262
|
-
* the engine; `healthCheck` polls the health URL; `stop` signals the tracked
|
|
263
|
-
* process and stops the named container.
|
|
264
|
-
*/
|
|
265
|
-
export class NInferAdapter extends EngineAdapter {
|
|
266
|
-
id = 'ninfer';
|
|
267
|
-
supports = { docker: true, local: true };
|
|
268
|
-
host;
|
|
269
|
-
children = new Map();
|
|
270
|
-
constructor(host = nodeLifecycleHost) {
|
|
271
|
-
super();
|
|
272
|
-
this.host = host;
|
|
273
|
-
}
|
|
274
|
-
parameters() {
|
|
275
|
-
return NINFER_PARAMETERS;
|
|
276
|
-
}
|
|
277
|
-
modelRef(config) {
|
|
278
|
-
// The id the engine advertises on its OpenAI endpoint is the artifact's
|
|
279
|
-
// --model-id alias, not the VramModel registry key the config names, so
|
|
280
|
-
// the registrar publishes the alias and a chat turn routes to it.
|
|
281
|
-
const artifact = artifactFor(typeof config.model === 'string' ? config.model : 'qwen3-27b');
|
|
282
|
-
return { id: artifact.modelId, name: artifact.modelId };
|
|
283
|
-
}
|
|
284
|
-
launchSignature(config = {}) {
|
|
285
|
-
// The local plan runs the ninfer-serve binary; the docker plan's host
|
|
286
|
-
// process is docker run of the ghcr image, so the image name is the
|
|
287
|
-
// command-line marker the reserve probe matches on.
|
|
288
|
-
const port = this.port(config);
|
|
289
|
-
if (this.buildMode(config) === 'docker') {
|
|
290
|
-
return { binary: NINFER_DOCKER_IMAGE, port };
|
|
291
|
-
}
|
|
292
|
-
return { binary: 'ninfer-serve', port };
|
|
293
|
-
}
|
|
294
|
-
renderLaunch(config, _hw) {
|
|
295
|
-
const port = this.port(config);
|
|
296
|
-
const modelId = typeof config.model === 'string' ? config.model : 'qwen3-27b';
|
|
297
|
-
const artifact = artifactFor(modelId);
|
|
298
|
-
// A model key that names a local artifact file (a directory-scanned
|
|
299
|
-
// model) overrides the artifact map's file, so a file the user has on
|
|
300
|
-
// disk launches as-is.
|
|
301
|
-
const file = modelId.endsWith('.ninfer') ? modelId : artifact.file;
|
|
302
|
-
const modelDirectory = typeof config.modelDirectory === 'string' ? config.modelDirectory : '';
|
|
303
|
-
const flags = flagsFromSpecs(NINFER_PARAMETERS, config);
|
|
304
|
-
if (this.buildMode(config) === 'docker') {
|
|
305
|
-
const modelPath = modelDirectory === '' ? file : join(modelDirectory, file);
|
|
306
|
-
const containerModelPath = `/models/${basename(file)}`;
|
|
307
|
-
const modelMount = modelDirectory === ''
|
|
308
|
-
? `${file}:${containerModelPath}`
|
|
309
|
-
: `${dirname(modelPath)}:/models`;
|
|
310
|
-
const containerName = `ninfer-${port}`;
|
|
311
|
-
const customArgs = customArgFlags(config);
|
|
312
|
-
return {
|
|
313
|
-
engineId: this.id,
|
|
314
|
-
kind: 'docker',
|
|
315
|
-
argv: [
|
|
316
|
-
'docker',
|
|
317
|
-
'run',
|
|
318
|
-
'--rm',
|
|
319
|
-
'--gpus',
|
|
320
|
-
'all',
|
|
321
|
-
'--name',
|
|
322
|
-
containerName,
|
|
323
|
-
'-p',
|
|
324
|
-
`127.0.0.1:${port}:8080`,
|
|
325
|
-
'--ulimit',
|
|
326
|
-
'memlock=-1',
|
|
327
|
-
'-v',
|
|
328
|
-
'ninfer-cache:/cache',
|
|
329
|
-
'-v',
|
|
330
|
-
modelMount,
|
|
331
|
-
NINFER_DOCKER_IMAGE,
|
|
332
|
-
'serve',
|
|
333
|
-
containerModelPath,
|
|
334
|
-
'--model-id',
|
|
335
|
-
artifact.modelId,
|
|
336
|
-
...flags,
|
|
337
|
-
...customArgs,
|
|
338
|
-
],
|
|
339
|
-
env: {},
|
|
340
|
-
containerName,
|
|
341
|
-
modelId,
|
|
342
|
-
...(modelDirectory === '' ? {} : { modelDirectory }),
|
|
343
|
-
port,
|
|
344
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
345
|
-
};
|
|
346
|
-
}
|
|
347
|
-
const modelPath = modelDirectory === '' ? file : join(modelDirectory, file);
|
|
348
|
-
const customArgs = customArgFlags(config);
|
|
349
|
-
return {
|
|
350
|
-
engineId: this.id,
|
|
351
|
-
kind: 'process',
|
|
352
|
-
argv: [
|
|
353
|
-
'ninfer-serve',
|
|
354
|
-
modelPath,
|
|
355
|
-
'--host',
|
|
356
|
-
'127.0.0.1',
|
|
357
|
-
'--port',
|
|
358
|
-
String(port),
|
|
359
|
-
'--model-id',
|
|
360
|
-
artifact.modelId,
|
|
361
|
-
...flags,
|
|
362
|
-
...customArgs,
|
|
363
|
-
],
|
|
364
|
-
env: {},
|
|
365
|
-
modelId,
|
|
366
|
-
...(modelDirectory === '' ? {} : { modelDirectory }),
|
|
367
|
-
port,
|
|
368
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
369
|
-
};
|
|
370
|
-
}
|
|
371
|
-
async deploy(plan, platform) {
|
|
372
|
-
if (plan.kind === 'process') {
|
|
373
|
-
const engineDir = platform?.engineDirectory;
|
|
374
|
-
if (engineDir !== undefined) {
|
|
375
|
-
const fork = platform?.engineFork;
|
|
376
|
-
if (fork !== undefined) {
|
|
377
|
-
// Mirror the qwen-ninfer launcher: an existing checkout (a .git under
|
|
378
|
-
// the engine directory) is updated in place with a best-effort fetch
|
|
379
|
-
// and a hard reset to the fork's master, then built as-is; a missing
|
|
380
|
-
// checkout is a fresh clone. The fetch is best-effort (a `; true`
|
|
381
|
-
// pins the branch to success) so an offline host still builds the
|
|
382
|
-
// checkout it has; only a missing checkout that cannot be cloned
|
|
383
|
-
// fails the step.
|
|
384
|
-
const sync = await this.host.run(`if [ -d ${engineDir}/.git ]; then ` +
|
|
385
|
-
`git -C ${engineDir} fetch ${fork} 2>/dev/null && git -C ${engineDir} reset --hard ${fork}/master 2>/dev/null; true; ` +
|
|
386
|
-
`else git clone ${fork} ${engineDir}; fi`);
|
|
387
|
-
if (!sync.ok) {
|
|
388
|
-
return { ok: false, detail: `git sync failed: ${sync.detail ?? 'unknown'}` };
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
const build = await this.host.run(`cmake -S ${engineDir} -B ${engineDir}/build -G Ninja -DCMAKE_BUILD_TYPE=Release && ` +
|
|
392
|
-
`cmake --build ${engineDir}/build -j`);
|
|
393
|
-
if (!build.ok) {
|
|
394
|
-
return { ok: false, detail: `build failed: ${build.detail ?? 'unknown'}` };
|
|
395
|
-
}
|
|
396
|
-
}
|
|
397
|
-
const download = await this.downloadModel(plan);
|
|
398
|
-
if (!download.ok)
|
|
399
|
-
return download;
|
|
400
|
-
// A chat template the user picked as an HF repo reference (the combo
|
|
401
|
-
// control's known choices) is not a path the engine can read, so the
|
|
402
|
-
// deploy step downloads it into the model directory and rewrites the
|
|
403
|
-
// --chat-template flag to the local file before the spawn.
|
|
404
|
-
const templateResult = await this.resolveChatTemplate(plan);
|
|
405
|
-
if (!templateResult.ok)
|
|
406
|
-
return templateResult;
|
|
407
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {}, engineDir);
|
|
408
|
-
this.children.set(this.planKey(plan), handle);
|
|
409
|
-
return { ok: true, handle };
|
|
410
|
-
}
|
|
411
|
-
if (plan.kind === 'docker') {
|
|
412
|
-
const pull = await this.host.run(`docker pull ${NINFER_DOCKER_IMAGE}`);
|
|
413
|
-
if (!pull.ok) {
|
|
414
|
-
return { ok: false, detail: `docker pull failed: ${pull.detail ?? 'unknown'}` };
|
|
415
|
-
}
|
|
416
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {});
|
|
417
|
-
this.children.set(this.planKey(plan), handle);
|
|
418
|
-
return { ok: true, handle };
|
|
419
|
-
}
|
|
420
|
-
// NInfer renders process and docker plans; a compose plan is a no-op.
|
|
421
|
-
return { ok: true };
|
|
422
|
-
}
|
|
423
|
-
async healthCheck(plan, options) {
|
|
424
|
-
return pollHealth(this.host, plan, options);
|
|
425
|
-
}
|
|
426
|
-
async stop(plan) {
|
|
427
|
-
if (plan.kind === 'docker' && plan.containerName !== undefined) {
|
|
428
|
-
await this.host.run(`docker stop ${plan.containerName}`);
|
|
429
|
-
}
|
|
430
|
-
const handle = this.children.get(this.planKey(plan));
|
|
431
|
-
if (handle !== undefined) {
|
|
432
|
-
handle.kill();
|
|
433
|
-
this.children.delete(this.planKey(plan));
|
|
434
|
-
}
|
|
435
|
-
}
|
|
436
|
-
buildMode(config) {
|
|
437
|
-
return config.buildMode === 'docker' ? 'docker' : 'local';
|
|
438
|
-
}
|
|
439
|
-
port(config) {
|
|
440
|
-
return typeof config.port === 'number' ? config.port : 18020;
|
|
441
|
-
}
|
|
442
|
-
planKey(plan) {
|
|
443
|
-
return `${plan.engineId}:${plan.port}`;
|
|
444
|
-
}
|
|
445
|
-
async downloadModel(plan) {
|
|
446
|
-
const artifact = plan.modelId !== undefined ? NINFER_MODEL_ARTIFACTS[plan.modelId] : undefined;
|
|
447
|
-
if (artifact === undefined)
|
|
448
|
-
return { ok: true };
|
|
449
|
-
const target = plan.modelDirectory ?? '.';
|
|
450
|
-
const download = await this.host.run(`hf download ${artifact.repo} ${artifact.file} --local-dir ${target}`);
|
|
451
|
-
if (!download.ok) {
|
|
452
|
-
return { ok: false, detail: `model download failed: ${download.detail ?? 'unknown'}` };
|
|
453
|
-
}
|
|
454
|
-
return { ok: true };
|
|
455
|
-
}
|
|
456
|
-
/**
|
|
457
|
-
* resolveChatTemplate rewrites a --chat-template flag that carries an HF repo
|
|
458
|
-
* reference into the local file the engine reads. A value with a slash that
|
|
459
|
-
* is not an absolute path (the combo control's known choices) is a repo the
|
|
460
|
-
* deploy step downloads into the model directory; an already-local path is
|
|
461
|
-
* left as-is. The download is best-effort: a template that is already on
|
|
462
|
-
* disk is used without a re-fetch, and a missing hf CLI leaves the flag
|
|
463
|
-
* untouched so the engine reports the missing template itself.
|
|
464
|
-
*/
|
|
465
|
-
async resolveChatTemplate(plan) {
|
|
466
|
-
const argv = plan.argv;
|
|
467
|
-
if (argv === undefined)
|
|
468
|
-
return { ok: true };
|
|
469
|
-
const index = argv.indexOf('--chat-template');
|
|
470
|
-
if (index === -1)
|
|
471
|
-
return { ok: true };
|
|
472
|
-
const value = argv[index + 1];
|
|
473
|
-
if (value === undefined)
|
|
474
|
-
return { ok: true };
|
|
475
|
-
// An absolute path (a leading /) is already local; a value with no slash
|
|
476
|
-
// is a bare file name the engine resolves against its own directory.
|
|
477
|
-
if (value.startsWith('/') || !value.includes('/'))
|
|
478
|
-
return { ok: true };
|
|
479
|
-
// An HF repo reference: download the template file into the model
|
|
480
|
-
// directory and point the flag at the local copy.
|
|
481
|
-
const target = plan.modelDirectory ?? '.';
|
|
482
|
-
const localPath = `${target.endsWith('/') ? '' : `${target}/`}chat_template.jinja`;
|
|
483
|
-
// A template already on disk skips the re-fetch; the hf CLI is optional,
|
|
484
|
-
// so a download failure is reported but does not abort the launch.
|
|
485
|
-
const probe = await this.host.run(`[ -f ${localPath} ]`);
|
|
486
|
-
if (!probe.ok) {
|
|
487
|
-
const download = await this.host.run(`hf download ${value} chat_template.jinja --local-dir ${target}`);
|
|
488
|
-
if (!download.ok) {
|
|
489
|
-
return {
|
|
490
|
-
ok: false,
|
|
491
|
-
detail: `chat template download failed: ${download.detail ?? 'unknown'}`,
|
|
492
|
-
};
|
|
493
|
-
}
|
|
494
|
-
}
|
|
495
|
-
argv[index + 1] = localPath;
|
|
496
|
-
return { ok: true };
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
/**
|
|
500
|
-
* HyperQwenAdapter is the docker-compose engine. `renderLaunch` renders a
|
|
501
|
-
* compose plan (the compose file, the profile, and the `.env` the spec surface
|
|
502
|
-
* maps to); `deploy` runs `docker compose up`; `healthCheck` polls the health
|
|
503
|
-
* URL; `stop` runs `docker compose down`.
|
|
504
|
-
*/
|
|
505
|
-
export class HyperQwenAdapter extends EngineAdapter {
|
|
506
|
-
id = 'hyperqwen';
|
|
507
|
-
supports = { docker: true, local: false };
|
|
508
|
-
host;
|
|
509
|
-
constructor(host = nodeLifecycleHost) {
|
|
510
|
-
super();
|
|
511
|
-
this.host = host;
|
|
512
|
-
}
|
|
513
|
-
parameters() {
|
|
514
|
-
return HYPERQWEN_PARAMETERS;
|
|
515
|
-
}
|
|
516
|
-
modelRef(_config) {
|
|
517
|
-
// vLLM serves the model under the name the compose file passes, and the
|
|
518
|
-
// plugin tracks no per-model data for Hyper-Qwen yet, so the engine id is
|
|
519
|
-
// the stable entry the registrar publishes until that data lands.
|
|
520
|
-
return { id: 'hyperqwen', name: 'Hyper-Qwen' };
|
|
521
|
-
}
|
|
522
|
-
launchSignature(config = {}) {
|
|
523
|
-
// The compose host process is `docker compose ... up`, so the marker is
|
|
524
|
-
// the command, not an image. The toEnv keys (SPEC, CTX, KV, ...) name the
|
|
525
|
-
// .env the compose file reads, not the host process env, so they ride
|
|
526
|
-
// along as the env-key element the matcher checks when a process line
|
|
527
|
-
// carries env data.
|
|
528
|
-
const port = this.port(config);
|
|
529
|
-
const envKeys = Object.keys(envFromSpecs(HYPERQWEN_PARAMETERS, config));
|
|
530
|
-
return { binary: 'docker compose', port, envKeys };
|
|
531
|
-
}
|
|
532
|
-
renderLaunch(config, _hw) {
|
|
533
|
-
const port = this.port(config);
|
|
534
|
-
const profile = this.profile(config);
|
|
535
|
-
// The spec env and the card's Custom Arguments merge into one env map: the
|
|
536
|
-
// spec keys are the .env the compose file reads, and the custom rows are
|
|
537
|
-
// the user's own overrides appended after them.
|
|
538
|
-
const env = { ...envFromSpecs(HYPERQWEN_PARAMETERS, config), ...customArgEnv(config) };
|
|
539
|
-
const composeFile = typeof config.composeFile === 'string' ? config.composeFile : 'docker-compose.yml';
|
|
540
|
-
return {
|
|
541
|
-
engineId: this.id,
|
|
542
|
-
kind: 'compose',
|
|
543
|
-
env,
|
|
544
|
-
composeFile,
|
|
545
|
-
profile,
|
|
546
|
-
port,
|
|
547
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
548
|
-
};
|
|
549
|
-
}
|
|
550
|
-
async deploy(plan) {
|
|
551
|
-
const up = await this.host.run(`docker compose -f ${plan.composeFile} --profile ${plan.profile} up --pull always -d`);
|
|
552
|
-
if (!up.ok) {
|
|
553
|
-
return { ok: false, detail: `compose up failed: ${up.detail ?? 'unknown'}` };
|
|
554
|
-
}
|
|
555
|
-
return { ok: true };
|
|
556
|
-
}
|
|
557
|
-
async healthCheck(plan, options) {
|
|
558
|
-
return pollHealth(this.host, plan, options);
|
|
559
|
-
}
|
|
560
|
-
async stop(plan) {
|
|
561
|
-
await this.host.run(`docker compose -f ${plan.composeFile} --profile ${plan.profile} down`);
|
|
562
|
-
}
|
|
563
|
-
port(config) {
|
|
564
|
-
return typeof config.port === 'number' ? config.port : 18020;
|
|
565
|
-
}
|
|
566
|
-
profile(config) {
|
|
567
|
-
return typeof config.profile === 'string' ? config.profile : 'single';
|
|
568
|
-
}
|
|
569
|
-
}
|
|
570
|
-
/**
|
|
571
|
-
* The llama.cpp docker image the adapter pulls and runs. The engineFork
|
|
572
|
-
* platform config overrides the git source for a local build; the image stays
|
|
573
|
-
* the upstream ggml artifact.
|
|
574
|
-
*/
|
|
575
|
-
export const LLAMACPP_DOCKER_IMAGE = 'ghcr.io/ggml-org/llama.cpp';
|
|
576
|
-
/**
|
|
577
|
-
* llamaCppArtifact resolves the artifact a model key deploys. A key no entry
|
|
578
|
-
* names falls back to the 27B artifact, so a bare config always renders a plan.
|
|
579
|
-
*/
|
|
580
|
-
function llamaCppArtifact(modelKey) {
|
|
581
|
-
const artifact = modelKey !== undefined ? LLAMACPP_MODEL_ARTIFACTS[modelKey] : undefined;
|
|
582
|
-
return artifact ?? LLAMACPP_MODEL_ARTIFACTS['qwen3-27b'];
|
|
583
|
-
}
|
|
584
|
-
/**
|
|
585
|
-
* LlamaCppAdapter is the llama.cpp engine, both-capable like NInfer.
|
|
586
|
-
* `renderLaunch` renders a local `llama-server` process or a `docker run` of the
|
|
587
|
-
* ggml image from the same `LLAMACPP_PARAMETERS` surface; `deploy` builds the
|
|
588
|
-
* engine in the engine directory (or pulls the image), downloads the GGUF model
|
|
589
|
-
* into the model directory, and starts the engine; `healthCheck` polls the health
|
|
590
|
-
* URL; `stop` signals the tracked process and stops the named container.
|
|
591
|
-
*/
|
|
592
|
-
export class LlamaCppAdapter extends EngineAdapter {
|
|
593
|
-
id = 'llamacpp';
|
|
594
|
-
supports = { docker: true, local: true };
|
|
595
|
-
host;
|
|
596
|
-
children = new Map();
|
|
597
|
-
constructor(host = nodeLifecycleHost) {
|
|
598
|
-
super();
|
|
599
|
-
this.host = host;
|
|
600
|
-
}
|
|
601
|
-
parameters() {
|
|
602
|
-
return LLAMACPP_PARAMETERS;
|
|
603
|
-
}
|
|
604
|
-
modelRef(config) {
|
|
605
|
-
// The engine serves the model under the artifact's advertised id, so the
|
|
606
|
-
// registrar publishes that alias and a chat turn routes to it.
|
|
607
|
-
const artifact = llamaCppArtifact(llamaCppModelKey(config));
|
|
608
|
-
return { id: artifact.modelId, name: artifact.modelId };
|
|
609
|
-
}
|
|
610
|
-
launchSignature(config = {}) {
|
|
611
|
-
// The local plan runs the llama-server binary; the docker plan's host
|
|
612
|
-
// process is docker run of the ggml image, so the image name is the
|
|
613
|
-
// command-line marker the reserve probe matches on.
|
|
614
|
-
const port = this.port(config);
|
|
615
|
-
if (this.buildMode(config) === 'docker') {
|
|
616
|
-
return { binary: LLAMACPP_DOCKER_IMAGE, port };
|
|
617
|
-
}
|
|
618
|
-
return { binary: 'llama-server', port };
|
|
619
|
-
}
|
|
620
|
-
renderLaunch(config, _hw) {
|
|
621
|
-
const port = this.port(config);
|
|
622
|
-
const modelId = llamaCppModelKey(config);
|
|
623
|
-
const artifact = llamaCppArtifact(modelId);
|
|
624
|
-
// A model key that names a local GGUF file (a directory-scanned model)
|
|
625
|
-
// overrides the artifact map's file, so a file the user has on disk
|
|
626
|
-
// launches as-is.
|
|
627
|
-
const file = modelId.endsWith('.gguf') ? modelId : artifact.file;
|
|
628
|
-
const modelDirectory = typeof config.modelDirectory === 'string' ? config.modelDirectory : '';
|
|
629
|
-
const flags = flagsFromSpecs(LLAMACPP_PARAMETERS, config);
|
|
630
|
-
if (this.buildMode(config) === 'docker') {
|
|
631
|
-
const modelPath = modelDirectory === '' ? file : join(modelDirectory, file);
|
|
632
|
-
const containerModelPath = `/models/${basename(file)}`;
|
|
633
|
-
const modelMount = modelDirectory === ''
|
|
634
|
-
? `${file}:${containerModelPath}`
|
|
635
|
-
: `${dirname(modelPath)}:/models`;
|
|
636
|
-
const containerName = `llamacpp-${port}`;
|
|
637
|
-
const customArgs = customArgFlags(config);
|
|
638
|
-
return {
|
|
639
|
-
engineId: this.id,
|
|
640
|
-
kind: 'docker',
|
|
641
|
-
argv: [
|
|
642
|
-
'docker',
|
|
643
|
-
'run',
|
|
644
|
-
'--rm',
|
|
645
|
-
'--gpus',
|
|
646
|
-
'all',
|
|
647
|
-
'--name',
|
|
648
|
-
containerName,
|
|
649
|
-
'-p',
|
|
650
|
-
`127.0.0.1:${port}:8080`,
|
|
651
|
-
'-v',
|
|
652
|
-
modelMount,
|
|
653
|
-
LLAMACPP_DOCKER_IMAGE,
|
|
654
|
-
'--model',
|
|
655
|
-
containerModelPath,
|
|
656
|
-
...flags,
|
|
657
|
-
...customArgs,
|
|
658
|
-
],
|
|
659
|
-
env: {},
|
|
660
|
-
containerName,
|
|
661
|
-
modelId,
|
|
662
|
-
...(modelDirectory === '' ? {} : { modelDirectory }),
|
|
663
|
-
port,
|
|
664
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
665
|
-
};
|
|
666
|
-
}
|
|
667
|
-
const modelPath = modelDirectory === '' ? file : join(modelDirectory, file);
|
|
668
|
-
const customArgs = customArgFlags(config);
|
|
669
|
-
return {
|
|
670
|
-
engineId: this.id,
|
|
671
|
-
kind: 'process',
|
|
672
|
-
argv: [
|
|
673
|
-
'llama-server',
|
|
674
|
-
'--model',
|
|
675
|
-
modelPath,
|
|
676
|
-
'--host',
|
|
677
|
-
'127.0.0.1',
|
|
678
|
-
'--port',
|
|
679
|
-
String(port),
|
|
680
|
-
...flags,
|
|
681
|
-
...customArgs,
|
|
682
|
-
],
|
|
683
|
-
env: {},
|
|
684
|
-
modelId,
|
|
685
|
-
...(modelDirectory === '' ? {} : { modelDirectory }),
|
|
686
|
-
port,
|
|
687
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
688
|
-
};
|
|
689
|
-
}
|
|
690
|
-
async deploy(plan, platform) {
|
|
691
|
-
if (plan.kind === 'process') {
|
|
692
|
-
const engineDir = platform?.engineDirectory;
|
|
693
|
-
if (engineDir !== undefined) {
|
|
694
|
-
const fork = platform?.engineFork;
|
|
695
|
-
if (fork !== undefined) {
|
|
696
|
-
// Mirror the qwen-ninfer launcher: an existing checkout is updated in
|
|
697
|
-
// place with a best-effort fetch and a hard reset to the fork's
|
|
698
|
-
// master, then built as-is; a missing checkout is a fresh clone. The
|
|
699
|
-
// fetch is best-effort (a `; true` pins the branch to success) so an
|
|
700
|
-
// offline host still builds the checkout it has.
|
|
701
|
-
const sync = await this.host.run(`if [ -d ${engineDir}/.git ]; then ` +
|
|
702
|
-
`git -C ${engineDir} fetch ${fork} 2>/dev/null && git -C ${engineDir} reset --hard ${fork}/master 2>/dev/null; true; ` +
|
|
703
|
-
`else git clone ${fork} ${engineDir}; fi`);
|
|
704
|
-
if (!sync.ok) {
|
|
705
|
-
return { ok: false, detail: `git sync failed: ${sync.detail ?? 'unknown'}` };
|
|
706
|
-
}
|
|
707
|
-
}
|
|
708
|
-
const build = await this.host.run(`cmake -S ${engineDir} -B ${engineDir}/build -G Ninja -DCMAKE_BUILD_TYPE=Release && ` +
|
|
709
|
-
`cmake --build ${engineDir}/build -j`);
|
|
710
|
-
if (!build.ok) {
|
|
711
|
-
return { ok: false, detail: `build failed: ${build.detail ?? 'unknown'}` };
|
|
712
|
-
}
|
|
713
|
-
}
|
|
714
|
-
const download = await this.downloadModel(plan);
|
|
715
|
-
if (!download.ok)
|
|
716
|
-
return download;
|
|
717
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {}, engineDir);
|
|
718
|
-
this.children.set(this.planKey(plan), handle);
|
|
719
|
-
return { ok: true };
|
|
720
|
-
}
|
|
721
|
-
if (plan.kind === 'docker') {
|
|
722
|
-
const pull = await this.host.run(`docker pull ${LLAMACPP_DOCKER_IMAGE}`);
|
|
723
|
-
if (!pull.ok) {
|
|
724
|
-
return { ok: false, detail: `docker pull failed: ${pull.detail ?? 'unknown'}` };
|
|
725
|
-
}
|
|
726
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {});
|
|
727
|
-
this.children.set(this.planKey(plan), handle);
|
|
728
|
-
return { ok: true };
|
|
729
|
-
}
|
|
730
|
-
// llama.cpp renders process and docker plans; a compose plan is a no-op.
|
|
731
|
-
return { ok: true };
|
|
732
|
-
}
|
|
733
|
-
async healthCheck(plan, options) {
|
|
734
|
-
return pollHealth(this.host, plan, options);
|
|
735
|
-
}
|
|
736
|
-
async stop(plan) {
|
|
737
|
-
if (plan.kind === 'docker' && plan.containerName !== undefined) {
|
|
738
|
-
await this.host.run(`docker stop ${plan.containerName}`);
|
|
739
|
-
}
|
|
740
|
-
const handle = this.children.get(this.planKey(plan));
|
|
741
|
-
if (handle !== undefined) {
|
|
742
|
-
handle.kill();
|
|
743
|
-
this.children.delete(this.planKey(plan));
|
|
744
|
-
}
|
|
745
|
-
}
|
|
746
|
-
buildMode(config) {
|
|
747
|
-
return config.buildMode === 'docker' ? 'docker' : 'local';
|
|
748
|
-
}
|
|
749
|
-
port(config) {
|
|
750
|
-
return typeof config.port === 'number' ? config.port : 8080;
|
|
751
|
-
}
|
|
752
|
-
planKey(plan) {
|
|
753
|
-
return `${plan.engineId}:${plan.port}`;
|
|
754
|
-
}
|
|
755
|
-
async downloadModel(plan) {
|
|
756
|
-
const artifact = plan.modelId !== undefined ? LLAMACPP_MODEL_ARTIFACTS[plan.modelId] : undefined;
|
|
757
|
-
if (artifact === undefined)
|
|
758
|
-
return { ok: true };
|
|
759
|
-
const target = plan.modelDirectory ?? '.';
|
|
760
|
-
const download = await this.host.run(`hf download ${artifact.repo} ${artifact.file} --local-dir ${target}`);
|
|
761
|
-
if (!download.ok) {
|
|
762
|
-
return { ok: false, detail: `model download failed: ${download.detail ?? 'unknown'}` };
|
|
763
|
-
}
|
|
764
|
-
return { ok: true };
|
|
765
|
-
}
|
|
766
|
-
}
|
|
767
|
-
/**
|
|
768
|
-
* The Strata docker image the adapter pulls and runs. Strata is the
|
|
769
|
-
* placeholder engine: the image and the parameter data land when the engine
|
|
770
|
-
* is available, and the adapter renders a bare docker plan until then.
|
|
771
|
-
*/
|
|
772
|
-
export const STRATA_DOCKER_IMAGE = 'ghcr.io/strata/strata';
|
|
773
|
-
/**
|
|
774
|
-
* StrataAdapter is the placeholder docker-only engine. `renderLaunch` renders
|
|
775
|
-
* a `docker run` of the Strata image (no spec surface yet, so no flags);
|
|
776
|
-
* `deploy` pulls the image and starts the container; `healthCheck` polls the
|
|
777
|
-
* health URL; `stop` stops the named container. The parameter surface and the
|
|
778
|
-
* per-model data land with the engine, as data edits, never an adapter rewire.
|
|
779
|
-
*/
|
|
780
|
-
export class StrataAdapter extends EngineAdapter {
|
|
781
|
-
id = 'strata';
|
|
782
|
-
supports = { docker: true, local: false };
|
|
783
|
-
host;
|
|
784
|
-
children = new Map();
|
|
785
|
-
constructor(host = nodeLifecycleHost) {
|
|
786
|
-
super();
|
|
787
|
-
this.host = host;
|
|
788
|
-
}
|
|
789
|
-
parameters() {
|
|
790
|
-
return STRATA_PARAMETERS;
|
|
791
|
-
}
|
|
792
|
-
modelRef(_config) {
|
|
793
|
-
// Strata serves the model under the name the container passes, and the
|
|
794
|
-
// plugin tracks no per-model data for Strata yet, so the engine id is the
|
|
795
|
-
// stable entry the registrar publishes until that data lands.
|
|
796
|
-
return { id: 'strata', name: 'Strata' };
|
|
797
|
-
}
|
|
798
|
-
launchSignature(config = {}) {
|
|
799
|
-
// Strata is docker-only, so the signature is the image name the docker
|
|
800
|
-
// run carries plus the port the plan binds.
|
|
801
|
-
return { binary: STRATA_DOCKER_IMAGE, port: this.port(config) };
|
|
802
|
-
}
|
|
803
|
-
renderLaunch(config, _hw) {
|
|
804
|
-
const port = this.port(config);
|
|
805
|
-
const containerName = `strata-${port}`;
|
|
806
|
-
const customArgs = customArgFlags(config);
|
|
807
|
-
return {
|
|
808
|
-
engineId: this.id,
|
|
809
|
-
kind: 'docker',
|
|
810
|
-
argv: [
|
|
811
|
-
'docker',
|
|
812
|
-
'run',
|
|
813
|
-
'--rm',
|
|
814
|
-
'--gpus',
|
|
815
|
-
'all',
|
|
816
|
-
'--name',
|
|
817
|
-
containerName,
|
|
818
|
-
'-p',
|
|
819
|
-
`127.0.0.1:${port}:8080`,
|
|
820
|
-
STRATA_DOCKER_IMAGE,
|
|
821
|
-
...customArgs,
|
|
822
|
-
],
|
|
823
|
-
env: {},
|
|
824
|
-
containerName,
|
|
825
|
-
port,
|
|
826
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
827
|
-
};
|
|
828
|
-
}
|
|
829
|
-
async deploy(plan) {
|
|
830
|
-
const pull = await this.host.run(`docker pull ${STRATA_DOCKER_IMAGE}`);
|
|
831
|
-
if (!pull.ok) {
|
|
832
|
-
return { ok: false, detail: `docker pull failed: ${pull.detail ?? 'unknown'}` };
|
|
833
|
-
}
|
|
834
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {});
|
|
835
|
-
this.children.set(this.planKey(plan), handle);
|
|
836
|
-
return { ok: true };
|
|
837
|
-
}
|
|
838
|
-
async healthCheck(plan, options) {
|
|
839
|
-
return pollHealth(this.host, plan, options);
|
|
840
|
-
}
|
|
841
|
-
async stop(plan) {
|
|
842
|
-
if (plan.containerName !== undefined) {
|
|
843
|
-
await this.host.run(`docker stop ${plan.containerName}`);
|
|
844
|
-
}
|
|
845
|
-
const handle = this.children.get(this.planKey(plan));
|
|
846
|
-
if (handle !== undefined) {
|
|
847
|
-
handle.kill();
|
|
848
|
-
this.children.delete(this.planKey(plan));
|
|
849
|
-
}
|
|
850
|
-
}
|
|
851
|
-
port(config) {
|
|
852
|
-
return typeof config.port === 'number' ? config.port : 8080;
|
|
853
|
-
}
|
|
854
|
-
planKey(plan) {
|
|
855
|
-
return `${plan.engineId}:${plan.port}`;
|
|
856
|
-
}
|
|
857
|
-
}
|
|
858
313
|
//# sourceMappingURL=adapters.js.map
|