dsh-plugin-local-infer 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/docs/zh/README.md +10 -2
- package/dsh/client.js +669 -191
- package/package.json +6 -1
- package/packages/core/lib/adapter.d.ts +44 -2
- package/packages/core/lib/deps.d.ts +4 -3
- package/packages/core/lib/index.d.ts +1 -1
- package/packages/core/lib/registrar.d.ts +5 -5
- package/packages/core/lib/registrar.js +3 -3
- package/packages/core/package.json +1 -1
- package/packages/providers/lib/acquire.d.ts +5 -3
- package/packages/providers/lib/acquire.js +30 -9
- package/packages/providers/lib/adapters.d.ts +19 -209
- package/packages/providers/lib/adapters.js +13 -886
- package/packages/providers/lib/deps.d.ts +24 -2
- package/packages/providers/lib/deps.js +23 -4
- package/packages/providers/lib/engines/hyperqwenAdapter.d.ts +38 -0
- package/packages/providers/lib/engines/hyperqwenAdapter.js +105 -0
- package/packages/providers/lib/{infernix.js → engines/infernix.js} +1 -1
- package/packages/providers/lib/engines/infernixAdapter.d.ts +53 -0
- package/packages/providers/lib/engines/infernixAdapter.js +202 -0
- package/packages/providers/lib/engines/llamacppAdapter.d.ts +51 -0
- package/packages/providers/lib/engines/llamacppAdapter.js +232 -0
- package/packages/providers/lib/{ninfer.js → engines/ninfer.js} +1 -1
- package/packages/providers/lib/engines/ninferAdapter.d.ts +82 -0
- package/packages/providers/lib/engines/ninferAdapter.js +324 -0
- package/packages/providers/lib/engines/strata.d.ts +53 -0
- package/packages/providers/lib/engines/strata.js +197 -0
- package/packages/providers/lib/engines/strataAdapter.d.ts +98 -0
- package/packages/providers/lib/engines/strataAdapter.js +427 -0
- package/packages/providers/lib/engines.d.ts +5 -4
- package/packages/providers/lib/engines.js +10 -10
- package/packages/providers/lib/guidelines.js +1 -1
- package/packages/providers/lib/index.d.ts +14 -9
- package/packages/providers/lib/index.js +17 -7
- package/packages/providers/lib/model-defaults.d.ts +5 -4
- package/packages/providers/lib/model-defaults.js +22 -4
- package/packages/providers/lib/platform.js +7 -1
- package/packages/providers/package.json +1 -1
- package/packages/service/lib/index.d.ts +13 -4
- package/packages/service/lib/index.js +79 -5
- package/packages/service/lib/types.d.ts +5 -4
- package/packages/service/package.json +1 -1
- package/packages/tool/package.json +1 -1
- package/packages/ui/lib/config.d.ts +79 -0
- package/packages/ui/lib/config.js +235 -0
- package/packages/ui/lib/deps.d.ts +3 -3
- package/packages/ui/lib/ops/engine.d.ts +108 -0
- package/packages/ui/lib/ops/engine.js +493 -0
- package/packages/ui/lib/ops/hf.d.ts +27 -0
- package/packages/ui/lib/ops/hf.js +60 -0
- package/packages/ui/lib/ops/logs.d.ts +23 -0
- package/packages/ui/lib/ops/logs.js +58 -0
- package/packages/ui/lib/ops/model.d.ts +79 -0
- package/packages/ui/lib/ops/model.js +130 -0
- package/packages/ui/lib/request.d.ts +106 -0
- package/packages/ui/lib/request.js +80 -0
- package/packages/ui/lib/route.d.ts +88 -420
- package/packages/ui/lib/route.js +111 -1843
- package/packages/ui/lib/routes/chat-template.d.ts +14 -0
- package/packages/ui/lib/routes/chat-template.js +37 -0
- package/packages/ui/lib/routes/deps.d.ts +16 -0
- package/packages/ui/lib/routes/deps.js +27 -0
- package/packages/ui/lib/routes/download.d.ts +16 -0
- package/packages/ui/lib/routes/download.js +48 -0
- package/packages/ui/lib/routes/get.d.ts +18 -0
- package/packages/ui/lib/routes/get.js +57 -0
- package/packages/ui/lib/routes/install.d.ts +16 -0
- package/packages/ui/lib/routes/install.js +36 -0
- package/packages/ui/lib/routes/launch.d.ts +18 -0
- package/packages/ui/lib/routes/launch.js +50 -0
- package/packages/ui/lib/routes/provider-delete.d.ts +15 -0
- package/packages/ui/lib/routes/provider-delete.js +37 -0
- package/packages/ui/lib/routes/register.d.ts +16 -0
- package/packages/ui/lib/routes/register.js +28 -0
- package/packages/ui/lib/routes/save.d.ts +18 -0
- package/packages/ui/lib/routes/save.js +139 -0
- package/packages/ui/lib/routes/stop.d.ts +16 -0
- package/packages/ui/lib/routes/stop.js +29 -0
- package/packages/ui/lib/routes/summary.d.ts +15 -0
- package/packages/ui/lib/routes/summary.js +26 -0
- package/packages/ui/lib/routes/unregister.d.ts +14 -0
- package/packages/ui/lib/routes/unregister.js +26 -0
- package/packages/ui/lib/viewmodels/deps.d.ts +31 -0
- package/packages/ui/lib/viewmodels/deps.js +89 -0
- package/packages/ui/lib/viewmodels/depsHelpers.d.ts +28 -0
- package/packages/ui/lib/viewmodels/depsHelpers.js +33 -0
- package/packages/ui/lib/viewmodels/summary.d.ts +140 -0
- package/packages/ui/lib/viewmodels/summary.js +225 -0
- package/packages/ui/lib/viewmodels/summaryHelpers.d.ts +30 -0
- package/packages/ui/lib/viewmodels/summaryHelpers.js +42 -0
- package/packages/ui/package.json +1 -1
- package/packages/providers/lib/strata.d.ts +0 -15
- package/packages/providers/lib/strata.js +0 -14
- /package/packages/providers/lib/{hyperqwen.d.ts → engines/hyperqwen.d.ts} +0 -0
- /package/packages/providers/lib/{hyperqwen.js → engines/hyperqwen.js} +0 -0
- /package/packages/providers/lib/{infernix.d.ts → engines/infernix.d.ts} +0 -0
- /package/packages/providers/lib/{llamacpp.d.ts → engines/llamacpp.d.ts} +0 -0
- /package/packages/providers/lib/{llamacpp.js → engines/llamacpp.js} +0 -0
- /package/packages/providers/lib/{ninfer.d.ts → engines/ninfer.d.ts} +0 -0
|
@@ -1,61 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The
|
|
3
|
-
*
|
|
4
|
-
* `LaunchPlan
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
2
|
+
* The shared engine-adapter helpers: the spec-to-argv and spec-to-env
|
|
3
|
+
* translators the concrete adapters (the `engines/*Adapter.ts` files) compose
|
|
4
|
+
* into a `LaunchPlan`, the card's Custom Arguments translators, the
|
|
5
|
+
* `LifecycleHost` the deploy, health-check, and stop steps drive, and the
|
|
6
|
+
* health poll. The node implementation (`nodeLifecycleHost`) is what the
|
|
7
|
+
* service wires in on a live host.
|
|
8
8
|
* @module dsh-plugin-local-infer/providers/adapters
|
|
9
9
|
*/
|
|
10
10
|
import { exec, spawn as nodeSpawn } from 'node:child_process';
|
|
11
|
-
import { basename, dirname, join } from 'node:path';
|
|
12
|
-
import { EngineAdapter, formatLaunchCommand } from '../../core/lib/index.js';
|
|
13
|
-
import { syncAndRebuildEngine } from "./acquire.js";
|
|
14
|
-
import { NINFER_PARAMETERS } from "./ninfer.js";
|
|
15
|
-
import { HYPERQWEN_PARAMETERS } from "./hyperqwen.js";
|
|
16
|
-
import { LLAMACPP_MODEL_ARTIFACTS, LLAMACPP_PARAMETERS, llamaCppModelKey, } from "./llamacpp.js";
|
|
17
|
-
import { INFERNIX_MODEL_ARTIFACTS, INFERNIX_PARAMETERS, infernixArtifactFor, infernixDraftModelPath, infernixModelKey, } from "./infernix.js";
|
|
18
|
-
export { INFERNIX_MODEL_ARTIFACTS } from "./infernix.js";
|
|
19
|
-
import { STRATA_PARAMETERS } from "./strata.js";
|
|
20
|
-
/**
|
|
21
|
-
* The NInfer model artifacts, keyed by the model registry id. The 27B entry
|
|
22
|
-
* is the default the DEFAULT_MODELS list names; the 35B-A3B entries are the
|
|
23
|
-
* Qwen3.8 measured artifact (`qwen3-35b`) and the Qwen3.6 default
|
|
24
|
-
* (`qwen36-35b`).
|
|
25
|
-
*/
|
|
26
|
-
export const NINFER_MODEL_ARTIFACTS = {
|
|
27
|
-
'qwen3-27b': {
|
|
28
|
-
file: 'qwen3_8_27b_nvfp4.ninfer',
|
|
29
|
-
repo: 'neroued/Qwen3.8-27B-nvfp4-NInfer',
|
|
30
|
-
modelId: 'Qwen3.8-27B',
|
|
31
|
-
},
|
|
32
|
-
// The Swift 1.5 ablated 27B the local launcher bootstraps: the nvfp4-full
|
|
33
|
-
// DFlash2 artifact the qwen-ninfer script serves. It reuses the 27B model id
|
|
34
|
-
// the engine advertises, so a chat turn routes to the same alias.
|
|
35
|
-
'qwen3-27b-swift15': {
|
|
36
|
-
file: 'qwen3_8_27b_swift15abl_nvfp4full-dflash2.ninfer',
|
|
37
|
-
repo: 'kaushikvira/Qwen3.8-27B-swift15-uncensored-ajgazin-nvfp4full-dflash2-NInfer-v3',
|
|
38
|
-
modelId: 'Qwen3.8-27B',
|
|
39
|
-
},
|
|
40
|
-
'qwen3-35b': {
|
|
41
|
-
file: 'qwen3_8_35b_a3b.ninfer',
|
|
42
|
-
repo: 'Neroued/Qwen3.8-35B-A3B-NInfer',
|
|
43
|
-
modelId: 'Qwen3.8-35B-A3B',
|
|
44
|
-
},
|
|
45
|
-
'qwen36-35b': {
|
|
46
|
-
file: 'qwen3_6_35b_a3b.ninfer',
|
|
47
|
-
repo: 'neroued/Qwen3.6-35B-A3B-NInfer',
|
|
48
|
-
modelId: 'Qwen3.6-35B-A3B',
|
|
49
|
-
},
|
|
50
|
-
};
|
|
51
|
-
/**
|
|
52
|
-
* artifactFor resolves the artifact a model id deploys. An id no entry names
|
|
53
|
-
* falls back to the 27B artifact, so a bare config always renders a plan.
|
|
54
|
-
*/
|
|
55
|
-
function artifactFor(modelId) {
|
|
56
|
-
const artifact = modelId !== undefined ? NINFER_MODEL_ARTIFACTS[modelId] : undefined;
|
|
57
|
-
return artifact ?? NINFER_MODEL_ARTIFACTS['qwen3-27b'];
|
|
58
|
-
}
|
|
59
11
|
/**
|
|
60
12
|
* flagsFromSpecs turns a spec surface and a config into the CLI flag list: each
|
|
61
13
|
* spec's `toArgv` runs over the config value (falling back to the spec
|
|
@@ -313,7 +265,7 @@ export const nodeLifecycleHost = createNodeLifecycleHost();
|
|
|
313
265
|
* signals a pid a recycled process has taken over. A handle whose host cannot
|
|
314
266
|
* watch (onExit absent) stays tracked until stop removes it.
|
|
315
267
|
*/
|
|
316
|
-
function trackChild(children, key, handle) {
|
|
268
|
+
export function trackChild(children, key, handle) {
|
|
317
269
|
children.set(key, handle);
|
|
318
270
|
handle.onExit?.(() => {
|
|
319
271
|
// Compare identity before deleting: a newer launch that already replaced
|
|
@@ -348,839 +300,14 @@ export async function pollHealth(host, plan, options) {
|
|
|
348
300
|
detail: `health check did not return 200 within ${maxAttempts} attempts`,
|
|
349
301
|
};
|
|
350
302
|
}
|
|
351
|
-
|
|
303
|
+
/**
|
|
304
|
+
* delay waits `ms` milliseconds. The health polls use it between attempts; a
|
|
305
|
+
* poll interval of 0 skips the wait, which the tests use to run the loop
|
|
306
|
+
* without a real sleep.
|
|
307
|
+
*/
|
|
308
|
+
export function delay(ms) {
|
|
352
309
|
return new Promise(resolve => {
|
|
353
310
|
setTimeout(resolve, ms);
|
|
354
311
|
});
|
|
355
312
|
}
|
|
356
|
-
/**
|
|
357
|
-
* The NInfer docker image the adapter pulls and runs. The engineFork platform
|
|
358
|
-
* config overrides the git source for a local build; the image stays the
|
|
359
|
-
* upstream ghcr artifact.
|
|
360
|
-
*/
|
|
361
|
-
export const NINFER_DOCKER_IMAGE = 'ghcr.io/iamwavecut/ninfer-all';
|
|
362
|
-
/**
|
|
363
|
-
* NInferAdapter is the canonical both-capable engine. `renderLaunch` renders a
|
|
364
|
-
* local `ninfer-serve` process or a `docker run` of the ghcr image from the
|
|
365
|
-
* same spec surface; `deploy` builds the engine in the engine directory (or
|
|
366
|
-
* pulls the image), downloads the model into the model directory, and starts
|
|
367
|
-
* the engine; `healthCheck` polls the health URL; `stop` signals the tracked
|
|
368
|
-
* process and stops the named container.
|
|
369
|
-
*/
|
|
370
|
-
export class NInferAdapter extends EngineAdapter {
|
|
371
|
-
id = 'ninfer';
|
|
372
|
-
supports = { docker: true, local: true };
|
|
373
|
-
host;
|
|
374
|
-
children = new Map();
|
|
375
|
-
constructor(host = nodeLifecycleHost) {
|
|
376
|
-
super();
|
|
377
|
-
this.host = host;
|
|
378
|
-
}
|
|
379
|
-
parameters() {
|
|
380
|
-
return NINFER_PARAMETERS;
|
|
381
|
-
}
|
|
382
|
-
modelRef(config) {
|
|
383
|
-
// The id the engine advertises on its OpenAI endpoint is the artifact's
|
|
384
|
-
// --model-id alias, not the VramModel registry key the config names, so
|
|
385
|
-
// the registrar publishes the alias and a chat turn routes to it.
|
|
386
|
-
const artifact = artifactFor(typeof config.model === 'string' ? config.model : 'qwen3-27b');
|
|
387
|
-
return { id: artifact.modelId, name: artifact.modelId };
|
|
388
|
-
}
|
|
389
|
-
launchSignature(config = {}) {
|
|
390
|
-
// The local plan runs the ninfer-serve binary; the docker plan's host
|
|
391
|
-
// process is docker run of the ghcr image, so the image name is the
|
|
392
|
-
// command-line marker the reserve probe matches on.
|
|
393
|
-
const port = this.port(config);
|
|
394
|
-
if (this.buildMode(config) === 'docker') {
|
|
395
|
-
return { binary: NINFER_DOCKER_IMAGE, port };
|
|
396
|
-
}
|
|
397
|
-
return { binary: 'ninfer-serve', port };
|
|
398
|
-
}
|
|
399
|
-
renderLaunch(config, _hw) {
|
|
400
|
-
const port = this.port(config);
|
|
401
|
-
const modelId = typeof config.model === 'string' ? config.model : 'qwen3-27b';
|
|
402
|
-
const artifact = artifactFor(modelId);
|
|
403
|
-
// A model key that names a local artifact file (a directory-scanned
|
|
404
|
-
// model) overrides the artifact map's file, so a file the user has on
|
|
405
|
-
// disk launches as-is.
|
|
406
|
-
const file = modelId.endsWith('.ninfer') ? modelId : artifact.file;
|
|
407
|
-
const modelDirectory = typeof config.modelDirectory === 'string' ? config.modelDirectory : '';
|
|
408
|
-
const flags = flagsFromSpecs(NINFER_PARAMETERS, config);
|
|
409
|
-
if (this.buildMode(config) === 'docker') {
|
|
410
|
-
const modelPath = modelDirectory === '' ? file : join(modelDirectory, file);
|
|
411
|
-
const containerModelPath = `/models/${basename(file)}`;
|
|
412
|
-
const modelMount = modelDirectory === ''
|
|
413
|
-
? `${file}:${containerModelPath}`
|
|
414
|
-
: `${dirname(modelPath)}:/models`;
|
|
415
|
-
const containerName = `ninfer-${port}`;
|
|
416
|
-
const customArgs = customArgFlags(config);
|
|
417
|
-
return {
|
|
418
|
-
engineId: this.id,
|
|
419
|
-
kind: 'docker',
|
|
420
|
-
argv: [
|
|
421
|
-
'docker',
|
|
422
|
-
'run',
|
|
423
|
-
'--rm',
|
|
424
|
-
'--gpus',
|
|
425
|
-
'all',
|
|
426
|
-
'--name',
|
|
427
|
-
containerName,
|
|
428
|
-
'-p',
|
|
429
|
-
`127.0.0.1:${port}:8080`,
|
|
430
|
-
'--ulimit',
|
|
431
|
-
'memlock=-1',
|
|
432
|
-
'-v',
|
|
433
|
-
'ninfer-cache:/cache',
|
|
434
|
-
'-v',
|
|
435
|
-
modelMount,
|
|
436
|
-
NINFER_DOCKER_IMAGE,
|
|
437
|
-
'serve',
|
|
438
|
-
containerModelPath,
|
|
439
|
-
'--model-id',
|
|
440
|
-
artifact.modelId,
|
|
441
|
-
...flags,
|
|
442
|
-
...customArgs,
|
|
443
|
-
],
|
|
444
|
-
env: {},
|
|
445
|
-
containerName,
|
|
446
|
-
modelId,
|
|
447
|
-
...(modelDirectory === '' ? {} : { modelDirectory }),
|
|
448
|
-
port,
|
|
449
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
450
|
-
};
|
|
451
|
-
}
|
|
452
|
-
const modelPath = modelDirectory === '' ? file : join(modelDirectory, file);
|
|
453
|
-
const customArgs = customArgFlags(config);
|
|
454
|
-
return {
|
|
455
|
-
engineId: this.id,
|
|
456
|
-
kind: 'process',
|
|
457
|
-
argv: [
|
|
458
|
-
'ninfer-serve',
|
|
459
|
-
modelPath,
|
|
460
|
-
'--host',
|
|
461
|
-
'127.0.0.1',
|
|
462
|
-
'--port',
|
|
463
|
-
String(port),
|
|
464
|
-
'--model-id',
|
|
465
|
-
artifact.modelId,
|
|
466
|
-
...flags,
|
|
467
|
-
...customArgs,
|
|
468
|
-
],
|
|
469
|
-
env: {},
|
|
470
|
-
modelId,
|
|
471
|
-
...(modelDirectory === '' ? {} : { modelDirectory }),
|
|
472
|
-
port,
|
|
473
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
474
|
-
};
|
|
475
|
-
}
|
|
476
|
-
async deploy(plan, platform, onProgress, options) {
|
|
477
|
-
if (plan.kind === 'process') {
|
|
478
|
-
const engineDir = platform?.engineDirectory;
|
|
479
|
-
let buildLogs = '';
|
|
480
|
-
// Auto Rebuild off (the card's checkbox): the git sync (fetch, rebase,
|
|
481
|
-
// rebuild) is skipped and the checkout launches as-is.
|
|
482
|
-
if (engineDir !== undefined && options?.autoRebuild !== false) {
|
|
483
|
-
// The git sync is the checkout download: the card's button reads
|
|
484
|
-
// "Downloading..." for it, not "Building...".
|
|
485
|
-
onProgress?.('download');
|
|
486
|
-
// The conditional rebuild fetches the checkout, prices it against the
|
|
487
|
-
// remote branch, and rebuilds only when it is behind (or a fresh
|
|
488
|
-
// clone). The decision, the build command, and its output are captured
|
|
489
|
-
// into buildLogs and threaded to the card's Logs section.
|
|
490
|
-
const sync = await syncAndRebuildEngine(this.host, {
|
|
491
|
-
engineId: this.id,
|
|
492
|
-
directory: engineDir,
|
|
493
|
-
...(platform?.engineFork !== undefined ? { fork: platform.engineFork } : {}),
|
|
494
|
-
...(plan.argv?.[0] !== undefined ? { binary: plan.argv[0] } : {}),
|
|
495
|
-
...(options?.allowRunDeps !== undefined ? { allowRunDeps: options.allowRunDeps } : {}),
|
|
496
|
-
}, () => { });
|
|
497
|
-
buildLogs = sync.logs;
|
|
498
|
-
if (!sync.ok) {
|
|
499
|
-
return { ok: false, detail: sync.detail ?? 'unknown', logs: buildLogs };
|
|
500
|
-
}
|
|
501
|
-
// The build step reads as "Building..." only when a build actually
|
|
502
|
-
// ran (a fresh clone, or a rebase of a checkout that was behind).
|
|
503
|
-
if (sync.rebuilt) {
|
|
504
|
-
onProgress?.('build');
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
const withLogs = (result) => buildLogs !== '' ? { ...result, logs: buildLogs } : result;
|
|
508
|
-
onProgress?.('download');
|
|
509
|
-
const download = await this.downloadModel(plan);
|
|
510
|
-
if (!download.ok)
|
|
511
|
-
return withLogs(download);
|
|
512
|
-
// A chat template the user picked as an HF repo reference (the combo
|
|
513
|
-
// control's known choices) is not a path the engine can read, so the
|
|
514
|
-
// deploy step downloads it into the model directory and rewrites the
|
|
515
|
-
// --chat-template flag to the local file before the spawn.
|
|
516
|
-
onProgress?.('download');
|
|
517
|
-
const templateResult = await this.resolveChatTemplate(plan);
|
|
518
|
-
if (!templateResult.ok)
|
|
519
|
-
return withLogs(templateResult);
|
|
520
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {}, engineDir);
|
|
521
|
-
trackChild(this.children, this.planKey(plan), handle);
|
|
522
|
-
return withLogs({ ok: true, handle });
|
|
523
|
-
}
|
|
524
|
-
if (plan.kind === 'docker') {
|
|
525
|
-
// The image pull is the docker build step: the card's button reads
|
|
526
|
-
// "Building..." for it.
|
|
527
|
-
onProgress?.('build');
|
|
528
|
-
const pull = await this.host.run(`docker pull ${NINFER_DOCKER_IMAGE}`);
|
|
529
|
-
if (!pull.ok) {
|
|
530
|
-
return { ok: false, detail: `docker pull failed: ${pull.detail ?? 'unknown'}` };
|
|
531
|
-
}
|
|
532
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {});
|
|
533
|
-
trackChild(this.children, this.planKey(plan), handle);
|
|
534
|
-
return { ok: true, handle };
|
|
535
|
-
}
|
|
536
|
-
// NInfer renders process and docker plans; a compose plan is a no-op.
|
|
537
|
-
return { ok: true };
|
|
538
|
-
}
|
|
539
|
-
async healthCheck(plan, options) {
|
|
540
|
-
return pollHealth(this.host, plan, options);
|
|
541
|
-
}
|
|
542
|
-
async stop(plan) {
|
|
543
|
-
if (plan.kind === 'docker' && plan.containerName !== undefined) {
|
|
544
|
-
await this.host.run(`docker stop ${plan.containerName}`);
|
|
545
|
-
}
|
|
546
|
-
const handle = this.children.get(this.planKey(plan));
|
|
547
|
-
if (handle !== undefined) {
|
|
548
|
-
handle.kill();
|
|
549
|
-
this.children.delete(this.planKey(plan));
|
|
550
|
-
}
|
|
551
|
-
}
|
|
552
|
-
buildMode(config) {
|
|
553
|
-
return config.buildMode === 'docker' ? 'docker' : 'local';
|
|
554
|
-
}
|
|
555
|
-
port(config) {
|
|
556
|
-
return typeof config.port === 'number' ? config.port : 18020;
|
|
557
|
-
}
|
|
558
|
-
planKey(plan) {
|
|
559
|
-
return `${plan.engineId}:${plan.port}`;
|
|
560
|
-
}
|
|
561
|
-
async downloadModel(plan) {
|
|
562
|
-
const artifact = plan.modelId !== undefined ? NINFER_MODEL_ARTIFACTS[plan.modelId] : undefined;
|
|
563
|
-
if (artifact === undefined)
|
|
564
|
-
return { ok: true };
|
|
565
|
-
const target = plan.modelDirectory ?? '.';
|
|
566
|
-
const download = await this.host.run(`hf download ${artifact.repo} ${artifact.file} --local-dir ${target}`);
|
|
567
|
-
if (!download.ok) {
|
|
568
|
-
return { ok: false, detail: `model download failed: ${download.detail ?? 'unknown'}` };
|
|
569
|
-
}
|
|
570
|
-
return { ok: true };
|
|
571
|
-
}
|
|
572
|
-
/**
|
|
573
|
-
* resolveChatTemplate rewrites a --chat-template flag that carries an HF repo
|
|
574
|
-
* reference into the local file the engine reads. A value with a slash that
|
|
575
|
-
* is not an absolute path (the combo control's known choices) is a repo the
|
|
576
|
-
* deploy step downloads into the model directory; an already-local path is
|
|
577
|
-
* left as-is. The download is best-effort: a template that is already on
|
|
578
|
-
* disk is used without a re-fetch, and a missing hf CLI leaves the flag
|
|
579
|
-
* untouched so the engine reports the missing template itself.
|
|
580
|
-
*/
|
|
581
|
-
async resolveChatTemplate(plan) {
|
|
582
|
-
const argv = plan.argv;
|
|
583
|
-
if (argv === undefined)
|
|
584
|
-
return { ok: true };
|
|
585
|
-
const index = argv.indexOf('--chat-template');
|
|
586
|
-
if (index === -1)
|
|
587
|
-
return { ok: true };
|
|
588
|
-
const value = argv[index + 1];
|
|
589
|
-
if (value === undefined)
|
|
590
|
-
return { ok: true };
|
|
591
|
-
// An absolute path (a leading /) is already local; a value with no slash
|
|
592
|
-
// is a bare file name the engine resolves against its own directory.
|
|
593
|
-
if (value.startsWith('/') || !value.includes('/'))
|
|
594
|
-
return { ok: true };
|
|
595
|
-
// An HF repo reference: download the template file into the model
|
|
596
|
-
// directory and point the flag at the local copy.
|
|
597
|
-
const target = plan.modelDirectory ?? '.';
|
|
598
|
-
const localPath = `${target.endsWith('/') ? '' : `${target}/`}chat_template.jinja`;
|
|
599
|
-
// A template already on disk skips the re-fetch; the hf CLI is optional,
|
|
600
|
-
// so a download failure is reported but does not abort the launch.
|
|
601
|
-
const probe = await this.host.run(`[ -f ${localPath} ]`);
|
|
602
|
-
if (!probe.ok) {
|
|
603
|
-
const download = await this.host.run(`hf download ${value} chat_template.jinja --local-dir ${target}`);
|
|
604
|
-
if (!download.ok) {
|
|
605
|
-
return {
|
|
606
|
-
ok: false,
|
|
607
|
-
detail: `chat template download failed: ${download.detail ?? 'unknown'}`,
|
|
608
|
-
};
|
|
609
|
-
}
|
|
610
|
-
}
|
|
611
|
-
argv[index + 1] = localPath;
|
|
612
|
-
return { ok: true };
|
|
613
|
-
}
|
|
614
|
-
}
|
|
615
|
-
/**
|
|
616
|
-
* HyperQwenAdapter is the docker-compose engine. `renderLaunch` renders a
|
|
617
|
-
* compose plan (the compose file, the profile, and the `.env` the spec surface
|
|
618
|
-
* maps to); `deploy` runs `docker compose up`; `healthCheck` polls the health
|
|
619
|
-
* URL; `stop` runs `docker compose down`.
|
|
620
|
-
*/
|
|
621
|
-
export class HyperQwenAdapter extends EngineAdapter {
|
|
622
|
-
id = 'hyperqwen';
|
|
623
|
-
supports = { docker: true, local: false };
|
|
624
|
-
host;
|
|
625
|
-
constructor(host = nodeLifecycleHost) {
|
|
626
|
-
super();
|
|
627
|
-
this.host = host;
|
|
628
|
-
}
|
|
629
|
-
parameters() {
|
|
630
|
-
return HYPERQWEN_PARAMETERS;
|
|
631
|
-
}
|
|
632
|
-
modelRef(_config) {
|
|
633
|
-
// vLLM serves the model under the name the compose file passes, and the
|
|
634
|
-
// plugin tracks no per-model data for Hyper-Qwen yet, so the engine id is
|
|
635
|
-
// the stable entry the registrar publishes until that data lands.
|
|
636
|
-
return { id: 'hyperqwen', name: 'Hyper-Qwen' };
|
|
637
|
-
}
|
|
638
|
-
launchSignature(config = {}) {
|
|
639
|
-
// The compose host process is `docker compose ... up`, so the marker is
|
|
640
|
-
// the command, not an image. The toEnv keys (SPEC, CTX, KV, ...) name the
|
|
641
|
-
// .env the compose file reads, not the host process env, so they ride
|
|
642
|
-
// along as the env-key element the matcher checks when a process line
|
|
643
|
-
// carries env data.
|
|
644
|
-
const port = this.port(config);
|
|
645
|
-
const envKeys = Object.keys(envFromSpecs(HYPERQWEN_PARAMETERS, config));
|
|
646
|
-
return { binary: 'docker compose', port, envKeys };
|
|
647
|
-
}
|
|
648
|
-
renderLaunch(config, _hw) {
|
|
649
|
-
const port = this.port(config);
|
|
650
|
-
const profile = this.profile(config);
|
|
651
|
-
// The spec env and the card's Custom Arguments merge into one env map: the
|
|
652
|
-
// spec keys are the .env the compose file reads, and the custom rows are
|
|
653
|
-
// the user's own overrides appended after them.
|
|
654
|
-
const env = { ...envFromSpecs(HYPERQWEN_PARAMETERS, config), ...customArgEnv(config) };
|
|
655
|
-
const composeFile = typeof config.composeFile === 'string' ? config.composeFile : 'docker-compose.yml';
|
|
656
|
-
return {
|
|
657
|
-
engineId: this.id,
|
|
658
|
-
kind: 'compose',
|
|
659
|
-
env,
|
|
660
|
-
composeFile,
|
|
661
|
-
profile,
|
|
662
|
-
port,
|
|
663
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
664
|
-
};
|
|
665
|
-
}
|
|
666
|
-
async deploy(plan, platform, onProgress, _options) {
|
|
667
|
-
// The card's docker-tag field carries the compose arguments (the profile
|
|
668
|
-
// the up step selects); the deploy step applies it to the plan so the up
|
|
669
|
-
// line and the card's Logs section agree.
|
|
670
|
-
if (platform?.dockerTag !== undefined) {
|
|
671
|
-
plan.composeArgs = platform.dockerTag;
|
|
672
|
-
}
|
|
673
|
-
// The core formatter is the single source of the compose up line: the
|
|
674
|
-
// card's Logs section prints the same string this run gets.
|
|
675
|
-
const command = formatLaunchCommand(plan);
|
|
676
|
-
if (command === undefined) {
|
|
677
|
-
return { ok: false, detail: 'compose plan carries no command' };
|
|
678
|
-
}
|
|
679
|
-
// The compose up is the docker build step: the card's button reads
|
|
680
|
-
// "Building..." for it.
|
|
681
|
-
onProgress?.('build');
|
|
682
|
-
const up = await this.host.run(command);
|
|
683
|
-
if (!up.ok) {
|
|
684
|
-
return { ok: false, detail: `compose up failed: ${up.detail ?? 'unknown'}` };
|
|
685
|
-
}
|
|
686
|
-
return { ok: true };
|
|
687
|
-
}
|
|
688
|
-
async healthCheck(plan, options) {
|
|
689
|
-
return pollHealth(this.host, plan, options);
|
|
690
|
-
}
|
|
691
|
-
async stop(plan) {
|
|
692
|
-
// The down step brings down the containers the up step started, so it
|
|
693
|
-
// runs the same compose arguments (the profile the up selected); an
|
|
694
|
-
// absent composeArgs falls back to the engine's default profile.
|
|
695
|
-
const composeArgs = plan.composeArgs?.split(/\s+/).filter((token) => token !== '');
|
|
696
|
-
const args = composeArgs !== undefined && composeArgs.length > 0
|
|
697
|
-
? composeArgs.join(' ')
|
|
698
|
-
: `--profile ${plan.profile ?? 'single'}`;
|
|
699
|
-
await this.host.run(`docker compose -f ${plan.composeFile ?? 'docker-compose.yml'} ${args} down`);
|
|
700
|
-
}
|
|
701
|
-
port(config) {
|
|
702
|
-
return typeof config.port === 'number' ? config.port : 18020;
|
|
703
|
-
}
|
|
704
|
-
profile(config) {
|
|
705
|
-
return typeof config.profile === 'string' ? config.profile : 'single';
|
|
706
|
-
}
|
|
707
|
-
}
|
|
708
|
-
/**
|
|
709
|
-
* The llama.cpp docker image the adapter pulls and runs. The engineFork
|
|
710
|
-
* platform config overrides the git source for a local build; the image stays
|
|
711
|
-
* the upstream ggml artifact.
|
|
712
|
-
*/
|
|
713
|
-
export const LLAMACPP_DOCKER_IMAGE = 'ghcr.io/ggml-org/llama.cpp';
|
|
714
|
-
/**
|
|
715
|
-
* llamaCppArtifact resolves the artifact a model key deploys. A key no entry
|
|
716
|
-
* names falls back to the 27B artifact, so a bare config always renders a plan.
|
|
717
|
-
*/
|
|
718
|
-
function llamaCppArtifact(modelKey) {
|
|
719
|
-
const artifact = modelKey !== undefined ? LLAMACPP_MODEL_ARTIFACTS[modelKey] : undefined;
|
|
720
|
-
return artifact ?? LLAMACPP_MODEL_ARTIFACTS['qwen3-27b'];
|
|
721
|
-
}
|
|
722
|
-
/**
|
|
723
|
-
* LlamaCppAdapter is the llama.cpp engine, both-capable like NInfer.
|
|
724
|
-
* `renderLaunch` renders a local `llama-server` process or a `docker run` of the
|
|
725
|
-
* ggml image from the same `LLAMACPP_PARAMETERS` surface; `deploy` builds the
|
|
726
|
-
* engine in the engine directory (or pulls the image), downloads the GGUF model
|
|
727
|
-
* into the model directory, and starts the engine; `healthCheck` polls the health
|
|
728
|
-
* URL; `stop` signals the tracked process and stops the named container.
|
|
729
|
-
*/
|
|
730
|
-
export class LlamaCppAdapter extends EngineAdapter {
|
|
731
|
-
id = 'llamacpp';
|
|
732
|
-
supports = { docker: true, local: true };
|
|
733
|
-
host;
|
|
734
|
-
children = new Map();
|
|
735
|
-
constructor(host = nodeLifecycleHost) {
|
|
736
|
-
super();
|
|
737
|
-
this.host = host;
|
|
738
|
-
}
|
|
739
|
-
parameters() {
|
|
740
|
-
return LLAMACPP_PARAMETERS;
|
|
741
|
-
}
|
|
742
|
-
modelRef(config) {
|
|
743
|
-
// The engine serves the model under the artifact's advertised id, so the
|
|
744
|
-
// registrar publishes that alias and a chat turn routes to it.
|
|
745
|
-
const artifact = llamaCppArtifact(llamaCppModelKey(config));
|
|
746
|
-
return { id: artifact.modelId, name: artifact.modelId };
|
|
747
|
-
}
|
|
748
|
-
launchSignature(config = {}) {
|
|
749
|
-
// The local plan runs the llama-server binary; the docker plan's host
|
|
750
|
-
// process is docker run of the ggml image, so the image name is the
|
|
751
|
-
// command-line marker the reserve probe matches on.
|
|
752
|
-
const port = this.port(config);
|
|
753
|
-
if (this.buildMode(config) === 'docker') {
|
|
754
|
-
return { binary: LLAMACPP_DOCKER_IMAGE, port };
|
|
755
|
-
}
|
|
756
|
-
return { binary: 'llama-server', port };
|
|
757
|
-
}
|
|
758
|
-
renderLaunch(config, _hw) {
|
|
759
|
-
const port = this.port(config);
|
|
760
|
-
const modelId = llamaCppModelKey(config);
|
|
761
|
-
const artifact = llamaCppArtifact(modelId);
|
|
762
|
-
// A model key that names a local GGUF file (a directory-scanned model)
|
|
763
|
-
// overrides the artifact map's file, so a file the user has on disk
|
|
764
|
-
// launches as-is.
|
|
765
|
-
const file = modelId.endsWith('.gguf') ? modelId : artifact.file;
|
|
766
|
-
const modelDirectory = typeof config.modelDirectory === 'string' ? config.modelDirectory : '';
|
|
767
|
-
const flags = flagsFromSpecs(LLAMACPP_PARAMETERS, config);
|
|
768
|
-
if (this.buildMode(config) === 'docker') {
|
|
769
|
-
const modelPath = modelDirectory === '' ? file : join(modelDirectory, file);
|
|
770
|
-
const containerModelPath = `/models/${basename(file)}`;
|
|
771
|
-
const modelMount = modelDirectory === ''
|
|
772
|
-
? `${file}:${containerModelPath}`
|
|
773
|
-
: `${dirname(modelPath)}:/models`;
|
|
774
|
-
const containerName = `llamacpp-${port}`;
|
|
775
|
-
const customArgs = customArgFlags(config);
|
|
776
|
-
return {
|
|
777
|
-
engineId: this.id,
|
|
778
|
-
kind: 'docker',
|
|
779
|
-
argv: [
|
|
780
|
-
'docker',
|
|
781
|
-
'run',
|
|
782
|
-
'--rm',
|
|
783
|
-
'--gpus',
|
|
784
|
-
'all',
|
|
785
|
-
'--name',
|
|
786
|
-
containerName,
|
|
787
|
-
'-p',
|
|
788
|
-
`127.0.0.1:${port}:8080`,
|
|
789
|
-
'-v',
|
|
790
|
-
modelMount,
|
|
791
|
-
LLAMACPP_DOCKER_IMAGE,
|
|
792
|
-
'--model',
|
|
793
|
-
containerModelPath,
|
|
794
|
-
...flags,
|
|
795
|
-
...customArgs,
|
|
796
|
-
],
|
|
797
|
-
env: {},
|
|
798
|
-
containerName,
|
|
799
|
-
modelId,
|
|
800
|
-
...(modelDirectory === '' ? {} : { modelDirectory }),
|
|
801
|
-
port,
|
|
802
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
803
|
-
};
|
|
804
|
-
}
|
|
805
|
-
const modelPath = modelDirectory === '' ? file : join(modelDirectory, file);
|
|
806
|
-
const customArgs = customArgFlags(config);
|
|
807
|
-
return {
|
|
808
|
-
engineId: this.id,
|
|
809
|
-
kind: 'process',
|
|
810
|
-
argv: [
|
|
811
|
-
'llama-server',
|
|
812
|
-
'--model',
|
|
813
|
-
modelPath,
|
|
814
|
-
'--host',
|
|
815
|
-
'127.0.0.1',
|
|
816
|
-
'--port',
|
|
817
|
-
String(port),
|
|
818
|
-
...flags,
|
|
819
|
-
...customArgs,
|
|
820
|
-
],
|
|
821
|
-
env: {},
|
|
822
|
-
modelId,
|
|
823
|
-
...(modelDirectory === '' ? {} : { modelDirectory }),
|
|
824
|
-
port,
|
|
825
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
826
|
-
};
|
|
827
|
-
}
|
|
828
|
-
async deploy(plan, platform, onProgress, options) {
|
|
829
|
-
if (plan.kind === 'process') {
|
|
830
|
-
const engineDir = platform?.engineDirectory;
|
|
831
|
-
let buildLogs = '';
|
|
832
|
-
// Auto Rebuild off (the card's checkbox): the git sync (fetch, rebase,
|
|
833
|
-
// rebuild) is skipped and the checkout launches as-is.
|
|
834
|
-
if (engineDir !== undefined && options?.autoRebuild !== false) {
|
|
835
|
-
// The git sync is the checkout download: the card's button reads
|
|
836
|
-
// "Downloading..." for it, not "Building...".
|
|
837
|
-
onProgress?.('download');
|
|
838
|
-
// The conditional rebuild fetches the checkout, prices it against the
|
|
839
|
-
// remote branch, and rebuilds only when it is behind (or a fresh
|
|
840
|
-
// clone). The decision, the build command, and its output are captured
|
|
841
|
-
// into buildLogs and threaded to the card's Logs section.
|
|
842
|
-
const sync = await syncAndRebuildEngine(this.host, {
|
|
843
|
-
engineId: this.id,
|
|
844
|
-
directory: engineDir,
|
|
845
|
-
...(platform?.engineFork !== undefined ? { fork: platform.engineFork } : {}),
|
|
846
|
-
...(plan.argv?.[0] !== undefined ? { binary: plan.argv[0] } : {}),
|
|
847
|
-
...(options?.allowRunDeps !== undefined ? { allowRunDeps: options.allowRunDeps } : {}),
|
|
848
|
-
}, () => { });
|
|
849
|
-
buildLogs = sync.logs;
|
|
850
|
-
if (!sync.ok) {
|
|
851
|
-
return { ok: false, detail: sync.detail ?? 'unknown', logs: buildLogs };
|
|
852
|
-
}
|
|
853
|
-
// The build step reads as "Building..." only when a build actually
|
|
854
|
-
// ran (a fresh clone, or a rebase of a checkout that was behind).
|
|
855
|
-
if (sync.rebuilt) {
|
|
856
|
-
onProgress?.('build');
|
|
857
|
-
}
|
|
858
|
-
}
|
|
859
|
-
const withLogs = (result) => buildLogs !== '' ? { ...result, logs: buildLogs } : result;
|
|
860
|
-
onProgress?.('download');
|
|
861
|
-
const download = await this.downloadModel(plan);
|
|
862
|
-
if (!download.ok)
|
|
863
|
-
return withLogs(download);
|
|
864
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {}, engineDir);
|
|
865
|
-
trackChild(this.children, this.planKey(plan), handle);
|
|
866
|
-
return withLogs({ ok: true, handle });
|
|
867
|
-
}
|
|
868
|
-
if (plan.kind === 'docker') {
|
|
869
|
-
// The image pull is the docker build step: the card's button reads
|
|
870
|
-
// "Building..." for it.
|
|
871
|
-
onProgress?.('build');
|
|
872
|
-
const pull = await this.host.run(`docker pull ${LLAMACPP_DOCKER_IMAGE}`);
|
|
873
|
-
if (!pull.ok) {
|
|
874
|
-
return { ok: false, detail: `docker pull failed: ${pull.detail ?? 'unknown'}` };
|
|
875
|
-
}
|
|
876
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {});
|
|
877
|
-
trackChild(this.children, this.planKey(plan), handle);
|
|
878
|
-
return { ok: true, handle };
|
|
879
|
-
}
|
|
880
|
-
// llama.cpp renders process and docker plans; a compose plan is a no-op.
|
|
881
|
-
return { ok: true };
|
|
882
|
-
}
|
|
883
|
-
async healthCheck(plan, options) {
|
|
884
|
-
return pollHealth(this.host, plan, options);
|
|
885
|
-
}
|
|
886
|
-
async stop(plan) {
|
|
887
|
-
if (plan.kind === 'docker' && plan.containerName !== undefined) {
|
|
888
|
-
await this.host.run(`docker stop ${plan.containerName}`);
|
|
889
|
-
}
|
|
890
|
-
const handle = this.children.get(this.planKey(plan));
|
|
891
|
-
if (handle !== undefined) {
|
|
892
|
-
handle.kill();
|
|
893
|
-
this.children.delete(this.planKey(plan));
|
|
894
|
-
}
|
|
895
|
-
}
|
|
896
|
-
buildMode(config) {
|
|
897
|
-
return config.buildMode === 'docker' ? 'docker' : 'local';
|
|
898
|
-
}
|
|
899
|
-
port(config) {
|
|
900
|
-
return typeof config.port === 'number' ? config.port : 8080;
|
|
901
|
-
}
|
|
902
|
-
planKey(plan) {
|
|
903
|
-
return `${plan.engineId}:${plan.port}`;
|
|
904
|
-
}
|
|
905
|
-
async downloadModel(plan) {
|
|
906
|
-
const artifact = plan.modelId !== undefined ? LLAMACPP_MODEL_ARTIFACTS[plan.modelId] : undefined;
|
|
907
|
-
if (artifact === undefined)
|
|
908
|
-
return { ok: true };
|
|
909
|
-
const target = plan.modelDirectory ?? '.';
|
|
910
|
-
const download = await this.host.run(`hf download ${artifact.repo} ${artifact.file} --local-dir ${target}`);
|
|
911
|
-
if (!download.ok) {
|
|
912
|
-
return { ok: false, detail: `model download failed: ${download.detail ?? 'unknown'}` };
|
|
913
|
-
}
|
|
914
|
-
return { ok: true };
|
|
915
|
-
}
|
|
916
|
-
}
|
|
917
|
-
/**
|
|
918
|
-
* The Strata docker image the adapter pulls and runs. Strata is the
|
|
919
|
-
* placeholder engine: the image and the parameter data land when the engine
|
|
920
|
-
* is available, and the adapter renders a bare docker plan until then.
|
|
921
|
-
*/
|
|
922
|
-
export const STRATA_DOCKER_IMAGE = 'ghcr.io/strata/strata';
|
|
923
|
-
/**
|
|
924
|
-
* StrataAdapter is the placeholder docker-only engine. `renderLaunch` renders
|
|
925
|
-
* a `docker run` of the Strata image (no spec surface yet, so no flags);
|
|
926
|
-
* `deploy` pulls the image and starts the container; `healthCheck` polls the
|
|
927
|
-
* health URL; `stop` stops the named container. The parameter surface and the
|
|
928
|
-
* per-model data land with the engine, as data edits, never an adapter rewire.
|
|
929
|
-
*/
|
|
930
|
-
export class StrataAdapter extends EngineAdapter {
|
|
931
|
-
id = 'strata';
|
|
932
|
-
supports = { docker: true, local: false };
|
|
933
|
-
host;
|
|
934
|
-
children = new Map();
|
|
935
|
-
constructor(host = nodeLifecycleHost) {
|
|
936
|
-
super();
|
|
937
|
-
this.host = host;
|
|
938
|
-
}
|
|
939
|
-
parameters() {
|
|
940
|
-
return STRATA_PARAMETERS;
|
|
941
|
-
}
|
|
942
|
-
modelRef(_config) {
|
|
943
|
-
// Strata serves the model under the name the container passes, and the
|
|
944
|
-
// plugin tracks no per-model data for Strata yet, so the engine id is the
|
|
945
|
-
// stable entry the registrar publishes until that data lands.
|
|
946
|
-
return { id: 'strata', name: 'Strata' };
|
|
947
|
-
}
|
|
948
|
-
launchSignature(config = {}) {
|
|
949
|
-
// Strata is docker-only, so the signature is the image name the docker
|
|
950
|
-
// run carries plus the port the plan binds.
|
|
951
|
-
return { binary: STRATA_DOCKER_IMAGE, port: this.port(config) };
|
|
952
|
-
}
|
|
953
|
-
renderLaunch(config, _hw) {
|
|
954
|
-
const port = this.port(config);
|
|
955
|
-
const containerName = `strata-${port}`;
|
|
956
|
-
const customArgs = customArgFlags(config);
|
|
957
|
-
return {
|
|
958
|
-
engineId: this.id,
|
|
959
|
-
kind: 'docker',
|
|
960
|
-
argv: [
|
|
961
|
-
'docker',
|
|
962
|
-
'run',
|
|
963
|
-
'--rm',
|
|
964
|
-
'--gpus',
|
|
965
|
-
'all',
|
|
966
|
-
'--name',
|
|
967
|
-
containerName,
|
|
968
|
-
'-p',
|
|
969
|
-
`127.0.0.1:${port}:8080`,
|
|
970
|
-
STRATA_DOCKER_IMAGE,
|
|
971
|
-
...customArgs,
|
|
972
|
-
],
|
|
973
|
-
env: {},
|
|
974
|
-
containerName,
|
|
975
|
-
port,
|
|
976
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
977
|
-
};
|
|
978
|
-
}
|
|
979
|
-
async deploy(plan, _platform, onProgress, _options) {
|
|
980
|
-
// The image pull is the docker build step: the card's button reads
|
|
981
|
-
// "Building..." for it.
|
|
982
|
-
onProgress?.('build');
|
|
983
|
-
const pull = await this.host.run(`docker pull ${STRATA_DOCKER_IMAGE}`);
|
|
984
|
-
if (!pull.ok) {
|
|
985
|
-
return { ok: false, detail: `docker pull failed: ${pull.detail ?? 'unknown'}` };
|
|
986
|
-
}
|
|
987
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {});
|
|
988
|
-
trackChild(this.children, this.planKey(plan), handle);
|
|
989
|
-
return { ok: true, handle };
|
|
990
|
-
}
|
|
991
|
-
async healthCheck(plan, options) {
|
|
992
|
-
return pollHealth(this.host, plan, options);
|
|
993
|
-
}
|
|
994
|
-
async stop(plan) {
|
|
995
|
-
if (plan.containerName !== undefined) {
|
|
996
|
-
await this.host.run(`docker stop ${plan.containerName}`);
|
|
997
|
-
}
|
|
998
|
-
const handle = this.children.get(this.planKey(plan));
|
|
999
|
-
if (handle !== undefined) {
|
|
1000
|
-
handle.kill();
|
|
1001
|
-
this.children.delete(this.planKey(plan));
|
|
1002
|
-
}
|
|
1003
|
-
}
|
|
1004
|
-
port(config) {
|
|
1005
|
-
return typeof config.port === 'number' ? config.port : 8080;
|
|
1006
|
-
}
|
|
1007
|
-
planKey(plan) {
|
|
1008
|
-
return `${plan.engineId}:${plan.port}`;
|
|
1009
|
-
}
|
|
1010
|
-
}
|
|
1011
|
-
/**
|
|
1012
|
-
* InfernixAdapter is the local-only engine for Infernix, the C++/CUDA fork
|
|
1013
|
-
* of NInfer (https://github.com/wallawalla47/infernix) tuned for Qwen models
|
|
1014
|
-
* on a single RTX 50-series GPU. `renderLaunch` renders a local
|
|
1015
|
-
* `infernix-serve` process from the `INFERNIX_PARAMETERS` surface (the
|
|
1016
|
-
* Infernix-only fields, from `--ngram-volume` to the quality routes, spell
|
|
1017
|
-
* onto the argv only for this engine); `deploy` builds the engine in the
|
|
1018
|
-
* engine directory (the engine's own cmake preset), downloads the model
|
|
1019
|
-
* artifact (and the n-gram volume, for the multi-file Flash-Next repo) into
|
|
1020
|
-
* the model directory, and starts the engine; `healthCheck` polls the health
|
|
1021
|
-
* URL; `stop` signals the tracked process. The engine ships no docker image,
|
|
1022
|
-
* so the adapter supports the local build mode only.
|
|
1023
|
-
*/
|
|
1024
|
-
export class InfernixAdapter extends EngineAdapter {
|
|
1025
|
-
id = 'infernix';
|
|
1026
|
-
supports = { docker: false, local: true };
|
|
1027
|
-
host;
|
|
1028
|
-
children = new Map();
|
|
1029
|
-
constructor(host = nodeLifecycleHost) {
|
|
1030
|
-
super();
|
|
1031
|
-
this.host = host;
|
|
1032
|
-
}
|
|
1033
|
-
parameters() {
|
|
1034
|
-
return INFERNIX_PARAMETERS;
|
|
1035
|
-
}
|
|
1036
|
-
modelRef(config) {
|
|
1037
|
-
// The engine serves the model under the artifact's advertised id, so the
|
|
1038
|
-
// registrar publishes that alias and a chat turn routes to it. A model
|
|
1039
|
-
// key that names a local artifact file (a directory-scanned model)
|
|
1040
|
-
// reuses the 27B alias, the same fallback the NInfer adapter applies.
|
|
1041
|
-
const artifact = infernixArtifactFor(infernixModelKey(config));
|
|
1042
|
-
return { id: artifact.modelId, name: artifact.modelId };
|
|
1043
|
-
}
|
|
1044
|
-
launchSignature(config = {}) {
|
|
1045
|
-
// The local plan runs the infernix-serve binary (the .exe on Windows),
|
|
1046
|
-
// so the binary name is the command-line marker the reserve probe
|
|
1047
|
-
// matches on, and the port the plan binds rides along.
|
|
1048
|
-
return { binary: 'infernix-serve', port: this.port(config) };
|
|
1049
|
-
}
|
|
1050
|
-
renderLaunch(config, _hw) {
|
|
1051
|
-
const port = this.port(config);
|
|
1052
|
-
const modelKey = infernixModelKey(config);
|
|
1053
|
-
const artifact = infernixArtifactFor(modelKey);
|
|
1054
|
-
// A model key that names a local artifact file (a directory-scanned
|
|
1055
|
-
// model, a .ninfer or .infernix name) overrides the artifact map's file,
|
|
1056
|
-
// so a file the user has on disk launches as-is.
|
|
1057
|
-
const file = modelKey.endsWith('.ninfer') || modelKey.endsWith('.infernix')
|
|
1058
|
-
? modelKey
|
|
1059
|
-
: artifact.file;
|
|
1060
|
-
const modelDirectory = typeof config.modelDirectory === 'string' ? config.modelDirectory : '';
|
|
1061
|
-
const modelPath = modelDirectory === '' ? file : join(modelDirectory, file);
|
|
1062
|
-
// The engine binary is the .exe on a Windows host, where the published
|
|
1063
|
-
// release zip ships the server and its DLLs.
|
|
1064
|
-
const binary = process.platform === 'win32' ? 'infernix-serve.exe' : 'infernix-serve';
|
|
1065
|
-
const flags = flagsFromSpecs(INFERNIX_PARAMETERS, config);
|
|
1066
|
-
// The dflash2 backend always loads a drafter through --draft-model, so the
|
|
1067
|
-
// flag is spelled here from the resolved path (the selected DFlash2 preset,
|
|
1068
|
-
// or the Custom Draft model field) rather than by a spec toArgv. A config
|
|
1069
|
-
// that names no drafter path is the misconfiguration the dflash2
|
|
1070
|
-
// guideline names, so the flag is spelled only when a path resolves.
|
|
1071
|
-
if (config.spec === 'dflash2') {
|
|
1072
|
-
const draftPath = infernixDraftModelPath(config);
|
|
1073
|
-
if (draftPath !== null) {
|
|
1074
|
-
flags.push('--draft-model', draftPath);
|
|
1075
|
-
}
|
|
1076
|
-
}
|
|
1077
|
-
const customArgs = customArgFlags(config);
|
|
1078
|
-
return {
|
|
1079
|
-
engineId: this.id,
|
|
1080
|
-
kind: 'process',
|
|
1081
|
-
argv: [
|
|
1082
|
-
binary,
|
|
1083
|
-
modelPath,
|
|
1084
|
-
'--host',
|
|
1085
|
-
'127.0.0.1',
|
|
1086
|
-
'--port',
|
|
1087
|
-
String(port),
|
|
1088
|
-
'--model-id',
|
|
1089
|
-
artifact.modelId,
|
|
1090
|
-
...flags,
|
|
1091
|
-
...customArgs,
|
|
1092
|
-
],
|
|
1093
|
-
env: {},
|
|
1094
|
-
modelId: modelKey,
|
|
1095
|
-
...(modelDirectory === '' ? {} : { modelDirectory }),
|
|
1096
|
-
port,
|
|
1097
|
-
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
1098
|
-
};
|
|
1099
|
-
}
|
|
1100
|
-
async deploy(plan, platform, onProgress, options) {
|
|
1101
|
-
if (plan.kind !== 'process') {
|
|
1102
|
-
// Infernix renders process plans only; a docker or compose plan is a
|
|
1103
|
-
// no-op.
|
|
1104
|
-
return { ok: true };
|
|
1105
|
-
}
|
|
1106
|
-
const engineDir = platform?.engineDirectory;
|
|
1107
|
-
let buildLogs = '';
|
|
1108
|
-
// Auto Rebuild off (the card's checkbox): the git sync (fetch, rebase,
|
|
1109
|
-
// rebuild) is skipped and the checkout launches as-is.
|
|
1110
|
-
if (engineDir !== undefined && options?.autoRebuild !== false) {
|
|
1111
|
-
// The git sync is the checkout download: the card's button reads
|
|
1112
|
-
// "Downloading..." for it, not "Building...".
|
|
1113
|
-
onProgress?.('download');
|
|
1114
|
-
// The conditional rebuild fetches the checkout, prices it against the
|
|
1115
|
-
// remote branch, and rebuilds only when it is behind (or a fresh
|
|
1116
|
-
// clone). The engine's own quick start (the cmake preset, then the
|
|
1117
|
-
// preset's build) rides the acquisition-data build steps. The decision,
|
|
1118
|
-
// the build command, and its output are captured into buildLogs and
|
|
1119
|
-
// threaded to the card's Logs section.
|
|
1120
|
-
const sync = await syncAndRebuildEngine(this.host, {
|
|
1121
|
-
engineId: this.id,
|
|
1122
|
-
directory: engineDir,
|
|
1123
|
-
...(platform?.engineFork !== undefined ? { fork: platform.engineFork } : {}),
|
|
1124
|
-
...(plan.argv?.[0] !== undefined ? { binary: plan.argv[0] } : {}),
|
|
1125
|
-
...(options?.allowRunDeps !== undefined ? { allowRunDeps: options.allowRunDeps } : {}),
|
|
1126
|
-
}, () => { });
|
|
1127
|
-
buildLogs = sync.logs;
|
|
1128
|
-
if (!sync.ok) {
|
|
1129
|
-
return { ok: false, detail: sync.detail ?? 'unknown', logs: buildLogs };
|
|
1130
|
-
}
|
|
1131
|
-
// The build step reads as "Building..." only when a build actually
|
|
1132
|
-
// ran (a fresh clone, or a rebase of a checkout that was behind).
|
|
1133
|
-
if (sync.rebuilt) {
|
|
1134
|
-
onProgress?.('build');
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
const withLogs = (result) => buildLogs !== '' ? { ...result, logs: buildLogs } : result;
|
|
1138
|
-
onProgress?.('download');
|
|
1139
|
-
const download = await this.downloadModel(plan);
|
|
1140
|
-
if (!download.ok)
|
|
1141
|
-
return withLogs(download);
|
|
1142
|
-
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {}, engineDir);
|
|
1143
|
-
this.children.set(this.planKey(plan), handle);
|
|
1144
|
-
return withLogs({ ok: true, handle });
|
|
1145
|
-
}
|
|
1146
|
-
async healthCheck(plan, options) {
|
|
1147
|
-
return pollHealth(this.host, plan, options);
|
|
1148
|
-
}
|
|
1149
|
-
async stop(plan) {
|
|
1150
|
-
const handle = this.children.get(this.planKey(plan));
|
|
1151
|
-
if (handle !== undefined) {
|
|
1152
|
-
handle.kill();
|
|
1153
|
-
this.children.delete(this.planKey(plan));
|
|
1154
|
-
}
|
|
1155
|
-
}
|
|
1156
|
-
port(config) {
|
|
1157
|
-
// 8080 is the infernix-serve CLI default, so the form seeds it and the
|
|
1158
|
-
// adapter falls back to it when a config names no port.
|
|
1159
|
-
return typeof config.port === 'number' ? config.port : 8080;
|
|
1160
|
-
}
|
|
1161
|
-
planKey(plan) {
|
|
1162
|
-
return `${plan.engineId}:${plan.port}`;
|
|
1163
|
-
}
|
|
1164
|
-
async downloadModel(plan) {
|
|
1165
|
-
const artifact = plan.modelId !== undefined ? INFERNIX_MODEL_ARTIFACTS[plan.modelId] : undefined;
|
|
1166
|
-
// A model key that names a local artifact file carries no artifact entry,
|
|
1167
|
-
// so the file the user has on disk launches as-is without a download.
|
|
1168
|
-
if (artifact === undefined)
|
|
1169
|
-
return { ok: true };
|
|
1170
|
-
const target = plan.modelDirectory ?? '.';
|
|
1171
|
-
// A multi-file artifact (the Flash-Next shards) downloads the whole repo,
|
|
1172
|
-
// which carries the n-gram volume beside the shards, so the
|
|
1173
|
-
// --ngram-volume flag points at a file the same download left on disk.
|
|
1174
|
-
// A single-file artifact downloads the file alone.
|
|
1175
|
-
const repo = artifact.repo;
|
|
1176
|
-
const files = artifact.files ?? [];
|
|
1177
|
-
const download = files.length > 1
|
|
1178
|
-
? await this.host.run(`hf download ${repo} --local-dir ${target}`)
|
|
1179
|
-
: await this.host.run(`hf download ${repo} ${artifact.file} --local-dir ${target}`);
|
|
1180
|
-
if (!download.ok) {
|
|
1181
|
-
return { ok: false, detail: `model download failed: ${download.detail ?? 'unknown'}` };
|
|
1182
|
-
}
|
|
1183
|
-
return { ok: true };
|
|
1184
|
-
}
|
|
1185
|
-
}
|
|
1186
313
|
//# sourceMappingURL=adapters.js.map
|