dsh-plugin-local-infer 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/docs/zh/README.md +10 -2
- package/dsh/client.js +669 -191
- package/package.json +6 -1
- package/packages/core/lib/adapter.d.ts +44 -2
- package/packages/core/lib/deps.d.ts +4 -3
- package/packages/core/lib/index.d.ts +1 -1
- package/packages/core/lib/registrar.d.ts +5 -5
- package/packages/core/lib/registrar.js +3 -3
- package/packages/core/package.json +1 -1
- package/packages/providers/lib/acquire.d.ts +5 -3
- package/packages/providers/lib/acquire.js +30 -9
- package/packages/providers/lib/adapters.d.ts +19 -209
- package/packages/providers/lib/adapters.js +13 -886
- package/packages/providers/lib/deps.d.ts +24 -2
- package/packages/providers/lib/deps.js +23 -4
- package/packages/providers/lib/engines/hyperqwenAdapter.d.ts +38 -0
- package/packages/providers/lib/engines/hyperqwenAdapter.js +105 -0
- package/packages/providers/lib/{infernix.js → engines/infernix.js} +1 -1
- package/packages/providers/lib/engines/infernixAdapter.d.ts +53 -0
- package/packages/providers/lib/engines/infernixAdapter.js +202 -0
- package/packages/providers/lib/engines/llamacppAdapter.d.ts +51 -0
- package/packages/providers/lib/engines/llamacppAdapter.js +232 -0
- package/packages/providers/lib/{ninfer.js → engines/ninfer.js} +1 -1
- package/packages/providers/lib/engines/ninferAdapter.d.ts +82 -0
- package/packages/providers/lib/engines/ninferAdapter.js +324 -0
- package/packages/providers/lib/engines/strata.d.ts +53 -0
- package/packages/providers/lib/engines/strata.js +197 -0
- package/packages/providers/lib/engines/strataAdapter.d.ts +98 -0
- package/packages/providers/lib/engines/strataAdapter.js +427 -0
- package/packages/providers/lib/engines.d.ts +5 -4
- package/packages/providers/lib/engines.js +10 -10
- package/packages/providers/lib/guidelines.js +1 -1
- package/packages/providers/lib/index.d.ts +14 -9
- package/packages/providers/lib/index.js +17 -7
- package/packages/providers/lib/model-defaults.d.ts +5 -4
- package/packages/providers/lib/model-defaults.js +22 -4
- package/packages/providers/lib/platform.js +7 -1
- package/packages/providers/package.json +1 -1
- package/packages/service/lib/index.d.ts +13 -4
- package/packages/service/lib/index.js +79 -5
- package/packages/service/lib/types.d.ts +5 -4
- package/packages/service/package.json +1 -1
- package/packages/tool/package.json +1 -1
- package/packages/ui/lib/config.d.ts +79 -0
- package/packages/ui/lib/config.js +235 -0
- package/packages/ui/lib/deps.d.ts +3 -3
- package/packages/ui/lib/ops/engine.d.ts +108 -0
- package/packages/ui/lib/ops/engine.js +493 -0
- package/packages/ui/lib/ops/hf.d.ts +27 -0
- package/packages/ui/lib/ops/hf.js +60 -0
- package/packages/ui/lib/ops/logs.d.ts +23 -0
- package/packages/ui/lib/ops/logs.js +58 -0
- package/packages/ui/lib/ops/model.d.ts +79 -0
- package/packages/ui/lib/ops/model.js +130 -0
- package/packages/ui/lib/request.d.ts +106 -0
- package/packages/ui/lib/request.js +80 -0
- package/packages/ui/lib/route.d.ts +88 -420
- package/packages/ui/lib/route.js +111 -1843
- package/packages/ui/lib/routes/chat-template.d.ts +14 -0
- package/packages/ui/lib/routes/chat-template.js +37 -0
- package/packages/ui/lib/routes/deps.d.ts +16 -0
- package/packages/ui/lib/routes/deps.js +27 -0
- package/packages/ui/lib/routes/download.d.ts +16 -0
- package/packages/ui/lib/routes/download.js +48 -0
- package/packages/ui/lib/routes/get.d.ts +18 -0
- package/packages/ui/lib/routes/get.js +57 -0
- package/packages/ui/lib/routes/install.d.ts +16 -0
- package/packages/ui/lib/routes/install.js +36 -0
- package/packages/ui/lib/routes/launch.d.ts +18 -0
- package/packages/ui/lib/routes/launch.js +50 -0
- package/packages/ui/lib/routes/provider-delete.d.ts +15 -0
- package/packages/ui/lib/routes/provider-delete.js +37 -0
- package/packages/ui/lib/routes/register.d.ts +16 -0
- package/packages/ui/lib/routes/register.js +28 -0
- package/packages/ui/lib/routes/save.d.ts +18 -0
- package/packages/ui/lib/routes/save.js +139 -0
- package/packages/ui/lib/routes/stop.d.ts +16 -0
- package/packages/ui/lib/routes/stop.js +29 -0
- package/packages/ui/lib/routes/summary.d.ts +15 -0
- package/packages/ui/lib/routes/summary.js +26 -0
- package/packages/ui/lib/routes/unregister.d.ts +14 -0
- package/packages/ui/lib/routes/unregister.js +26 -0
- package/packages/ui/lib/viewmodels/deps.d.ts +31 -0
- package/packages/ui/lib/viewmodels/deps.js +89 -0
- package/packages/ui/lib/viewmodels/depsHelpers.d.ts +28 -0
- package/packages/ui/lib/viewmodels/depsHelpers.js +33 -0
- package/packages/ui/lib/viewmodels/summary.d.ts +140 -0
- package/packages/ui/lib/viewmodels/summary.js +225 -0
- package/packages/ui/lib/viewmodels/summaryHelpers.d.ts +30 -0
- package/packages/ui/lib/viewmodels/summaryHelpers.js +42 -0
- package/packages/ui/package.json +1 -1
- package/packages/providers/lib/strata.d.ts +0 -15
- package/packages/providers/lib/strata.js +0 -14
- /package/packages/providers/lib/{hyperqwen.d.ts → engines/hyperqwen.d.ts} +0 -0
- /package/packages/providers/lib/{hyperqwen.js → engines/hyperqwen.js} +0 -0
- /package/packages/providers/lib/{infernix.d.ts → engines/infernix.d.ts} +0 -0
- /package/packages/providers/lib/{llamacpp.d.ts → engines/llamacpp.d.ts} +0 -0
- /package/packages/providers/lib/{llamacpp.js → engines/llamacpp.js} +0 -0
- /package/packages/providers/lib/{ninfer.d.ts → engines/ninfer.d.ts} +0 -0
|
@@ -152,6 +152,26 @@ export declare const SYSTEM_DEPS: {
|
|
|
152
152
|
};
|
|
153
153
|
};
|
|
154
154
|
};
|
|
155
|
+
git: {
|
|
156
|
+
id: string;
|
|
157
|
+
purpose: string;
|
|
158
|
+
requiresSudo: true;
|
|
159
|
+
detect: (hw: HardwareProfile) => Promise<boolean>;
|
|
160
|
+
install: {
|
|
161
|
+
apt: {
|
|
162
|
+
commands: string[];
|
|
163
|
+
};
|
|
164
|
+
dnf: {
|
|
165
|
+
commands: string[];
|
|
166
|
+
};
|
|
167
|
+
pacman: {
|
|
168
|
+
commands: string[];
|
|
169
|
+
};
|
|
170
|
+
brew: {
|
|
171
|
+
commands: string[];
|
|
172
|
+
};
|
|
173
|
+
};
|
|
174
|
+
};
|
|
155
175
|
docker: {
|
|
156
176
|
id: string;
|
|
157
177
|
purpose: string;
|
|
@@ -219,8 +239,10 @@ export declare const SYSTEM_DEPS: {
|
|
|
219
239
|
* docker lists are the container runtime plus the GPU toolkit plus the hf
|
|
220
240
|
* CLI. Infernix builds through the engine's own cmake preset, so its local
|
|
221
241
|
* list is the same toolchain as NInfer's, and its docker list stays empty
|
|
222
|
-
* because the engine ships no container image. llama.cpp
|
|
223
|
-
*
|
|
242
|
+
* because the engine ships no container image. llama.cpp carries a thin
|
|
243
|
+
* cmake-plus-hf list. Strata carries one list per build mode: the source
|
|
244
|
+
* build needs git and curl, the binary install needs curl, and the docker
|
|
245
|
+
* mode needs the container runtime plus the GPU toolkit.
|
|
224
246
|
*/
|
|
225
247
|
export declare const ENGINE_DEP_PLANS: Record<string, EngineDepPlan>;
|
|
226
248
|
//# sourceMappingURL=deps.d.ts.map
|
|
@@ -136,6 +136,18 @@ export const SYSTEM_DEPS = {
|
|
|
136
136
|
},
|
|
137
137
|
},
|
|
138
138
|
},
|
|
139
|
+
git: {
|
|
140
|
+
id: 'git',
|
|
141
|
+
purpose: 'clones the engine source',
|
|
142
|
+
requiresSudo: true,
|
|
143
|
+
detect: probe('git', ['--version']),
|
|
144
|
+
install: {
|
|
145
|
+
apt: { commands: ['sudo apt install git'] },
|
|
146
|
+
dnf: { commands: ['sudo dnf install git'] },
|
|
147
|
+
pacman: { commands: ['sudo pacman -S --noconfirm git'] },
|
|
148
|
+
brew: { commands: ['brew install git'] },
|
|
149
|
+
},
|
|
150
|
+
},
|
|
139
151
|
docker: {
|
|
140
152
|
id: 'docker',
|
|
141
153
|
purpose: 'runs the engine in a container',
|
|
@@ -191,8 +203,10 @@ export const SYSTEM_DEPS = {
|
|
|
191
203
|
* docker lists are the container runtime plus the GPU toolkit plus the hf
|
|
192
204
|
* CLI. Infernix builds through the engine's own cmake preset, so its local
|
|
193
205
|
* list is the same toolchain as NInfer's, and its docker list stays empty
|
|
194
|
-
* because the engine ships no container image. llama.cpp
|
|
195
|
-
*
|
|
206
|
+
* because the engine ships no container image. llama.cpp carries a thin
|
|
207
|
+
* cmake-plus-hf list. Strata carries one list per build mode: the source
|
|
208
|
+
* build needs git and curl, the binary install needs curl, and the docker
|
|
209
|
+
* mode needs the container runtime plus the GPU toolkit.
|
|
196
210
|
*/
|
|
197
211
|
export const ENGINE_DEP_PLANS = {
|
|
198
212
|
ninfer: {
|
|
@@ -228,8 +242,13 @@ export const ENGINE_DEP_PLANS = {
|
|
|
228
242
|
docker: [],
|
|
229
243
|
},
|
|
230
244
|
strata: {
|
|
231
|
-
|
|
232
|
-
|
|
245
|
+
// The source build git-clones the repo and runs setup.sh, which downloads
|
|
246
|
+
// what it needs; the binary install is the one-line curl script; the docker
|
|
247
|
+
// mode needs the container runtime plus the GPU toolkit (the model downloads
|
|
248
|
+
// inside the container, so no hf CLI).
|
|
249
|
+
local: [SYSTEM_DEPS.git, SYSTEM_DEPS.curl],
|
|
250
|
+
docker: [SYSTEM_DEPS.docker, SYSTEM_DEPS.nvidiaContainerToolkit],
|
|
251
|
+
binary: [SYSTEM_DEPS.curl],
|
|
233
252
|
},
|
|
234
253
|
};
|
|
235
254
|
//# sourceMappingURL=deps.js.map
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Hyper-Qwen engine adapter. HyperQwenAdapter is the docker-compose
|
|
3
|
+
* engine. `renderLaunch` renders a compose plan (the compose file, the
|
|
4
|
+
* profile, and the `.env` the spec surface maps to); `deploy` runs
|
|
5
|
+
* `docker compose up`; `healthCheck` polls the health URL; `stop` runs
|
|
6
|
+
* `docker compose down`.
|
|
7
|
+
* @module dsh-plugin-local-infer/providers/engines/hyperqwenAdapter
|
|
8
|
+
*/
|
|
9
|
+
import { EngineAdapter } from '../../../core/lib/index.js';
|
|
10
|
+
import type { Config, DeployOptions, DeployResult, EngineLaunchSignature, EnginePlatformConfig, HealthCheckOptions, HealthResult, HardwareProfile, LaunchPlan, LaunchProgress, LifecycleHost, ParameterSpec } from '../../../core/lib/index.js';
|
|
11
|
+
/**
|
|
12
|
+
* HyperQwenAdapter is the docker-compose engine. `renderLaunch` renders a
|
|
13
|
+
* compose plan (the compose file, the profile, and the `.env` the spec surface
|
|
14
|
+
* maps to); `deploy` runs `docker compose up`; `healthCheck` polls the health
|
|
15
|
+
* URL; `stop` runs `docker compose down`.
|
|
16
|
+
*/
|
|
17
|
+
export declare class HyperQwenAdapter extends EngineAdapter {
|
|
18
|
+
readonly id = "hyperqwen";
|
|
19
|
+
readonly supports: {
|
|
20
|
+
docker: boolean;
|
|
21
|
+
local: boolean;
|
|
22
|
+
};
|
|
23
|
+
private readonly host;
|
|
24
|
+
constructor(host?: LifecycleHost);
|
|
25
|
+
parameters(): readonly ParameterSpec[];
|
|
26
|
+
modelRef(_config: Config): {
|
|
27
|
+
id: string;
|
|
28
|
+
name: string;
|
|
29
|
+
};
|
|
30
|
+
launchSignature(config?: Config): EngineLaunchSignature;
|
|
31
|
+
renderLaunch(config: Config, _hw: HardwareProfile): LaunchPlan;
|
|
32
|
+
deploy(plan: LaunchPlan, platform?: EnginePlatformConfig, onProgress?: LaunchProgress, _options?: DeployOptions): Promise<DeployResult>;
|
|
33
|
+
healthCheck(plan: LaunchPlan, options?: HealthCheckOptions): Promise<HealthResult>;
|
|
34
|
+
stop(plan: LaunchPlan): Promise<void>;
|
|
35
|
+
private port;
|
|
36
|
+
private profile;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=hyperqwenAdapter.d.ts.map
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Hyper-Qwen engine adapter. HyperQwenAdapter is the docker-compose
|
|
3
|
+
* engine. `renderLaunch` renders a compose plan (the compose file, the
|
|
4
|
+
* profile, and the `.env` the spec surface maps to); `deploy` runs
|
|
5
|
+
* `docker compose up`; `healthCheck` polls the health URL; `stop` runs
|
|
6
|
+
* `docker compose down`.
|
|
7
|
+
* @module dsh-plugin-local-infer/providers/engines/hyperqwenAdapter
|
|
8
|
+
*/
|
|
9
|
+
import { EngineAdapter, formatLaunchCommand } from '../../../core/lib/index.js';
|
|
10
|
+
import { customArgEnv, envFromSpecs, nodeLifecycleHost, pollHealth } from "../adapters.js";
|
|
11
|
+
import { HYPERQWEN_PARAMETERS } from "./hyperqwen.js";
|
|
12
|
+
/**
|
|
13
|
+
* HyperQwenAdapter is the docker-compose engine. `renderLaunch` renders a
|
|
14
|
+
* compose plan (the compose file, the profile, and the `.env` the spec surface
|
|
15
|
+
* maps to); `deploy` runs `docker compose up`; `healthCheck` polls the health
|
|
16
|
+
* URL; `stop` runs `docker compose down`.
|
|
17
|
+
*/
|
|
18
|
+
export class HyperQwenAdapter extends EngineAdapter {
|
|
19
|
+
id = 'hyperqwen';
|
|
20
|
+
supports = { docker: true, local: false };
|
|
21
|
+
host;
|
|
22
|
+
constructor(host = nodeLifecycleHost) {
|
|
23
|
+
super();
|
|
24
|
+
this.host = host;
|
|
25
|
+
}
|
|
26
|
+
parameters() {
|
|
27
|
+
return HYPERQWEN_PARAMETERS;
|
|
28
|
+
}
|
|
29
|
+
modelRef(_config) {
|
|
30
|
+
// vLLM serves the model under the name the compose file passes, and the
|
|
31
|
+
// plugin tracks no per-model data for Hyper-Qwen yet, so the engine id is
|
|
32
|
+
// the stable entry the registrar publishes until that data lands.
|
|
33
|
+
return { id: 'hyperqwen', name: 'Hyper-Qwen' };
|
|
34
|
+
}
|
|
35
|
+
launchSignature(config = {}) {
|
|
36
|
+
// The compose host process is `docker compose ... up`, so the marker is
|
|
37
|
+
// the command, not an image. The toEnv keys (SPEC, CTX, KV, ...) name the
|
|
38
|
+
// .env the compose file reads, not the host process env, so they ride
|
|
39
|
+
// along as the env-key element the matcher checks when a process line
|
|
40
|
+
// carries env data.
|
|
41
|
+
const port = this.port(config);
|
|
42
|
+
const envKeys = Object.keys(envFromSpecs(HYPERQWEN_PARAMETERS, config));
|
|
43
|
+
return { binary: 'docker compose', port, envKeys };
|
|
44
|
+
}
|
|
45
|
+
renderLaunch(config, _hw) {
|
|
46
|
+
const port = this.port(config);
|
|
47
|
+
const profile = this.profile(config);
|
|
48
|
+
// The spec env and the card's Custom Arguments merge into one env map: the
|
|
49
|
+
// spec keys are the .env the compose file reads, and the custom rows are
|
|
50
|
+
// the user's own overrides appended after them.
|
|
51
|
+
const env = { ...envFromSpecs(HYPERQWEN_PARAMETERS, config), ...customArgEnv(config) };
|
|
52
|
+
const composeFile = typeof config.composeFile === 'string' ? config.composeFile : 'docker-compose.yml';
|
|
53
|
+
return {
|
|
54
|
+
engineId: this.id,
|
|
55
|
+
kind: 'compose',
|
|
56
|
+
env,
|
|
57
|
+
composeFile,
|
|
58
|
+
profile,
|
|
59
|
+
port,
|
|
60
|
+
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
async deploy(plan, platform, onProgress, _options) {
|
|
64
|
+
// The card's docker-tag field carries the compose arguments (the profile
|
|
65
|
+
// the up step selects); the deploy step applies it to the plan so the up
|
|
66
|
+
// line and the card's Logs section agree.
|
|
67
|
+
if (platform?.dockerTag !== undefined) {
|
|
68
|
+
plan.composeArgs = platform.dockerTag;
|
|
69
|
+
}
|
|
70
|
+
// The core formatter is the single source of the compose up line: the
|
|
71
|
+
// card's Logs section prints the same string this run gets.
|
|
72
|
+
const command = formatLaunchCommand(plan);
|
|
73
|
+
if (command === undefined) {
|
|
74
|
+
return { ok: false, detail: 'compose plan carries no command' };
|
|
75
|
+
}
|
|
76
|
+
// The compose up is the docker build step: the card's button reads
|
|
77
|
+
// "Building..." for it.
|
|
78
|
+
onProgress?.('build');
|
|
79
|
+
const up = await this.host.run(command);
|
|
80
|
+
if (!up.ok) {
|
|
81
|
+
return { ok: false, detail: `compose up failed: ${up.detail ?? 'unknown'}` };
|
|
82
|
+
}
|
|
83
|
+
return { ok: true };
|
|
84
|
+
}
|
|
85
|
+
async healthCheck(plan, options) {
|
|
86
|
+
return pollHealth(this.host, plan, options);
|
|
87
|
+
}
|
|
88
|
+
async stop(plan) {
|
|
89
|
+
// The down step brings down the containers the up step started, so it
|
|
90
|
+
// runs the same compose arguments (the profile the up selected); an
|
|
91
|
+
// absent composeArgs falls back to the engine's default profile.
|
|
92
|
+
const composeArgs = plan.composeArgs?.split(/\s+/).filter((token) => token !== '');
|
|
93
|
+
const args = composeArgs !== undefined && composeArgs.length > 0
|
|
94
|
+
? composeArgs.join(' ')
|
|
95
|
+
: `--profile ${plan.profile ?? 'single'}`;
|
|
96
|
+
await this.host.run(`docker compose -f ${plan.composeFile ?? 'docker-compose.yml'} ${args} down`);
|
|
97
|
+
}
|
|
98
|
+
port(config) {
|
|
99
|
+
return typeof config.port === 'number' ? config.port : 18020;
|
|
100
|
+
}
|
|
101
|
+
profile(config) {
|
|
102
|
+
return typeof config.profile === 'string' ? config.profile : 'single';
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=hyperqwenAdapter.js.map
|
|
@@ -404,7 +404,7 @@ export const INFERNIX_PARAMETERS = [
|
|
|
404
404
|
{
|
|
405
405
|
key: 'lmHeadDraft',
|
|
406
406
|
label: 'LM-head draft',
|
|
407
|
-
description: 'Drafts from the small proposal head instead of the full output head, cutting the per-draft projection cost.
|
|
407
|
+
description: 'Drafts from the small proposal head instead of the full output head, cutting the per-draft projection cost.',
|
|
408
408
|
form: { kind: 'boolean' },
|
|
409
409
|
defaultValue: false,
|
|
410
410
|
group: 'Speculation',
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Infernix engine adapter. InfernixAdapter is the local-only engine for
|
|
3
|
+
* Infernix, the C++/CUDA fork of NInfer (https://github.com/wallawalla47/infernix)
|
|
4
|
+
* tuned for Qwen models on a single RTX 50-series GPU. `renderLaunch` renders a
|
|
5
|
+
* local `infernix-serve` process from the `INFERNIX_PARAMETERS` surface (the
|
|
6
|
+
* Infernix-only fields, from `--ngram-volume` to the quality routes, spell
|
|
7
|
+
* onto the argv only for this engine); `deploy` builds the engine in the
|
|
8
|
+
* engine directory (the engine's own cmake preset), downloads the model
|
|
9
|
+
* artifact (and the n-gram volume, for the multi-file Flash-Next repo) into
|
|
10
|
+
* the model directory, and starts the engine; `healthCheck` polls the health
|
|
11
|
+
* URL; `stop` signals the tracked process. The engine ships no docker image,
|
|
12
|
+
* so the adapter supports the local build mode only.
|
|
13
|
+
* @module dsh-plugin-local-infer/providers/engines/infernixAdapter
|
|
14
|
+
*/
|
|
15
|
+
import { EngineAdapter } from '../../../core/lib/index.js';
|
|
16
|
+
import type { Config, DeployOptions, DeployResult, EngineLaunchSignature, EnginePlatformConfig, HealthCheckOptions, HealthResult, HardwareProfile, LaunchPlan, LaunchProgress, LifecycleHost, ParameterSpec } from '../../../core/lib/index.js';
|
|
17
|
+
/**
|
|
18
|
+
* InfernixAdapter is the local-only engine for Infernix, the C++/CUDA fork
|
|
19
|
+
* of NInfer (https://github.com/wallawalla47/infernix) tuned for Qwen models
|
|
20
|
+
* on a single RTX 50-series GPU. `renderLaunch` renders a local
|
|
21
|
+
* `infernix-serve` process from the `INFERNIX_PARAMETERS` surface (the
|
|
22
|
+
* Infernix-only fields, from `--ngram-volume` to the quality routes, spell
|
|
23
|
+
* onto the argv only for this engine); `deploy` builds the engine in the
|
|
24
|
+
* engine directory (the engine's own cmake preset), downloads the model
|
|
25
|
+
* artifact (and the n-gram volume, for the multi-file Flash-Next repo) into
|
|
26
|
+
* the model directory, and starts the engine; `healthCheck` polls the health
|
|
27
|
+
* URL; `stop` signals the tracked process. The engine ships no docker image,
|
|
28
|
+
* so the adapter supports the local build mode only.
|
|
29
|
+
*/
|
|
30
|
+
export declare class InfernixAdapter extends EngineAdapter {
|
|
31
|
+
readonly id = "infernix";
|
|
32
|
+
readonly supports: {
|
|
33
|
+
docker: boolean;
|
|
34
|
+
local: boolean;
|
|
35
|
+
};
|
|
36
|
+
private readonly host;
|
|
37
|
+
private readonly children;
|
|
38
|
+
constructor(host?: LifecycleHost);
|
|
39
|
+
parameters(): readonly ParameterSpec[];
|
|
40
|
+
modelRef(config: Config): {
|
|
41
|
+
id: string;
|
|
42
|
+
name: string;
|
|
43
|
+
};
|
|
44
|
+
launchSignature(config?: Config): EngineLaunchSignature;
|
|
45
|
+
renderLaunch(config: Config, _hw: HardwareProfile): LaunchPlan;
|
|
46
|
+
deploy(plan: LaunchPlan, platform?: EnginePlatformConfig, onProgress?: LaunchProgress, options?: DeployOptions): Promise<DeployResult>;
|
|
47
|
+
healthCheck(plan: LaunchPlan, options?: HealthCheckOptions): Promise<HealthResult>;
|
|
48
|
+
stop(plan: LaunchPlan): Promise<void>;
|
|
49
|
+
private port;
|
|
50
|
+
private planKey;
|
|
51
|
+
private downloadModel;
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=infernixAdapter.d.ts.map
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Infernix engine adapter. InfernixAdapter is the local-only engine for
|
|
3
|
+
* Infernix, the C++/CUDA fork of NInfer (https://github.com/wallawalla47/infernix)
|
|
4
|
+
* tuned for Qwen models on a single RTX 50-series GPU. `renderLaunch` renders a
|
|
5
|
+
* local `infernix-serve` process from the `INFERNIX_PARAMETERS` surface (the
|
|
6
|
+
* Infernix-only fields, from `--ngram-volume` to the quality routes, spell
|
|
7
|
+
* onto the argv only for this engine); `deploy` builds the engine in the
|
|
8
|
+
* engine directory (the engine's own cmake preset), downloads the model
|
|
9
|
+
* artifact (and the n-gram volume, for the multi-file Flash-Next repo) into
|
|
10
|
+
* the model directory, and starts the engine; `healthCheck` polls the health
|
|
11
|
+
* URL; `stop` signals the tracked process. The engine ships no docker image,
|
|
12
|
+
* so the adapter supports the local build mode only.
|
|
13
|
+
* @module dsh-plugin-local-infer/providers/engines/infernixAdapter
|
|
14
|
+
*/
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
import { EngineAdapter } from '../../../core/lib/index.js';
|
|
17
|
+
import { customArgFlags, flagsFromSpecs, nodeLifecycleHost, pollHealth, } from "../adapters.js";
|
|
18
|
+
import { syncAndRebuildEngine } from "../acquire.js";
|
|
19
|
+
import { INFERNIX_MODEL_ARTIFACTS, INFERNIX_PARAMETERS, infernixArtifactFor, infernixDraftModelPath, infernixModelKey, } from "./infernix.js";
|
|
20
|
+
/**
|
|
21
|
+
* InfernixAdapter is the local-only engine for Infernix, the C++/CUDA fork
|
|
22
|
+
* of NInfer (https://github.com/wallawalla47/infernix) tuned for Qwen models
|
|
23
|
+
* on a single RTX 50-series GPU. `renderLaunch` renders a local
|
|
24
|
+
* `infernix-serve` process from the `INFERNIX_PARAMETERS` surface (the
|
|
25
|
+
* Infernix-only fields, from `--ngram-volume` to the quality routes, spell
|
|
26
|
+
* onto the argv only for this engine); `deploy` builds the engine in the
|
|
27
|
+
* engine directory (the engine's own cmake preset), downloads the model
|
|
28
|
+
* artifact (and the n-gram volume, for the multi-file Flash-Next repo) into
|
|
29
|
+
* the model directory, and starts the engine; `healthCheck` polls the health
|
|
30
|
+
* URL; `stop` signals the tracked process. The engine ships no docker image,
|
|
31
|
+
* so the adapter supports the local build mode only.
|
|
32
|
+
*/
|
|
33
|
+
export class InfernixAdapter extends EngineAdapter {
|
|
34
|
+
id = 'infernix';
|
|
35
|
+
supports = { docker: false, local: true };
|
|
36
|
+
host;
|
|
37
|
+
children = new Map();
|
|
38
|
+
constructor(host = nodeLifecycleHost) {
|
|
39
|
+
super();
|
|
40
|
+
this.host = host;
|
|
41
|
+
}
|
|
42
|
+
parameters() {
|
|
43
|
+
return INFERNIX_PARAMETERS;
|
|
44
|
+
}
|
|
45
|
+
modelRef(config) {
|
|
46
|
+
// The engine serves the model under the artifact's advertised id, so the
|
|
47
|
+
// registrar publishes that alias and a chat turn routes to it. A model
|
|
48
|
+
// key that names a local artifact file (a directory-scanned model)
|
|
49
|
+
// reuses the 27B alias, the same fallback the NInfer adapter applies.
|
|
50
|
+
const artifact = infernixArtifactFor(infernixModelKey(config));
|
|
51
|
+
return { id: artifact.modelId, name: artifact.modelId };
|
|
52
|
+
}
|
|
53
|
+
launchSignature(config = {}) {
|
|
54
|
+
// The local plan runs the infernix-serve binary (the .exe on Windows),
|
|
55
|
+
// so the binary name is the command-line marker the reserve probe
|
|
56
|
+
// matches on, and the port the plan binds rides along.
|
|
57
|
+
return { binary: 'infernix-serve', port: this.port(config) };
|
|
58
|
+
}
|
|
59
|
+
renderLaunch(config, _hw) {
|
|
60
|
+
const port = this.port(config);
|
|
61
|
+
const modelKey = infernixModelKey(config);
|
|
62
|
+
const artifact = infernixArtifactFor(modelKey);
|
|
63
|
+
// A model key that names a local artifact file (a directory-scanned
|
|
64
|
+
// model, a .ninfer or .infernix name) overrides the artifact map's file,
|
|
65
|
+
// so a file the user has on disk launches as-is.
|
|
66
|
+
const file = modelKey.endsWith('.ninfer') || modelKey.endsWith('.infernix')
|
|
67
|
+
? modelKey
|
|
68
|
+
: artifact.file;
|
|
69
|
+
const modelDirectory = typeof config.modelDirectory === 'string' ? config.modelDirectory : '';
|
|
70
|
+
const modelPath = modelDirectory === '' ? file : join(modelDirectory, file);
|
|
71
|
+
// The engine binary is the .exe on a Windows host, where the published
|
|
72
|
+
// release zip ships the server and its DLLs.
|
|
73
|
+
const binary = process.platform === 'win32' ? 'infernix-serve.exe' : 'infernix-serve';
|
|
74
|
+
const flags = flagsFromSpecs(INFERNIX_PARAMETERS, config);
|
|
75
|
+
// The dflash2 backend always loads a drafter through --draft-model, so the
|
|
76
|
+
// flag is spelled here from the resolved path (the selected DFlash2 preset,
|
|
77
|
+
// or the Custom Draft model field) rather than by a spec toArgv. A config
|
|
78
|
+
// that names no drafter path is the misconfiguration the dflash2
|
|
79
|
+
// guideline names, so the flag is spelled only when a path resolves.
|
|
80
|
+
if (config.spec === 'dflash2') {
|
|
81
|
+
const draftPath = infernixDraftModelPath(config);
|
|
82
|
+
if (draftPath !== null) {
|
|
83
|
+
flags.push('--draft-model', draftPath);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const customArgs = customArgFlags(config);
|
|
87
|
+
return {
|
|
88
|
+
engineId: this.id,
|
|
89
|
+
kind: 'process',
|
|
90
|
+
argv: [
|
|
91
|
+
binary,
|
|
92
|
+
modelPath,
|
|
93
|
+
'--host',
|
|
94
|
+
'127.0.0.1',
|
|
95
|
+
'--port',
|
|
96
|
+
String(port),
|
|
97
|
+
'--model-id',
|
|
98
|
+
artifact.modelId,
|
|
99
|
+
...flags,
|
|
100
|
+
...customArgs,
|
|
101
|
+
],
|
|
102
|
+
env: {},
|
|
103
|
+
modelId: modelKey,
|
|
104
|
+
...(modelDirectory === '' ? {} : { modelDirectory }),
|
|
105
|
+
port,
|
|
106
|
+
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
async deploy(plan, platform, onProgress, options) {
|
|
110
|
+
if (plan.kind !== 'process') {
|
|
111
|
+
// Infernix renders process plans only; a docker or compose plan is a
|
|
112
|
+
// no-op.
|
|
113
|
+
return { ok: true };
|
|
114
|
+
}
|
|
115
|
+
const engineDir = platform?.engineDirectory;
|
|
116
|
+
let buildLogs = '';
|
|
117
|
+
// Auto Rebuild off (the card's checkbox): the git sync (fetch, rebase,
|
|
118
|
+
// rebuild) is skipped and the checkout launches as-is.
|
|
119
|
+
if (engineDir !== undefined && options?.autoRebuild !== false) {
|
|
120
|
+
// The git sync is the checkout download: the card's button reads
|
|
121
|
+
// "Downloading..." for it, not "Building...".
|
|
122
|
+
onProgress?.('download');
|
|
123
|
+
// The conditional rebuild fetches the checkout, prices it against the
|
|
124
|
+
// remote branch, and rebuilds only when it is behind (or a fresh
|
|
125
|
+
// clone). The engine's own quick start (the cmake preset, then the
|
|
126
|
+
// preset's build) rides the acquisition-data build steps. The decision,
|
|
127
|
+
// the build command, and its output are captured into buildLogs and
|
|
128
|
+
// threaded to the card's Logs section.
|
|
129
|
+
const sync = await syncAndRebuildEngine(this.host, {
|
|
130
|
+
engineId: this.id,
|
|
131
|
+
directory: engineDir,
|
|
132
|
+
...(platform?.engineFork !== undefined ? { fork: platform.engineFork } : {}),
|
|
133
|
+
...(plan.argv?.[0] !== undefined ? { binary: plan.argv[0] } : {}),
|
|
134
|
+
...(options?.allowRunDeps !== undefined ? { allowRunDeps: options.allowRunDeps } : {}),
|
|
135
|
+
...(platform?.buildCommands !== undefined
|
|
136
|
+
? { buildCommands: platform.buildCommands }
|
|
137
|
+
: {}),
|
|
138
|
+
// Report the build phase the moment the build steps start, so the
|
|
139
|
+
// card's button reads "Building..." while the build is in flight.
|
|
140
|
+
onBuildStart: () => onProgress?.('build'),
|
|
141
|
+
}, () => { });
|
|
142
|
+
buildLogs = sync.logs;
|
|
143
|
+
if (!sync.ok) {
|
|
144
|
+
return { ok: false, detail: sync.detail ?? 'unknown', logs: buildLogs };
|
|
145
|
+
}
|
|
146
|
+
// The build output reads as a fresh Logs section: a "Building..."
|
|
147
|
+
// separator leads the build decision and output, the same line the
|
|
148
|
+
// card writes when the build step event lands.
|
|
149
|
+
if (sync.rebuilt && buildLogs !== '') {
|
|
150
|
+
buildLogs = `--- Building...\n${buildLogs}`;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
const withLogs = (result) => buildLogs !== '' ? { ...result, logs: buildLogs } : result;
|
|
154
|
+
onProgress?.('download');
|
|
155
|
+
const download = await this.downloadModel(plan);
|
|
156
|
+
if (!download.ok)
|
|
157
|
+
return withLogs(download);
|
|
158
|
+
const handle = this.host.spawn(plan.argv ?? [], plan.env ?? {}, engineDir);
|
|
159
|
+
this.children.set(this.planKey(plan), handle);
|
|
160
|
+
return withLogs({ ok: true, handle });
|
|
161
|
+
}
|
|
162
|
+
async healthCheck(plan, options) {
|
|
163
|
+
return pollHealth(this.host, plan, options);
|
|
164
|
+
}
|
|
165
|
+
async stop(plan) {
|
|
166
|
+
const handle = this.children.get(this.planKey(plan));
|
|
167
|
+
if (handle !== undefined) {
|
|
168
|
+
handle.kill();
|
|
169
|
+
this.children.delete(this.planKey(plan));
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
port(config) {
|
|
173
|
+
// 8080 is the infernix-serve CLI default, so the form seeds it and the
|
|
174
|
+
// adapter falls back to it when a config names no port.
|
|
175
|
+
return typeof config.port === 'number' ? config.port : 8080;
|
|
176
|
+
}
|
|
177
|
+
planKey(plan) {
|
|
178
|
+
return `${plan.engineId}:${plan.port}`;
|
|
179
|
+
}
|
|
180
|
+
async downloadModel(plan) {
|
|
181
|
+
const artifact = plan.modelId !== undefined ? INFERNIX_MODEL_ARTIFACTS[plan.modelId] : undefined;
|
|
182
|
+
// A model key that names a local artifact file carries no artifact entry,
|
|
183
|
+
// so the file the user has on disk launches as-is without a download.
|
|
184
|
+
if (artifact === undefined)
|
|
185
|
+
return { ok: true };
|
|
186
|
+
const target = plan.modelDirectory ?? '.';
|
|
187
|
+
// A multi-file artifact (the Flash-Next shards) downloads the whole repo,
|
|
188
|
+
// which carries the n-gram volume beside the shards, so the
|
|
189
|
+
// --ngram-volume flag points at a file the same download left on disk.
|
|
190
|
+
// A single-file artifact downloads the file alone.
|
|
191
|
+
const repo = artifact.repo;
|
|
192
|
+
const files = artifact.files ?? [];
|
|
193
|
+
const download = files.length > 1
|
|
194
|
+
? await this.host.run(`hf download ${repo} --local-dir ${target}`)
|
|
195
|
+
: await this.host.run(`hf download ${repo} ${artifact.file} --local-dir ${target}`);
|
|
196
|
+
if (!download.ok) {
|
|
197
|
+
return { ok: false, detail: `model download failed: ${download.detail ?? 'unknown'}` };
|
|
198
|
+
}
|
|
199
|
+
return { ok: true };
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
//# sourceMappingURL=infernixAdapter.js.map
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The llama.cpp engine adapter. LlamaCppAdapter is the llama.cpp engine,
|
|
3
|
+
* both-capable like NInfer. `renderLaunch` renders a local `llama-server`
|
|
4
|
+
* process or a `docker run` of the ggml image from the same
|
|
5
|
+
* `LLAMACPP_PARAMETERS` surface; `deploy` builds the engine in the engine
|
|
6
|
+
* directory (or pulls the image), downloads the GGUF model into the model
|
|
7
|
+
* directory, and starts the engine; `healthCheck` polls the health URL;
|
|
8
|
+
* `stop` signals the tracked process and stops the named container.
|
|
9
|
+
* @module dsh-plugin-local-infer/providers/engines/llamacppAdapter
|
|
10
|
+
*/
|
|
11
|
+
import { EngineAdapter } from '../../../core/lib/index.js';
|
|
12
|
+
import type { Config, DeployOptions, DeployResult, EngineLaunchSignature, EnginePlatformConfig, HealthCheckOptions, HealthResult, HardwareProfile, LaunchPlan, LaunchProgress, LifecycleHost, ParameterSpec } from '../../../core/lib/index.js';
|
|
13
|
+
/**
|
|
14
|
+
* The llama.cpp docker image the adapter pulls and runs. The engineFork
|
|
15
|
+
* platform config overrides the git source for a local build; the image stays
|
|
16
|
+
* the upstream ggml artifact.
|
|
17
|
+
*/
|
|
18
|
+
export declare const LLAMACPP_DOCKER_IMAGE = "ghcr.io/ggml-org/llama.cpp";
|
|
19
|
+
/**
|
|
20
|
+
* LlamaCppAdapter is the llama.cpp engine, both-capable like NInfer.
|
|
21
|
+
* `renderLaunch` renders a local `llama-server` process or a `docker run` of the
|
|
22
|
+
* ggml image from the same `LLAMACPP_PARAMETERS` surface; `deploy` builds the
|
|
23
|
+
* engine in the engine directory (or pulls the image), downloads the GGUF model
|
|
24
|
+
* into the model directory, and starts the engine; `healthCheck` polls the health
|
|
25
|
+
* URL; `stop` signals the tracked process and stops the named container.
|
|
26
|
+
*/
|
|
27
|
+
export declare class LlamaCppAdapter extends EngineAdapter {
|
|
28
|
+
readonly id = "llamacpp";
|
|
29
|
+
readonly supports: {
|
|
30
|
+
docker: boolean;
|
|
31
|
+
local: boolean;
|
|
32
|
+
};
|
|
33
|
+
private readonly host;
|
|
34
|
+
private readonly children;
|
|
35
|
+
constructor(host?: LifecycleHost);
|
|
36
|
+
parameters(): readonly ParameterSpec[];
|
|
37
|
+
modelRef(config: Config): {
|
|
38
|
+
id: string;
|
|
39
|
+
name: string;
|
|
40
|
+
};
|
|
41
|
+
launchSignature(config?: Config): EngineLaunchSignature;
|
|
42
|
+
renderLaunch(config: Config, _hw: HardwareProfile): LaunchPlan;
|
|
43
|
+
deploy(plan: LaunchPlan, platform?: EnginePlatformConfig, onProgress?: LaunchProgress, options?: DeployOptions): Promise<DeployResult>;
|
|
44
|
+
healthCheck(plan: LaunchPlan, options?: HealthCheckOptions): Promise<HealthResult>;
|
|
45
|
+
stop(plan: LaunchPlan): Promise<void>;
|
|
46
|
+
private buildMode;
|
|
47
|
+
private port;
|
|
48
|
+
private planKey;
|
|
49
|
+
private downloadModel;
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=llamacppAdapter.d.ts.map
|