@bendyline/gezel-inference 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog/chat-models/index.json +1 -1
- package/dist/catalog/chat-models/na/nanbeige4.2-3b-q4/manifest.json +1 -1
- package/dist/catalog/chat-models/ne/nemotron3.5-bolt-e4b-q4/manifest.json +142 -0
- package/dist/catalog/chat-models/ne/nemotron3.5-bolt-e4b-q4/versions/1.0.0/manifest.json +14 -0
- package/dist/catalog/chat-models/ne/nemotron3.5-bolt-e4b-q4/versions/1.0.1/manifest.json +14 -0
- package/dist/catalog/chat-models/raw-index.json +1 -1
- package/dist/catalog/image-models/an/animagine-xl-4/manifest.json +40 -0
- package/dist/catalog/image-models/an/animagine-xl-4/versions/4.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/an/animagine-xl-4/versions/4.0.1/manifest.json +16 -0
- package/dist/catalog/image-models/fl/flux-2-klein-4b-q4/manifest.json +44 -0
- package/dist/catalog/image-models/fl/flux-2-klein-4b-q4/versions/1.0.0/manifest.json +23 -0
- package/dist/catalog/image-models/fl/flux-2-klein-4b-q4/versions/1.0.1/manifest.json +29 -0
- package/dist/catalog/image-models/index.json +1 -0
- package/dist/catalog/image-models/ir/iris-3b-f32/manifest.json +41 -0
- package/dist/catalog/image-models/ir/iris-3b-f32/versions/1.0.0/manifest.json +19 -0
- package/dist/catalog/image-models/ir/iris-3b-f32/versions/1.0.1/manifest.json +25 -0
- package/dist/catalog/image-models/ir/iris-3b-f32/versions/1.0.2/manifest.json +25 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q4/manifest.json +42 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q4/versions/1.0.0/manifest.json +23 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q4/versions/1.0.1/manifest.json +29 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q6/manifest.json +42 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q6/versions/1.0.0/manifest.json +23 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q6/versions/1.0.1/manifest.json +29 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q8/manifest.json +42 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q8/versions/1.0.0/manifest.json +23 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q8/versions/1.0.1/manifest.json +29 -0
- package/dist/catalog/image-models/mi/ming-image-0.1-design-bf16/manifest.json +41 -0
- package/dist/catalog/image-models/mi/ming-image-0.1-design-bf16/versions/1.0.0/manifest.json +31 -0
- package/dist/catalog/image-models/mi/ming-image-0.1-design-bf16/versions/1.0.1/manifest.json +50 -0
- package/dist/catalog/image-models/mi/ming-image-0.1-design-bf16/versions/1.0.2/manifest.json +50 -0
- package/dist/catalog/image-models/mo/moss-image-sft-q4/manifest.json +40 -0
- package/dist/catalog/image-models/mo/moss-image-sft-q4/versions/1.0.0/manifest.json +37 -0
- package/dist/catalog/image-models/mo/moss-image-sft-q4/versions/1.0.1/manifest.json +37 -0
- package/dist/catalog/image-models/raw-index.json +1 -0
- package/dist/catalog/image-models/sd/sd-1.5/manifest.json +40 -0
- package/dist/catalog/image-models/sd/sd-1.5/versions/1.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/sd/sd-1.5/versions/1.0.1/manifest.json +16 -0
- package/dist/catalog/image-models/sd/sd-turbo/manifest.json +40 -0
- package/dist/catalog/image-models/sd/sd-turbo/versions/1.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/sd/sd-turbo/versions/1.0.1/manifest.json +16 -0
- package/dist/catalog/image-models/sd/sdxl-base-1.0/manifest.json +39 -0
- package/dist/catalog/image-models/sd/sdxl-base-1.0/versions/1.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/sd/sdxl-base-1.0/versions/1.0.1/manifest.json +16 -0
- package/dist/catalog/image-models/sd/sdxl-lightning-4step/manifest.json +40 -0
- package/dist/catalog/image-models/sd/sdxl-lightning-4step/versions/1.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/sd/sdxl-lightning-4step/versions/1.0.1/manifest.json +16 -0
- package/dist/catalog/image-models/sd/sdxl-turbo/manifest.json +41 -0
- package/dist/catalog/image-models/sd/sdxl-turbo/versions/1.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/sd/sdxl-turbo/versions/1.0.1/manifest.json +16 -0
- package/dist/catalog/provenance.json +2 -1
- package/dist/engines/native-file-manifest.json +129 -119
- package/dist/engines/native-manifest.d.ts +4 -4
- package/dist/engines/native-manifest.js +36 -36
- package/dist/engines/resolver.js +9 -0
- package/dist/http/openai-compat/engine-failure.js +7 -2
- package/dist/http/openai-compat/responses-streaming.d.ts +13 -0
- package/dist/http/openai-compat/responses-streaming.js +30 -0
- package/dist/http/openai-compat/streaming.d.ts +9 -0
- package/dist/http/openai-compat/streaming.js +49 -9
- package/dist/http/openai-compat/tool-call-screen.d.ts +28 -0
- package/dist/http/openai-compat/tool-call-screen.js +43 -0
- package/dist/http/routes/v1-chat.js +15 -1
- package/dist/http/routes/v1-images.d.ts +44 -0
- package/dist/http/routes/v1-images.js +370 -0
- package/dist/http/routes/v1-responses.js +13 -1
- package/dist/inference-service.d.ts +3 -0
- package/dist/inference-service.js +24 -2
- package/dist/kokoro-lexicon/manifest.json +6 -3
- package/dist/models/image-fit.d.ts +83 -0
- package/dist/models/image-fit.js +163 -0
- package/dist/models/weight-facts.d.ts +36 -0
- package/dist/models/weight-facts.js +171 -0
- package/dist/providers/ds4/build-provider.js +28 -21
- package/dist/providers/ds4/dspark.d.ts +23 -5
- package/dist/providers/ds4/dspark.js +36 -8
- package/dist/providers/image/img2img-support.d.ts +40 -0
- package/dist/providers/image/img2img-support.js +74 -0
- package/dist/providers/image/local-engine.d.ts +31 -0
- package/dist/providers/image/local-engine.js +47 -0
- package/dist/providers/image/local-factory.d.ts +95 -0
- package/dist/providers/image/local-factory.js +270 -0
- package/dist/providers/image/mock.d.ts +23 -0
- package/dist/providers/image/mock.js +151 -0
- package/dist/providers/image/pull-registry.d.ts +106 -0
- package/dist/providers/image/pull-registry.js +227 -0
- package/dist/providers/image/sd-cpp.d.ts +200 -0
- package/dist/providers/image/sd-cpp.js +845 -0
- package/dist/providers/image/types.d.ts +237 -0
- package/dist/providers/image/types.js +20 -0
- package/dist/providers/llama-cpp/backend-health.d.ts +7 -0
- package/dist/providers/llama-cpp/backend-health.js +41 -1
- package/dist/providers/mlx/provider.js +13 -3
- package/dist/providers/native/supervisor.d.ts +8 -0
- package/dist/providers/native/supervisor.js +52 -4
- package/dist/providers/project-macro-loop-bail.js +5 -3
- package/dist/providers/provider-lifecycle.d.ts +21 -0
- package/dist/providers/provider-lifecycle.js +83 -0
- package/dist/providers/retirement-gate.d.ts +20 -0
- package/dist/providers/retirement-gate.js +55 -0
- package/dist/providers/types.d.ts +7 -1
- package/dist/system/memory.js +6 -0
- package/dist/utils/image-meta.d.ts +12 -0
- package/dist/utils/image-meta.js +77 -0
- package/package.json +6 -6
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import type { ImageAspectRatio, ImageModelAuxiliaryRole, ImageQuality } from '@bendyline/gezel';
|
|
2
|
+
export type { ImageModelAuxiliaryRole } from '@bendyline/gezel';
|
|
3
|
+
/**
|
|
4
|
+
* ImageProvider — the service-side abstraction for local image generation.
|
|
5
|
+
*
|
|
6
|
+
* Mirrors the role `LLMProvider` plays for text. Deliberately simpler:
|
|
7
|
+
* image gen is one-shot rather than session-based, so there is no
|
|
8
|
+
* `createSession`/`sendAndWait` stream — just a single `generate` call
|
|
9
|
+
* that returns a PNG buffer plus metadata.
|
|
10
|
+
*
|
|
11
|
+
* Implementations today:
|
|
12
|
+
* - `MockImageProvider` — deterministic 1x1 PNG for tests and
|
|
13
|
+
* GEZEL_MOCK_PROVIDER=1 flows.
|
|
14
|
+
* - `StableDiffusionCppProvider` — talks to a `sd-server` binary over HTTP.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* One source image for editing / reference-driven generation, after
|
|
18
|
+
* the route has resolved any artifact paths to bytes. Provider sees
|
|
19
|
+
* a uniform `{ data: Buffer; mimeType }` shape.
|
|
20
|
+
*/
|
|
21
|
+
export interface ImageInputBytes {
|
|
22
|
+
data: Buffer;
|
|
23
|
+
mimeType: string;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Where a local generation is. sd-server loads weights lazily inside the
|
|
27
|
+
* first request after a launch, so `loading_model` happens mid-request, and
|
|
28
|
+
* its tiled VAE decode prints a progress bar of its own after sampling.
|
|
29
|
+
*/
|
|
30
|
+
export type ImageGenerationPhase = 'queued' | 'loading_model' | 'sampling' | 'decoding';
|
|
31
|
+
/**
|
|
32
|
+
* Progress info reported mid-generate by providers that can observe
|
|
33
|
+
* per-step advancement. Today only the local sd-cpp provider emits
|
|
34
|
+
* these (parsed from sd-server's `|==> | 3/20 - 18.20s/it` stdout).
|
|
35
|
+
* Cloud providers stay silent — their HTTP APIs are one-shot, with
|
|
36
|
+
* no streaming step counter to surface.
|
|
37
|
+
*
|
|
38
|
+
* `step`/`totalSteps` count within the phase: tensors loaded while
|
|
39
|
+
* `loading_model`, sampling steps while `sampling`, tiles while `decoding`.
|
|
40
|
+
* Both are 0 while `queued`. A consumer that only draws a sampling bar
|
|
41
|
+
* reads events whose `phase` is `sampling` or absent.
|
|
42
|
+
*/
|
|
43
|
+
export interface ImageGenerationProgress {
|
|
44
|
+
/** Unit the engine just completed in this phase (1-indexed). */
|
|
45
|
+
step: number;
|
|
46
|
+
/** Units in this phase. */
|
|
47
|
+
totalSteps: number;
|
|
48
|
+
/** Wall-clock seconds for the most recent step, when reported. */
|
|
49
|
+
secondsPerStep?: number;
|
|
50
|
+
/** Absent from providers that only report sampling. */
|
|
51
|
+
phase?: ImageGenerationPhase;
|
|
52
|
+
}
|
|
53
|
+
export interface ImageGenerationInput {
|
|
54
|
+
prompt: string;
|
|
55
|
+
negativePrompt?: string;
|
|
56
|
+
/** Manifest id of the model to use. Provider falls back to its default. */
|
|
57
|
+
model?: string;
|
|
58
|
+
width?: number;
|
|
59
|
+
height?: number;
|
|
60
|
+
/**
|
|
61
|
+
* The shape and size tier the caller asked for. Local engines render the
|
|
62
|
+
* pixels the route already resolved from these; cloud providers map them
|
|
63
|
+
* to their own aspect ratios and size tiers, which explicit pixels
|
|
64
|
+
* override.
|
|
65
|
+
*/
|
|
66
|
+
aspectRatio?: ImageAspectRatio;
|
|
67
|
+
quality?: ImageQuality;
|
|
68
|
+
steps?: number;
|
|
69
|
+
seed?: number;
|
|
70
|
+
/**
|
|
71
|
+
* Source images for image editing / reference-driven generation.
|
|
72
|
+
* Cloud providers accept multiple; sd-cpp's img2img path uses the
|
|
73
|
+
* first only (the others are dropped with a warning).
|
|
74
|
+
*/
|
|
75
|
+
inputImages?: ImageInputBytes[];
|
|
76
|
+
/** Img2img strength (0 = keep source, 1 = ignore source). sd-cpp only. */
|
|
77
|
+
strength?: number;
|
|
78
|
+
/**
|
|
79
|
+
* Optional callback fired as the engine advances through sampling
|
|
80
|
+
* steps. Today only the local sd-cpp provider emits these; cloud
|
|
81
|
+
* providers ignore the field. Errors thrown from the callback are
|
|
82
|
+
* swallowed so a buggy listener can't take down a generation.
|
|
83
|
+
*/
|
|
84
|
+
onProgress?: (progress: ImageGenerationProgress) => void;
|
|
85
|
+
/**
|
|
86
|
+
* Cancels the generation. A queued request leaves without touching the
|
|
87
|
+
* engine. sd-server cannot interrupt a render it has started, and keeps
|
|
88
|
+
* rendering after its client disconnects, so the local provider stops the
|
|
89
|
+
* engine process instead; the next generation pays a fresh weight load.
|
|
90
|
+
*/
|
|
91
|
+
signal?: AbortSignal;
|
|
92
|
+
}
|
|
93
|
+
/** The caller's signal aborted the generation. */
|
|
94
|
+
export declare class ImageGenerationCancelledError extends Error {
|
|
95
|
+
readonly code = "image_generation_cancelled";
|
|
96
|
+
constructor();
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* The engine could not fit the render into graphics memory. sd-server plans
|
|
100
|
+
* placement at launch and can misjudge free memory held by another engine;
|
|
101
|
+
* it then fails at the first sampling step with a bare "no results".
|
|
102
|
+
*/
|
|
103
|
+
export declare class ImageCapacityError extends Error {
|
|
104
|
+
readonly code = "image_capacity";
|
|
105
|
+
constructor(detail?: string);
|
|
106
|
+
}
|
|
107
|
+
export interface ImageGenerationMeta {
|
|
108
|
+
model: string;
|
|
109
|
+
seed: number;
|
|
110
|
+
steps: number;
|
|
111
|
+
widthPx: number;
|
|
112
|
+
heightPx: number;
|
|
113
|
+
durationMs: number;
|
|
114
|
+
/**
|
|
115
|
+
* Set when `inputImages` were supplied but the resolved model does not
|
|
116
|
+
* support img2img — the provider dropped the sources and generated from
|
|
117
|
+
* the prompt alone. Mirrors the core wire schema so the route can
|
|
118
|
+
* forward it untouched.
|
|
119
|
+
*/
|
|
120
|
+
img2imgSkippedReason?: string;
|
|
121
|
+
/** Provider limits or adjustments to the requested image settings. */
|
|
122
|
+
settingsNotes?: string[];
|
|
123
|
+
}
|
|
124
|
+
export interface ImageGenerationOutput {
|
|
125
|
+
png: Buffer;
|
|
126
|
+
meta: ImageGenerationMeta;
|
|
127
|
+
}
|
|
128
|
+
export interface InstalledImageModelInfo {
|
|
129
|
+
id: string;
|
|
130
|
+
name: string;
|
|
131
|
+
approxSizeBytes: number;
|
|
132
|
+
installedAt: string;
|
|
133
|
+
/** Loading shape recorded at pull time; absent on pre-capability installs. */
|
|
134
|
+
weightsKind?: 'checkpoint' | 'diffusion-model';
|
|
135
|
+
/** Explicit catalog capability recorded at pull time; absent → resolve via the ladder. */
|
|
136
|
+
supportsImg2Img?: boolean;
|
|
137
|
+
}
|
|
138
|
+
export type ImageModelPullEvent = {
|
|
139
|
+
type: 'progress';
|
|
140
|
+
bytesWritten: number;
|
|
141
|
+
totalBytes?: number;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Surfaced when the shared `downloadWithRetry` helper hit a transient
|
|
145
|
+
* network error and is about to try again. The UI can render
|
|
146
|
+
* "Connection lost — retrying in 4s (attempt 3/5)…" instead of a
|
|
147
|
+
* fatal error banner. `reason` is already a user-friendly sentence;
|
|
148
|
+
* don't wrap it.
|
|
149
|
+
*/
|
|
150
|
+
| {
|
|
151
|
+
type: 'retrying';
|
|
152
|
+
attempt: number;
|
|
153
|
+
maxAttempts: number;
|
|
154
|
+
delayMs: number;
|
|
155
|
+
reason: string;
|
|
156
|
+
} | {
|
|
157
|
+
type: 'error';
|
|
158
|
+
error: string;
|
|
159
|
+
} | {
|
|
160
|
+
type: 'done';
|
|
161
|
+
id: string;
|
|
162
|
+
};
|
|
163
|
+
export interface ImageModelAuxiliaryPullSpec {
|
|
164
|
+
role: ImageModelAuxiliaryRole;
|
|
165
|
+
downloadUrl: string;
|
|
166
|
+
sha256: string;
|
|
167
|
+
approxSizeBytes: number;
|
|
168
|
+
}
|
|
169
|
+
export interface ImageModelPullSpec {
|
|
170
|
+
downloadUrl: string;
|
|
171
|
+
sha256: string;
|
|
172
|
+
approxSizeBytes: number;
|
|
173
|
+
/** Human-readable name copied into the local metadata. */
|
|
174
|
+
name: string;
|
|
175
|
+
/**
|
|
176
|
+
* `checkpoint` — single-file (SD 1.x / SDXL / SD3.5 safetensors). The
|
|
177
|
+
* supervisor passes `--model <weights>` at sd-server launch.
|
|
178
|
+
* `diffusion-model` — unet-only (FLUX / SD3 GGUF); supervisor passes
|
|
179
|
+
* `--diffusion-model <weights>` plus one `--<role>` flag per aux file.
|
|
180
|
+
*/
|
|
181
|
+
weightsKind: 'checkpoint' | 'diffusion-model';
|
|
182
|
+
/**
|
|
183
|
+
* Explicit img2img capability from the catalog manifest, persisted
|
|
184
|
+
* into the installed metadata so local state stays self-describing
|
|
185
|
+
* even if the catalog entry later disappears. Absent when the catalog
|
|
186
|
+
* didn't declare it — resolution falls back to the assessment map +
|
|
187
|
+
* weights-kind default.
|
|
188
|
+
*/
|
|
189
|
+
supportsImg2Img?: boolean;
|
|
190
|
+
/** Auxiliary files (VAE, text encoders) downloaded alongside the unet. */
|
|
191
|
+
auxiliaryFiles: ImageModelAuxiliaryPullSpec[];
|
|
192
|
+
}
|
|
193
|
+
export interface ImageEngineHealth {
|
|
194
|
+
/**
|
|
195
|
+
* `ok` — the engine is reachable and ready to accept generation requests.
|
|
196
|
+
* `unreachable` — the configured engine URL is not responding (no
|
|
197
|
+
* process listening, network error, etc).
|
|
198
|
+
* `not-configured` — no engine binary or URL is wired up; the provider
|
|
199
|
+
* would fall back to a default loopback URL that nothing is going to
|
|
200
|
+
* answer. UI surfaces this distinctly so the user gets actionable
|
|
201
|
+
* guidance rather than a generic "unreachable" state.
|
|
202
|
+
*/
|
|
203
|
+
status: 'ok' | 'unreachable' | 'not-configured';
|
|
204
|
+
/** Resolved base URL the provider is pointed at, for diagnostic display. */
|
|
205
|
+
baseUrl: string;
|
|
206
|
+
/** Free-form reason when `status !== 'ok'`. */
|
|
207
|
+
error?: string;
|
|
208
|
+
}
|
|
209
|
+
export interface ImageProvider {
|
|
210
|
+
readonly name: string;
|
|
211
|
+
generate(input: ImageGenerationInput): Promise<ImageGenerationOutput>;
|
|
212
|
+
listInstalledModels(): Promise<InstalledImageModelInfo[]>;
|
|
213
|
+
/**
|
|
214
|
+
* The model a generate that names none will use. Callers that look up
|
|
215
|
+
* per-model defaults (the route's recommended steps) must ask this rather
|
|
216
|
+
* than guess, or the defaults of one model are applied to another.
|
|
217
|
+
* Providers without a local default omit it.
|
|
218
|
+
*/
|
|
219
|
+
resolveDefaultModelId?(): Promise<string | undefined>;
|
|
220
|
+
/**
|
|
221
|
+
* Stream the install of one catalog model. When `signal` is supplied
|
|
222
|
+
* and aborts mid-download, providers should leave any `.partial` file
|
|
223
|
+
* in place so a follow-up pull resumes via Range requests — disconnect
|
|
224
|
+
* ≠ delete. Cloud providers ignore the field.
|
|
225
|
+
*/
|
|
226
|
+
pullModel(id: string, spec: ImageModelPullSpec, signal?: AbortSignal): AsyncIterable<ImageModelPullEvent>;
|
|
227
|
+
deleteModel(id: string): Promise<void>;
|
|
228
|
+
/**
|
|
229
|
+
* Probe the underlying engine for reachability without doing real
|
|
230
|
+
* work. Implementations should be fast (sub-second) and never throw
|
|
231
|
+
* — return `unreachable` / `not-configured` instead. Called by the
|
|
232
|
+
* Settings UI to render an honest readiness pill.
|
|
233
|
+
*/
|
|
234
|
+
health(): Promise<ImageEngineHealth>;
|
|
235
|
+
/** Best-effort shutdown of any backing engine. Called when the service stops. */
|
|
236
|
+
shutdown?(): Promise<void>;
|
|
237
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** The caller's signal aborted the generation. */
|
|
2
|
+
export class ImageGenerationCancelledError extends Error {
|
|
3
|
+
code = 'image_generation_cancelled';
|
|
4
|
+
constructor() {
|
|
5
|
+
super('Image generation was cancelled');
|
|
6
|
+
this.name = 'ImageGenerationCancelledError';
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* The engine could not fit the render into graphics memory. sd-server plans
|
|
11
|
+
* placement at launch and can misjudge free memory held by another engine;
|
|
12
|
+
* it then fails at the first sampling step with a bare "no results".
|
|
13
|
+
*/
|
|
14
|
+
export class ImageCapacityError extends Error {
|
|
15
|
+
code = 'image_capacity';
|
|
16
|
+
constructor(detail) {
|
|
17
|
+
super(`Not enough graphics memory to render this image${detail ? ` (${detail})` : ''}. Close other models or try a smaller size.`);
|
|
18
|
+
this.name = 'ImageCapacityError';
|
|
19
|
+
}
|
|
20
|
+
}
|
|
@@ -11,6 +11,13 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import type { LlamaCppBackendHealthResponse } from '@bendyline/gezel';
|
|
13
13
|
import type { GpuUnused } from './gpu-startup-watch.js';
|
|
14
|
+
type MissingRuntimeState = NonNullable<LlamaCppBackendHealthResponse['missingRuntime']>;
|
|
15
|
+
/**
|
|
16
|
+
* Windows: whether this PC can load the runtime every engine imports, read from
|
|
17
|
+
* the first engine binary discovery found. One is enough: they all link the
|
|
18
|
+
* same runtime.
|
|
19
|
+
*/
|
|
20
|
+
export declare function readMissingRuntime(now?: number, platform?: NodeJS.Platform): MissingRuntimeState | null;
|
|
14
21
|
/** Record what the latest start of a build found; null clears it. */
|
|
15
22
|
export declare function noteGpuStartup(verdict: GpuUnused | null, now?: Date): void;
|
|
16
23
|
export declare function noteHostOutOfMemory(input: {
|
|
@@ -9,7 +9,44 @@
|
|
|
9
9
|
* Process-wide like the backend env it sits beside: every llama provider in
|
|
10
10
|
* this process launches the same build.
|
|
11
11
|
*/
|
|
12
|
-
import { readLlamaCudaBlocker } from '@bendyline/gezel/native';
|
|
12
|
+
import { missingMachineRuntime, readLlamaCudaBlocker, vcRedistributableUrl, } from '@bendyline/gezel/native';
|
|
13
|
+
/**
|
|
14
|
+
* Reading an engine's import tables takes tens of milliseconds of synchronous
|
|
15
|
+
* file reads, and the answer only changes when someone installs the runtime.
|
|
16
|
+
*/
|
|
17
|
+
const RUNTIME_CHECK_TTL_MS = 60_000;
|
|
18
|
+
let runtimeCheck = null;
|
|
19
|
+
/**
|
|
20
|
+
* Windows: whether this PC can load the runtime every engine imports, read from
|
|
21
|
+
* the first engine binary discovery found. One is enough: they all link the
|
|
22
|
+
* same runtime.
|
|
23
|
+
*/
|
|
24
|
+
export function readMissingRuntime(now = Date.now(), platform = process.platform) {
|
|
25
|
+
if (platform !== 'win32')
|
|
26
|
+
return null;
|
|
27
|
+
const binary = [
|
|
28
|
+
process.env.GEZEL_LLAMA_SERVER_BIN,
|
|
29
|
+
process.env.GEZEL_SD_SERVER_BIN,
|
|
30
|
+
process.env.GEZEL_WHISPER_SERVER_BIN,
|
|
31
|
+
].find((path) => Boolean(path));
|
|
32
|
+
if (!binary)
|
|
33
|
+
return null;
|
|
34
|
+
if (runtimeCheck?.binary === binary && now - runtimeCheck.at < RUNTIME_CHECK_TTL_MS) {
|
|
35
|
+
return runtimeCheck.value;
|
|
36
|
+
}
|
|
37
|
+
const failure = missingMachineRuntime(binary, process.env);
|
|
38
|
+
const arch = failure?.arch === 'arm64' || failure?.arch === 'x64' ? failure.arch : null;
|
|
39
|
+
const value = failure && arch
|
|
40
|
+
? {
|
|
41
|
+
arch,
|
|
42
|
+
dll: failure.dll ?? 'VCRUNTIME140.dll',
|
|
43
|
+
downloadUrl: vcRedistributableUrl(arch),
|
|
44
|
+
detail: `${binary}: ${failure.reason}`,
|
|
45
|
+
}
|
|
46
|
+
: null;
|
|
47
|
+
runtimeCheck = { binary, at: now, value };
|
|
48
|
+
return value;
|
|
49
|
+
}
|
|
13
50
|
let gpuUnused = null;
|
|
14
51
|
let outOfMemory = null;
|
|
15
52
|
/** Record what the latest start of a build found; null clears it. */
|
|
@@ -51,14 +88,17 @@ export function llamaBackendHealthView() {
|
|
|
51
88
|
// A verdict about a build the process has since moved off (a quarantine
|
|
52
89
|
// fallback, a lifted quarantine) no longer describes what runs.
|
|
53
90
|
const unused = gpuUnused && gpuUnused.backend === running ? gpuUnused : null;
|
|
91
|
+
const missingRuntime = readMissingRuntime();
|
|
54
92
|
return {
|
|
55
93
|
...(running ? { running } : {}),
|
|
56
94
|
...(cudaBlocker ? { cudaBlocker } : {}),
|
|
57
95
|
...(unused ? { gpuUnused: unused } : {}),
|
|
96
|
+
...(missingRuntime ? { missingRuntime } : {}),
|
|
58
97
|
...(outOfMemory ? { outOfMemory } : {}),
|
|
59
98
|
};
|
|
60
99
|
}
|
|
61
100
|
export function __resetLlamaBackendHealthForTest() {
|
|
62
101
|
gpuUnused = null;
|
|
63
102
|
outOfMemory = null;
|
|
103
|
+
runtimeCheck = null;
|
|
64
104
|
}
|
|
@@ -1497,11 +1497,21 @@ class MlxSession extends StreamingSessionBase {
|
|
|
1497
1497
|
// Paused mid-reply: the stream stops until the chat ahead is
|
|
1498
1498
|
// served. The reply so far stays on screen; the pill says why.
|
|
1499
1499
|
const detail = engineHoldLabelFor(queue);
|
|
1500
|
-
this.emitEnginePhase({
|
|
1500
|
+
this.emitEnginePhase({
|
|
1501
|
+
provider: 'mlx',
|
|
1502
|
+
phase: 'generating',
|
|
1503
|
+
detail,
|
|
1504
|
+
hold: 'yielding',
|
|
1505
|
+
});
|
|
1501
1506
|
return;
|
|
1502
1507
|
}
|
|
1503
1508
|
const detail = this.engineHold.begin(queue, opts?.queue?.onQueueWait);
|
|
1504
|
-
this.emitEnginePhase({
|
|
1509
|
+
this.emitEnginePhase({
|
|
1510
|
+
provider: 'mlx',
|
|
1511
|
+
phase: 'prefill',
|
|
1512
|
+
detail,
|
|
1513
|
+
hold: queue.state === 'paused' ? 'yielding' : 'behind',
|
|
1514
|
+
});
|
|
1505
1515
|
};
|
|
1506
1516
|
// Anchored at request dispatch: only work the engine did while THIS
|
|
1507
1517
|
// turn was already waiting can explain why it never got a first byte.
|
|
@@ -2666,7 +2676,7 @@ class MlxSession extends StreamingSessionBase {
|
|
|
2666
2676
|
// tool result makes the success unambiguous; the STOP
|
|
2667
2677
|
// instruction is the closing emphasis, not the framing.
|
|
2668
2678
|
projectMacroInterceptCount++;
|
|
2669
|
-
output = `✓ The project was successfully created on your earlier \`${startedProjectOrJobThisTurn.tool}\` call this turn — kickoff is complete and the lead is on it.\n\nOriginal result:\n${startedProjectOrJobThisTurn.firstResult}\n\nSTOP — do NOT emit any more tool calls this turn. Each project-start call creates a real project, lead, and kickoff task; repeating the macro would create duplicates. END YOUR TURN NOW with a one-sentence summary to the user (e.g. "Project is
|
|
2679
|
+
output = `✓ The project was successfully created on your earlier \`${startedProjectOrJobThisTurn.tool}\` call this turn — kickoff is complete and the lead is on it.\n\nOriginal result:\n${startedProjectOrJobThisTurn.firstResult}\n\nSTOP — do NOT emit any more tool calls this turn. Each project-start call creates a real project, lead, and kickoff task; repeating the macro would create duplicates. END YOUR TURN NOW with a one-sentence summary to the user (e.g. "Project is underway — the crew lead is on it"). The lead handles the work from here.`;
|
|
2670
2680
|
if (projectMacroInterceptCount >= PROJECT_MACRO_INTERCEPT_CAP) {
|
|
2671
2681
|
forceProjectMacroBail = {
|
|
2672
2682
|
closingText: deriveProjectMacroClosing(startedProjectOrJobThisTurn.firstResult),
|
|
@@ -289,6 +289,14 @@ export declare class NativeEngineSupervisor {
|
|
|
289
289
|
private currentOomKills;
|
|
290
290
|
private readonly logListeners;
|
|
291
291
|
private recentStarts;
|
|
292
|
+
/**
|
|
293
|
+
* The {@link recentStarts} entry for the child running now. A child that
|
|
294
|
+
* became ready and was then stopped on purpose (idle, memory pressure, a
|
|
295
|
+
* model swap) was not a crash, so its start is returned to the budget.
|
|
296
|
+
* Otherwise an engine asked to swap models or yield memory three times in
|
|
297
|
+
* a minute would refuse the fourth start as a crash loop.
|
|
298
|
+
*/
|
|
299
|
+
private currentStartAt;
|
|
292
300
|
private lastUsedAt;
|
|
293
301
|
/**
|
|
294
302
|
* The same mark on the awake clock. Every idle/pressure DECISION reads this
|
|
@@ -25,7 +25,7 @@ import { readFileSync } from 'node:fs';
|
|
|
25
25
|
import { basename } from 'node:path';
|
|
26
26
|
import { awakeNow, createLogger } from '@bendyline/gezel';
|
|
27
27
|
import { listProcessSnapshots } from '@bendyline/gezel-client/node';
|
|
28
|
-
import { windowsHeadlessSpawnOptions } from '@bendyline/gezel/native';
|
|
28
|
+
import { diagnoseDllLoadFailure, windowsHeadlessSpawnOptions } from '@bendyline/gezel/native';
|
|
29
29
|
import { acquireNativeCapacity, } from './device-capacity.js';
|
|
30
30
|
const log = createLogger('native');
|
|
31
31
|
/**
|
|
@@ -204,6 +204,14 @@ export class NativeEngineSupervisor {
|
|
|
204
204
|
// swallowed so a buggy parser can't take down the supervisor.
|
|
205
205
|
logListeners = new Set();
|
|
206
206
|
recentStarts = [];
|
|
207
|
+
/**
|
|
208
|
+
* The {@link recentStarts} entry for the child running now. A child that
|
|
209
|
+
* became ready and was then stopped on purpose (idle, memory pressure, a
|
|
210
|
+
* model swap) was not a crash, so its start is returned to the budget.
|
|
211
|
+
* Otherwise an engine asked to swap models or yield memory three times in
|
|
212
|
+
* a minute would refuse the fourth start as a crash loop.
|
|
213
|
+
*/
|
|
214
|
+
currentStartAt;
|
|
207
215
|
lastUsedAt = 0;
|
|
208
216
|
/**
|
|
209
217
|
* The same mark on the awake clock. Every idle/pressure DECISION reads this
|
|
@@ -485,7 +493,11 @@ export class NativeEngineSupervisor {
|
|
|
485
493
|
const { child } = this.state;
|
|
486
494
|
if (this.state.kind === 'running' && reason !== 'health-restart') {
|
|
487
495
|
await this.runBeforeStop(reason, this.state.launch.baseUrl);
|
|
496
|
+
const index = this.currentStartAt === undefined ? -1 : this.recentStarts.indexOf(this.currentStartAt);
|
|
497
|
+
if (index >= 0)
|
|
498
|
+
this.recentStarts.splice(index, 1);
|
|
488
499
|
}
|
|
500
|
+
this.currentStartAt = undefined;
|
|
489
501
|
this.pressureDeadlineAt = undefined;
|
|
490
502
|
this.expectedExitReason = reason;
|
|
491
503
|
this.state = { kind: 'stopped' };
|
|
@@ -581,7 +593,9 @@ export class NativeEngineSupervisor {
|
|
|
581
593
|
}
|
|
582
594
|
});
|
|
583
595
|
}
|
|
584
|
-
|
|
596
|
+
const startedAt = Date.now();
|
|
597
|
+
this.recentStarts.push(startedAt);
|
|
598
|
+
this.currentStartAt = startedAt;
|
|
585
599
|
let child;
|
|
586
600
|
try {
|
|
587
601
|
child = this.spawn(launch.command, launch.args, {
|
|
@@ -1021,6 +1035,21 @@ export class NativeEngineSupervisor {
|
|
|
1021
1035
|
// shortage, and the GPU-to-RAM ladder those lines trigger cannot help
|
|
1022
1036
|
// when RAM is what ran out.
|
|
1023
1037
|
const oomKilled = signal === 'SIGKILL' && !expected ? this.oomKillsSince(this.currentOomKills) : null;
|
|
1038
|
+
// Windows reports a DLL the engine could not load only as a status, before
|
|
1039
|
+
// the engine prints a word. Name the DLL. A missing machine-wide runtime is
|
|
1040
|
+
// a panic of its own, so no build is quarantined for it (missing-dll.ts).
|
|
1041
|
+
const dllFailure = process.platform === 'win32' &&
|
|
1042
|
+
priorState.kind === 'starting' &&
|
|
1043
|
+
!expected &&
|
|
1044
|
+
!this.currentReachedReady &&
|
|
1045
|
+
!this.currentPanic
|
|
1046
|
+
? diagnoseDllLoadFailure(code, priorState.launch.command, {
|
|
1047
|
+
...process.env,
|
|
1048
|
+
...(priorState.launch.env ?? {}),
|
|
1049
|
+
})
|
|
1050
|
+
: null;
|
|
1051
|
+
if (dllFailure)
|
|
1052
|
+
this.onLog(`${this.logPrefix} startup DLL check: ${dllFailure.reason}`);
|
|
1024
1053
|
const panic = oomKilled
|
|
1025
1054
|
? {
|
|
1026
1055
|
kind: 'host-out-of-memory',
|
|
@@ -1032,7 +1061,9 @@ export class NativeEngineSupervisor {
|
|
|
1032
1061
|
kind: 'illegal-instruction',
|
|
1033
1062
|
line: 'engine terminated by SIGILL (illegal instruction) with no diagnostic output',
|
|
1034
1063
|
}
|
|
1035
|
-
:
|
|
1064
|
+
: dllFailure?.machineWide
|
|
1065
|
+
? { kind: 'missing-runtime', line: dllFailure.summary }
|
|
1066
|
+
: undefined));
|
|
1036
1067
|
const snapshot = {
|
|
1037
1068
|
incidentId: `native-${pid ?? 'unknown'}-${exitedAt}`,
|
|
1038
1069
|
...(pid !== undefined ? { pid } : {}),
|
|
@@ -1045,6 +1076,15 @@ export class NativeEngineSupervisor {
|
|
|
1045
1076
|
reachedReady: this.currentReachedReady,
|
|
1046
1077
|
...(expectedReason ? { expectedReason } : {}),
|
|
1047
1078
|
...(panic ? { panicKind: panic.kind, panicLine: panic.line } : {}),
|
|
1079
|
+
...(dllFailure
|
|
1080
|
+
? {
|
|
1081
|
+
dllLoadFailure: {
|
|
1082
|
+
kind: dllFailure.kind,
|
|
1083
|
+
...(dllFailure.dll ? { dll: dllFailure.dll } : {}),
|
|
1084
|
+
summary: dllFailure.summary,
|
|
1085
|
+
},
|
|
1086
|
+
}
|
|
1087
|
+
: {}),
|
|
1048
1088
|
outputTail: this.currentOutput,
|
|
1049
1089
|
...(this.currentDiagnostics ? { diagnostics: { ...this.currentDiagnostics } } : {}),
|
|
1050
1090
|
};
|
|
@@ -1105,7 +1145,15 @@ export class NativeEngineSupervisor {
|
|
|
1105
1145
|
// call wins because it lands first (the classifier sees the leaf
|
|
1106
1146
|
// exception line slightly before the kernel reports the exit).
|
|
1107
1147
|
if (this.startupAbort && priorState.kind === 'starting') {
|
|
1108
|
-
this.startupAbort.abort(
|
|
1148
|
+
this.startupAbort.abort(dllFailure
|
|
1149
|
+
? Object.assign(new Error(`${this.logPrefix} could not start: ${dllFailure.summary}`), {
|
|
1150
|
+
// The engine never came up, and the summary says what to do, so
|
|
1151
|
+
// callers pass it through unchanged.
|
|
1152
|
+
code: ENGINE_LAUNCH_ERROR_CODE,
|
|
1153
|
+
isActionable: true,
|
|
1154
|
+
...(panic?.kind === 'missing-runtime' ? { panicKind: panic.kind } : {}),
|
|
1155
|
+
})
|
|
1156
|
+
: new Error(`${this.logPrefix} child exited (code=${code} signal=${signal ?? 'none'}) before becoming ready`));
|
|
1109
1157
|
}
|
|
1110
1158
|
this.state = { kind: 'stopped' };
|
|
1111
1159
|
}
|
|
@@ -6,12 +6,14 @@ export const PROJECT_MACRO_FAILURE_CAP = 2;
|
|
|
6
6
|
/** Derive a concise user-facing closing from a successful kickoff result. */
|
|
7
7
|
export function deriveProjectMacroClosing(firstResult) {
|
|
8
8
|
const projectMatch = firstResult.match(/Started (?:project|job) "([^"]+)"/);
|
|
9
|
-
|
|
9
|
+
// "voorman" is what results said before the crew lead was renamed.
|
|
10
|
+
const leadMatch = firstResult.match(/Recruited (\S+) as (crew lead|voorman|builder|lead)/i);
|
|
10
11
|
const projectName = projectMatch?.[1];
|
|
11
12
|
const leadName = leadMatch?.[1];
|
|
12
|
-
const leadRole = leadMatch?.[2];
|
|
13
|
+
const leadRole = leadMatch?.[2]?.toLowerCase();
|
|
13
14
|
if (projectName && leadName) {
|
|
14
|
-
const
|
|
15
|
+
const crewLead = leadRole === 'crew lead' || leadRole === 'voorman';
|
|
16
|
+
const trailer = crewLead ? ' is leading the crew' : ' is on it';
|
|
15
17
|
return `Project "${projectName}" is kicked off — ${leadName}${trailer}. You can watch their work surface in this thread.`;
|
|
16
18
|
}
|
|
17
19
|
if (projectName) {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
interface Provider {
|
|
2
|
+
shutdown?(): Promise<void>;
|
|
3
|
+
}
|
|
4
|
+
/**
|
|
5
|
+
* Owns one lazy provider generation. Reset and broker retirement close admission
|
|
6
|
+
* before waiting for an in-flight build and admitted work. A failed shutdown
|
|
7
|
+
* retains its generation for retry; no replacement can leak past it.
|
|
8
|
+
*/
|
|
9
|
+
export declare class ProviderLifecycle<T extends Provider> {
|
|
10
|
+
private readonly operations;
|
|
11
|
+
private generation?;
|
|
12
|
+
private transition?;
|
|
13
|
+
private brokerRetired;
|
|
14
|
+
constructor(operations: ReadonlySet<PropertyKey>);
|
|
15
|
+
current(build: () => Promise<T>, retireWithBroker?: boolean): Promise<T>;
|
|
16
|
+
reset(): Promise<void>;
|
|
17
|
+
/** Permanent for native generations; user-owned cloud providers remain usable. */
|
|
18
|
+
retireForMachineBroker(): Promise<void>;
|
|
19
|
+
private drain;
|
|
20
|
+
}
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { ProviderRetirementGate, trackProviderOperations } from './retirement-gate.js';
|
|
2
|
+
/**
|
|
3
|
+
* Owns one lazy provider generation. Reset and broker retirement close admission
|
|
4
|
+
* before waiting for an in-flight build and admitted work. A failed shutdown
|
|
5
|
+
* retains its generation for retry; no replacement can leak past it.
|
|
6
|
+
*/
|
|
7
|
+
export class ProviderLifecycle {
|
|
8
|
+
operations;
|
|
9
|
+
generation;
|
|
10
|
+
transition;
|
|
11
|
+
brokerRetired = false;
|
|
12
|
+
constructor(operations) {
|
|
13
|
+
this.operations = operations;
|
|
14
|
+
}
|
|
15
|
+
async current(build, retireWithBroker = true) {
|
|
16
|
+
if (retireWithBroker && this.brokerRetired)
|
|
17
|
+
throw new Error('local provider retired after machine engine adoption');
|
|
18
|
+
if (this.transition) {
|
|
19
|
+
await this.transition;
|
|
20
|
+
return this.current(build, retireWithBroker);
|
|
21
|
+
}
|
|
22
|
+
if (this.generation?.closed) {
|
|
23
|
+
await this.reset();
|
|
24
|
+
return this.current(build, retireWithBroker);
|
|
25
|
+
}
|
|
26
|
+
if (this.generation)
|
|
27
|
+
return this.generation.build;
|
|
28
|
+
const gate = new ProviderRetirementGate();
|
|
29
|
+
const generation = {
|
|
30
|
+
gate,
|
|
31
|
+
closed: false,
|
|
32
|
+
retireWithBroker,
|
|
33
|
+
build: Promise.resolve()
|
|
34
|
+
.then(build)
|
|
35
|
+
.then((provider) => {
|
|
36
|
+
generation.provider = provider;
|
|
37
|
+
return trackProviderOperations(provider, gate, this.operations);
|
|
38
|
+
})
|
|
39
|
+
.catch((error) => {
|
|
40
|
+
if (this.generation === generation)
|
|
41
|
+
this.generation = undefined;
|
|
42
|
+
throw error;
|
|
43
|
+
}),
|
|
44
|
+
};
|
|
45
|
+
this.generation = generation;
|
|
46
|
+
return generation.build;
|
|
47
|
+
}
|
|
48
|
+
reset() {
|
|
49
|
+
return this.drain('local provider retired during reset; obtain the current provider');
|
|
50
|
+
}
|
|
51
|
+
/** Permanent for native generations; user-owned cloud providers remain usable. */
|
|
52
|
+
retireForMachineBroker() {
|
|
53
|
+
this.brokerRetired = true;
|
|
54
|
+
if (this.generation && !this.generation.retireWithBroker)
|
|
55
|
+
return Promise.resolve();
|
|
56
|
+
return this.drain();
|
|
57
|
+
}
|
|
58
|
+
drain(message) {
|
|
59
|
+
if (this.transition)
|
|
60
|
+
return this.transition;
|
|
61
|
+
const generation = this.generation;
|
|
62
|
+
if (!generation)
|
|
63
|
+
return Promise.resolve();
|
|
64
|
+
generation.closed = true;
|
|
65
|
+
generation.gate.beginRetirement(message);
|
|
66
|
+
const run = (async () => {
|
|
67
|
+
// A failed build produced no provider to shut down. Its caller still
|
|
68
|
+
// receives the build error, while retirement can finish successfully.
|
|
69
|
+
await generation.build.catch(() => { });
|
|
70
|
+
await generation.gate.waitForIdle();
|
|
71
|
+
await generation.provider?.shutdown?.();
|
|
72
|
+
if (this.generation === generation)
|
|
73
|
+
this.generation = undefined;
|
|
74
|
+
})();
|
|
75
|
+
this.transition = run;
|
|
76
|
+
const clear = () => {
|
|
77
|
+
if (this.transition === run)
|
|
78
|
+
this.transition = undefined;
|
|
79
|
+
};
|
|
80
|
+
void run.then(clear, clear);
|
|
81
|
+
return run;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tracks long-running provider operations while a machine-engine adoption is
|
|
3
|
+
* retiring the user daemon's local provider. The gate closes before draining:
|
|
4
|
+
* callers that already hold a provider reference cannot start a second local
|
|
5
|
+
* operation after the broker has become authoritative.
|
|
6
|
+
*/
|
|
7
|
+
export declare class ProviderRetirementGate {
|
|
8
|
+
private retiring;
|
|
9
|
+
private retirementMessage;
|
|
10
|
+
private active;
|
|
11
|
+
private readonly idleWaiters;
|
|
12
|
+
beginRetirement(message?: string): void;
|
|
13
|
+
run<T>(operation: () => Promise<T>): Promise<T>;
|
|
14
|
+
waitForIdle(): Promise<void>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Wrap only the operations which can own a native engine. Management calls
|
|
18
|
+
* remain direct, and shutdown is deliberately not counted as user work.
|
|
19
|
+
*/
|
|
20
|
+
export declare function trackProviderOperations<T extends object>(provider: T, gate: ProviderRetirementGate, operationNames: ReadonlySet<PropertyKey>): T;
|