@bendyline/gezel-inference 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog/chat-models/index.json +1 -1
- package/dist/catalog/chat-models/na/nanbeige4.2-3b-q4/manifest.json +1 -1
- package/dist/catalog/chat-models/ne/nemotron3.5-bolt-e4b-q4/manifest.json +142 -0
- package/dist/catalog/chat-models/ne/nemotron3.5-bolt-e4b-q4/versions/1.0.0/manifest.json +14 -0
- package/dist/catalog/chat-models/ne/nemotron3.5-bolt-e4b-q4/versions/1.0.1/manifest.json +14 -0
- package/dist/catalog/chat-models/raw-index.json +1 -1
- package/dist/catalog/image-models/an/animagine-xl-4/manifest.json +40 -0
- package/dist/catalog/image-models/an/animagine-xl-4/versions/4.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/an/animagine-xl-4/versions/4.0.1/manifest.json +16 -0
- package/dist/catalog/image-models/fl/flux-2-klein-4b-q4/manifest.json +44 -0
- package/dist/catalog/image-models/fl/flux-2-klein-4b-q4/versions/1.0.0/manifest.json +23 -0
- package/dist/catalog/image-models/fl/flux-2-klein-4b-q4/versions/1.0.1/manifest.json +29 -0
- package/dist/catalog/image-models/index.json +1 -0
- package/dist/catalog/image-models/ir/iris-3b-f32/manifest.json +41 -0
- package/dist/catalog/image-models/ir/iris-3b-f32/versions/1.0.0/manifest.json +19 -0
- package/dist/catalog/image-models/ir/iris-3b-f32/versions/1.0.1/manifest.json +25 -0
- package/dist/catalog/image-models/ir/iris-3b-f32/versions/1.0.2/manifest.json +25 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q4/manifest.json +42 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q4/versions/1.0.0/manifest.json +23 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q4/versions/1.0.1/manifest.json +29 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q6/manifest.json +42 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q6/versions/1.0.0/manifest.json +23 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q6/versions/1.0.1/manifest.json +29 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q8/manifest.json +42 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q8/versions/1.0.0/manifest.json +23 -0
- package/dist/catalog/image-models/kr/krea-2-turbo-q8/versions/1.0.1/manifest.json +29 -0
- package/dist/catalog/image-models/mi/ming-image-0.1-design-bf16/manifest.json +41 -0
- package/dist/catalog/image-models/mi/ming-image-0.1-design-bf16/versions/1.0.0/manifest.json +31 -0
- package/dist/catalog/image-models/mi/ming-image-0.1-design-bf16/versions/1.0.1/manifest.json +50 -0
- package/dist/catalog/image-models/mi/ming-image-0.1-design-bf16/versions/1.0.2/manifest.json +50 -0
- package/dist/catalog/image-models/mo/moss-image-sft-q4/manifest.json +40 -0
- package/dist/catalog/image-models/mo/moss-image-sft-q4/versions/1.0.0/manifest.json +37 -0
- package/dist/catalog/image-models/mo/moss-image-sft-q4/versions/1.0.1/manifest.json +37 -0
- package/dist/catalog/image-models/raw-index.json +1 -0
- package/dist/catalog/image-models/sd/sd-1.5/manifest.json +40 -0
- package/dist/catalog/image-models/sd/sd-1.5/versions/1.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/sd/sd-1.5/versions/1.0.1/manifest.json +16 -0
- package/dist/catalog/image-models/sd/sd-turbo/manifest.json +40 -0
- package/dist/catalog/image-models/sd/sd-turbo/versions/1.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/sd/sd-turbo/versions/1.0.1/manifest.json +16 -0
- package/dist/catalog/image-models/sd/sdxl-base-1.0/manifest.json +39 -0
- package/dist/catalog/image-models/sd/sdxl-base-1.0/versions/1.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/sd/sdxl-base-1.0/versions/1.0.1/manifest.json +16 -0
- package/dist/catalog/image-models/sd/sdxl-lightning-4step/manifest.json +40 -0
- package/dist/catalog/image-models/sd/sdxl-lightning-4step/versions/1.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/sd/sdxl-lightning-4step/versions/1.0.1/manifest.json +16 -0
- package/dist/catalog/image-models/sd/sdxl-turbo/manifest.json +41 -0
- package/dist/catalog/image-models/sd/sdxl-turbo/versions/1.0.0/manifest.json +10 -0
- package/dist/catalog/image-models/sd/sdxl-turbo/versions/1.0.1/manifest.json +16 -0
- package/dist/catalog/provenance.json +2 -1
- package/dist/engines/native-file-manifest.json +129 -119
- package/dist/engines/native-manifest.d.ts +4 -4
- package/dist/engines/native-manifest.js +36 -36
- package/dist/engines/resolver.js +9 -0
- package/dist/http/openai-compat/engine-failure.js +7 -2
- package/dist/http/openai-compat/responses-streaming.d.ts +13 -0
- package/dist/http/openai-compat/responses-streaming.js +30 -0
- package/dist/http/openai-compat/streaming.d.ts +9 -0
- package/dist/http/openai-compat/streaming.js +49 -9
- package/dist/http/openai-compat/tool-call-screen.d.ts +28 -0
- package/dist/http/openai-compat/tool-call-screen.js +43 -0
- package/dist/http/routes/v1-chat.js +15 -1
- package/dist/http/routes/v1-images.d.ts +44 -0
- package/dist/http/routes/v1-images.js +370 -0
- package/dist/http/routes/v1-responses.js +13 -1
- package/dist/inference-service.d.ts +3 -0
- package/dist/inference-service.js +24 -2
- package/dist/kokoro-lexicon/manifest.json +6 -3
- package/dist/models/image-fit.d.ts +83 -0
- package/dist/models/image-fit.js +163 -0
- package/dist/models/weight-facts.d.ts +36 -0
- package/dist/models/weight-facts.js +171 -0
- package/dist/providers/ds4/build-provider.js +28 -21
- package/dist/providers/ds4/dspark.d.ts +23 -5
- package/dist/providers/ds4/dspark.js +36 -8
- package/dist/providers/image/img2img-support.d.ts +40 -0
- package/dist/providers/image/img2img-support.js +74 -0
- package/dist/providers/image/local-engine.d.ts +31 -0
- package/dist/providers/image/local-engine.js +47 -0
- package/dist/providers/image/local-factory.d.ts +95 -0
- package/dist/providers/image/local-factory.js +270 -0
- package/dist/providers/image/mock.d.ts +23 -0
- package/dist/providers/image/mock.js +151 -0
- package/dist/providers/image/pull-registry.d.ts +106 -0
- package/dist/providers/image/pull-registry.js +227 -0
- package/dist/providers/image/sd-cpp.d.ts +200 -0
- package/dist/providers/image/sd-cpp.js +845 -0
- package/dist/providers/image/types.d.ts +237 -0
- package/dist/providers/image/types.js +20 -0
- package/dist/providers/llama-cpp/backend-health.d.ts +7 -0
- package/dist/providers/llama-cpp/backend-health.js +41 -1
- package/dist/providers/mlx/provider.js +13 -3
- package/dist/providers/native/supervisor.d.ts +8 -0
- package/dist/providers/native/supervisor.js +52 -4
- package/dist/providers/project-macro-loop-bail.js +5 -3
- package/dist/providers/provider-lifecycle.d.ts +21 -0
- package/dist/providers/provider-lifecycle.js +83 -0
- package/dist/providers/retirement-gate.d.ts +20 -0
- package/dist/providers/retirement-gate.js +55 -0
- package/dist/providers/types.d.ts +7 -1
- package/dist/system/memory.js +6 -0
- package/dist/utils/image-meta.d.ts +12 -0
- package/dist/utils/image-meta.js +77 -0
- package/package.json +6 -6
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-model img2img capability resolution for the local sd-cpp engine.
|
|
3
|
+
*
|
|
4
|
+
* stable-diffusion.cpp's `/sdapi/v1/img2img` endpoint accepts
|
|
5
|
+
* `init_images` for any loaded model, but whether the *architecture*
|
|
6
|
+
* actually honors the init latents varies: the classic SD 1.x / SDXL
|
|
7
|
+
* denoising path is mature, while newer diffusion-model architectures
|
|
8
|
+
* (FLUX.2's LLM-encoder family, Krea 2's Qwen-Image lineage) are unverified on
|
|
9
|
+
* the pinned sd-server — the same server build that already ignores
|
|
10
|
+
* per-request step overrides for Krea (see MODEL_SAMPLE_STEP_DEFAULTS).
|
|
11
|
+
*
|
|
12
|
+
* Resolution ladder, most explicit wins:
|
|
13
|
+
* 1. `supportsImg2Img` declared on the catalog manifest (persisted
|
|
14
|
+
* into the installed `manifest.json` at pull time),
|
|
15
|
+
* 2. the assessment map below (repo-local truth for shipped models,
|
|
16
|
+
* same pattern as MODEL_CFG_DEFAULTS),
|
|
17
|
+
* 3. a `weightsKind` default: `checkpoint` (and unknown/legacy
|
|
18
|
+
* installs, which are all SD-era checkpoints) → supported;
|
|
19
|
+
* `diffusion-model` → not supported until assessed.
|
|
20
|
+
*
|
|
21
|
+
* When a model resolves unsupported, callers drop the input images and
|
|
22
|
+
* report `img2imgSkippedReason` instead of silently producing an
|
|
23
|
+
* "edit" that never saw the source image. Flip a map entry (or set the
|
|
24
|
+
* catalog field) after verifying a family on real hardware.
|
|
25
|
+
*/
|
|
26
|
+
export const MODEL_IMG2IMG_SUPPORT = new Map([
|
|
27
|
+
// Classic SD checkpoints — the original, well-exercised img2img path.
|
|
28
|
+
['sd-1.5', true],
|
|
29
|
+
['sd-turbo', true],
|
|
30
|
+
['sdxl-base-1.0', true],
|
|
31
|
+
['sdxl-turbo', true],
|
|
32
|
+
['sdxl-lightning-4step', true],
|
|
33
|
+
['animagine-xl-4', true],
|
|
34
|
+
// FLUX.1 — img2img documented working in upstream stable-diffusion.cpp.
|
|
35
|
+
// Still in the pinned catalog and on people's disks.
|
|
36
|
+
['flux-1-schnell-q4', true],
|
|
37
|
+
['flux-1-dev-q4', true],
|
|
38
|
+
// FLUX.2 Klein — LLM-text-encoder flow model; img2img unverified on the
|
|
39
|
+
// pinned sd-server. Treat as txt2img-only until assessed.
|
|
40
|
+
['flux-2-klein-4b-q4', false],
|
|
41
|
+
// Krea 2 Turbo — Qwen-Image architecture; the pinned server already has
|
|
42
|
+
// known per-request gaps for these models (steps override ignored).
|
|
43
|
+
['krea-2-turbo-q4', false],
|
|
44
|
+
['krea-2-turbo-q6', false],
|
|
45
|
+
['krea-2-turbo-q8', false],
|
|
46
|
+
]);
|
|
47
|
+
export function resolveImg2ImgSupport(input) {
|
|
48
|
+
if (input.explicit !== undefined) {
|
|
49
|
+
return input.explicit
|
|
50
|
+
? { supported: true }
|
|
51
|
+
: {
|
|
52
|
+
supported: false,
|
|
53
|
+
reason: `model ${input.modelId ?? 'unknown'} does not support image editing (img2img)`,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
if (input.modelId !== undefined) {
|
|
57
|
+
const assessed = MODEL_IMG2IMG_SUPPORT.get(input.modelId);
|
|
58
|
+
if (assessed !== undefined) {
|
|
59
|
+
return assessed
|
|
60
|
+
? { supported: true }
|
|
61
|
+
: {
|
|
62
|
+
supported: false,
|
|
63
|
+
reason: `model ${input.modelId} does not support image editing (img2img) on the bundled engine`,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (input.weightsKind === 'diffusion-model') {
|
|
68
|
+
return {
|
|
69
|
+
supported: false,
|
|
70
|
+
reason: `model ${input.modelId ?? 'unknown'} has not been verified for image editing (img2img) on the bundled engine`,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
return { supported: true };
|
|
74
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { GezelConfig } from '@bendyline/gezel';
|
|
2
|
+
import type { CatalogService } from '@bendyline/gezel-catalog';
|
|
3
|
+
import type { GpuArbiter } from '../gpu-arbiter.js';
|
|
4
|
+
import { ImageModelPullRegistry } from './pull-registry.js';
|
|
5
|
+
import type { ImageProvider } from './types.js';
|
|
6
|
+
export interface LocalImageEngineOptions {
|
|
7
|
+
home: string;
|
|
8
|
+
catalog: CatalogService;
|
|
9
|
+
arbiter: GpuArbiter;
|
|
10
|
+
readConfig: () => Promise<Pick<GezelConfig, 'defaultImageModel'>>;
|
|
11
|
+
/** Override for tests. */
|
|
12
|
+
env?: NodeJS.ProcessEnv;
|
|
13
|
+
/** Test seam: a provider in place of the local engine. */
|
|
14
|
+
provider?: ImageProvider;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Hosted inference's image engine: the local sd-server provider, built on
|
|
18
|
+
* first use, plus the pulls that install its models. Reset drains in-flight
|
|
19
|
+
* renders before an engine binary swap takes effect.
|
|
20
|
+
*/
|
|
21
|
+
export declare class LocalImageEngine {
|
|
22
|
+
private readonly opts;
|
|
23
|
+
readonly pulls: ImageModelPullRegistry;
|
|
24
|
+
private readonly lifecycle;
|
|
25
|
+
constructor(opts: LocalImageEngineOptions);
|
|
26
|
+
/** Whether this host can render at all: an engine binary, a server URL, or the mock. */
|
|
27
|
+
available(): boolean;
|
|
28
|
+
current(): Promise<ImageProvider>;
|
|
29
|
+
reset(): Promise<void>;
|
|
30
|
+
shutdown(): Promise<void>;
|
|
31
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { ProviderLifecycle } from '../provider-lifecycle.js';
|
|
2
|
+
import { createLocalImageProvider } from './local-factory.js';
|
|
3
|
+
import { ImageModelPullRegistry } from './pull-registry.js';
|
|
4
|
+
/**
|
|
5
|
+
* Hosted inference's image engine: the local sd-server provider, built on
|
|
6
|
+
* first use, plus the pulls that install its models. Reset drains in-flight
|
|
7
|
+
* renders before an engine binary swap takes effect.
|
|
8
|
+
*/
|
|
9
|
+
export class LocalImageEngine {
|
|
10
|
+
opts;
|
|
11
|
+
pulls;
|
|
12
|
+
// Only promise-returning operations can be tracked; a pull is an async
|
|
13
|
+
// iterable and is cancelled through its own registry instead.
|
|
14
|
+
lifecycle = new ProviderLifecycle(new Set(['generate', 'deleteModel']));
|
|
15
|
+
constructor(opts) {
|
|
16
|
+
this.opts = opts;
|
|
17
|
+
this.pulls = new ImageModelPullRegistry({ imageProvider: this, catalog: opts.catalog });
|
|
18
|
+
}
|
|
19
|
+
/** Whether this host can render at all: an engine binary, a server URL, or the mock. */
|
|
20
|
+
available() {
|
|
21
|
+
if (this.opts.provider)
|
|
22
|
+
return true;
|
|
23
|
+
const env = this.opts.env ?? process.env;
|
|
24
|
+
return Boolean(env.GEZEL_MOCK_PROVIDER === '1' || env.GEZEL_SD_SERVER_BIN || env.GEZEL_SD_SERVER_URL);
|
|
25
|
+
}
|
|
26
|
+
current() {
|
|
27
|
+
return this.lifecycle.current(async () => {
|
|
28
|
+
if (this.opts.provider)
|
|
29
|
+
return this.opts.provider;
|
|
30
|
+
const config = await this.opts.readConfig().catch(() => ({}));
|
|
31
|
+
const defaultModelId = config.defaultImageModel?.['sd-cpp'];
|
|
32
|
+
return createLocalImageProvider({
|
|
33
|
+
home: this.opts.home,
|
|
34
|
+
arbiter: this.opts.arbiter,
|
|
35
|
+
...(this.opts.env ? { env: this.opts.env } : {}),
|
|
36
|
+
...(defaultModelId ? { defaultModelId } : {}),
|
|
37
|
+
});
|
|
38
|
+
}, false);
|
|
39
|
+
}
|
|
40
|
+
reset() {
|
|
41
|
+
return this.lifecycle.reset();
|
|
42
|
+
}
|
|
43
|
+
async shutdown() {
|
|
44
|
+
this.pulls.clear();
|
|
45
|
+
await this.lifecycle.reset();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The local image provider: stable-diffusion.cpp's `sd-server` on this
|
|
3
|
+
* machine. Shared by the full service (behind its cloud and remote routing)
|
|
4
|
+
* and by hosted inference, which serves only this.
|
|
5
|
+
*
|
|
6
|
+
* Selection rules (first-match):
|
|
7
|
+
* 1. `GEZEL_MOCK_PROVIDER=1` → `MockImageProvider` (tests, CI).
|
|
8
|
+
* 2. `GEZEL_SD_SERVER_URL` present → `StableDiffusionCppProvider`
|
|
9
|
+
* pointed at that URL, no supervisor ("user runs their own
|
|
10
|
+
* server" mode — useful during development).
|
|
11
|
+
* 3. `GEZEL_SD_SERVER_BIN` present → `StableDiffusionCppProvider`
|
|
12
|
+
* backed by a `NativeEngineSupervisor` that spawns the binary
|
|
13
|
+
* on-demand at an ephemeral port.
|
|
14
|
+
* 4. Fallback: `StableDiffusionCppProvider` with the default
|
|
15
|
+
* base URL (127.0.0.1:9081) and no supervisor. `health()` reports
|
|
16
|
+
* `not-configured` so the UI can say the engine is not installed.
|
|
17
|
+
*/
|
|
18
|
+
import type { GpuArbiter } from '../gpu-arbiter.js';
|
|
19
|
+
import type { NativeCapacityOptions } from '../native/device-capacity.js';
|
|
20
|
+
import type { ImageProvider, InstalledImageModelInfo } from './types.js';
|
|
21
|
+
export interface LocalImageProviderOptions {
|
|
22
|
+
home: string;
|
|
23
|
+
/** Override for tests. */
|
|
24
|
+
env?: NodeJS.ProcessEnv;
|
|
25
|
+
/**
|
|
26
|
+
* The person's default local model (`config.defaultImageModel['sd-cpp']`).
|
|
27
|
+
* A configured id that is no longer installed falls through to the first
|
|
28
|
+
* installed model.
|
|
29
|
+
*/
|
|
30
|
+
defaultModelId?: string;
|
|
31
|
+
/**
|
|
32
|
+
* Cross-engine GPU coordinator. Forwarded only to the supervised branch —
|
|
33
|
+
* the user-runs-their-own (`GEZEL_SD_SERVER_URL`) path doesn't manage
|
|
34
|
+
* VRAM tenancy.
|
|
35
|
+
*/
|
|
36
|
+
arbiter?: GpuArbiter;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Nothing to generate with: the person has to download a model first. Its
|
|
40
|
+
* own type so a route can answer with the fix, not a bare 500, which reads
|
|
41
|
+
* to an outside app as "something went wrong inside Gezel".
|
|
42
|
+
*/
|
|
43
|
+
export declare class NoLocalImageModelError extends Error {
|
|
44
|
+
readonly code = "no_local_image_model";
|
|
45
|
+
constructor();
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* How sd-server is admitted to the device memory ledger: by bytes, like any
|
|
49
|
+
* other engine, never exclusively. The ledger already refuses co-residency
|
|
50
|
+
* where the byte or GPU budget cannot hold both, and `detectGpuPolicy` states
|
|
51
|
+
* the rule: automatic heuristics must not serialize models that fit.
|
|
52
|
+
* Exclusive admission did exactly that. On a DGX Spark with gemma4-31b
|
|
53
|
+
* resident (95 GB of a 128 GB machine), a 4-second SDXL render waited up to
|
|
54
|
+
* 89 s for the chat engine to go idle, then was evicted as soon as the next
|
|
55
|
+
* chat turn asked for memory, so every render paid a fresh load.
|
|
56
|
+
*/
|
|
57
|
+
export declare function sdServerCapacity(home: string): NativeCapacityOptions;
|
|
58
|
+
export declare function createLocalImageProvider(opts: LocalImageProviderOptions): Promise<ImageProvider>;
|
|
59
|
+
export interface ResolvedInstalledModel extends InstalledImageModelInfo {
|
|
60
|
+
weightsPath: string;
|
|
61
|
+
weightsKind: 'checkpoint' | 'diffusion-model';
|
|
62
|
+
auxiliaryFiles: Array<{
|
|
63
|
+
role: string;
|
|
64
|
+
path: string;
|
|
65
|
+
}>;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Translate a resolved on-disk model into stable-diffusion.cpp's `sd-server`
|
|
69
|
+
* launch flags. `checkpoint` weights load via `--model`; `diffusion-model`
|
|
70
|
+
* weights are unet-only and ride alongside `--vae`/`--clip_l`/`--clip_g`/
|
|
71
|
+
* `--t5xxl` from the auxiliary files. Aux roles map 1:1 to flag names.
|
|
72
|
+
*
|
|
73
|
+
* Bind flags use `--listen-ip` / `--listen-port` (NOT `--host` / `--port`)
|
|
74
|
+
* — that's the upstream stable-diffusion.cpp server CLI as of master-587;
|
|
75
|
+
* earlier llama.cpp-style `--host` / `--port` will be rejected with
|
|
76
|
+
* `error: unknown argument: --host` and the supervisor restart-loops.
|
|
77
|
+
*
|
|
78
|
+
* `--vae-tiling` is the default because SDXL/Flux-class VAE decode at
|
|
79
|
+
* 800–1024px wants a single ~7 GB Vulkan buffer, which exceeds the
|
|
80
|
+
* per-allocation cap most consumer drivers enforce (~25–50% of total
|
|
81
|
+
* VRAM). The OOM lands at the very last step (sampling completes, then
|
|
82
|
+
* VAE blows up), wasting the entire generation. Tiled decode splits
|
|
83
|
+
* that buffer into chunks at ~10–20% decode-speed cost; on a 12 GB
|
|
84
|
+
* card it's the difference between succeeding and dying on `vae alloc
|
|
85
|
+
* compute buffer failed`. Quality difference is imperceptible for the
|
|
86
|
+
* tile sizes sd.cpp picks.
|
|
87
|
+
*
|
|
88
|
+
* The text encoder never takes flash attention. ggml's CPU kernel sums the
|
|
89
|
+
* attention values in FP16, which a Qwen-family encoder overflows: FLUX.2
|
|
90
|
+
* klein on a Windows-on-ARM CPU rendered unrelated shapes with a 200
|
|
91
|
+
* (native-v0.1.53). It bites wherever the encoder lands on the CPU, and the
|
|
92
|
+
* encoder runs once per image, so GPUs lose nothing. `--diffusion-fa` means
|
|
93
|
+
* `--flash-attn off,diffusion=on`, so that branch already leaves it off.
|
|
94
|
+
*/
|
|
95
|
+
export declare function buildSdServerArgs(model: ResolvedInstalledModel, port: number): string[];
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The local image provider: stable-diffusion.cpp's `sd-server` on this
|
|
3
|
+
* machine. Shared by the full service (behind its cloud and remote routing)
|
|
4
|
+
* and by hosted inference, which serves only this.
|
|
5
|
+
*
|
|
6
|
+
* Selection rules (first-match):
|
|
7
|
+
* 1. `GEZEL_MOCK_PROVIDER=1` → `MockImageProvider` (tests, CI).
|
|
8
|
+
* 2. `GEZEL_SD_SERVER_URL` present → `StableDiffusionCppProvider`
|
|
9
|
+
* pointed at that URL, no supervisor ("user runs their own
|
|
10
|
+
* server" mode — useful during development).
|
|
11
|
+
* 3. `GEZEL_SD_SERVER_BIN` present → `StableDiffusionCppProvider`
|
|
12
|
+
* backed by a `NativeEngineSupervisor` that spawns the binary
|
|
13
|
+
* on-demand at an ephemeral port.
|
|
14
|
+
* 4. Fallback: `StableDiffusionCppProvider` with the default
|
|
15
|
+
* base URL (127.0.0.1:9081) and no supervisor. `health()` reports
|
|
16
|
+
* `not-configured` so the UI can say the engine is not installed.
|
|
17
|
+
*/
|
|
18
|
+
import { readFile } from 'node:fs/promises';
|
|
19
|
+
import { join } from 'node:path';
|
|
20
|
+
import { imageMediaPool, sdServerLedgerClaim, sdServerRequirement, } from '../../models/image-fit.js';
|
|
21
|
+
import { findModelRoot, listOverlayModelIds, modelStorageRoots, verifyReadOnlyModelPayload, } from '../../models/storage-roots.js';
|
|
22
|
+
import { localSdCppWeightFiles } from '../../models/weight-facts.js';
|
|
23
|
+
import { pressureIdleGraceMs } from '../native/capacity-broker.js';
|
|
24
|
+
import { measuredCapacityBudget } from '../native/measured-budget.js';
|
|
25
|
+
import { pickFreePort } from '../native/port.js';
|
|
26
|
+
import { NativeEngineSupervisor } from '../native/supervisor.js';
|
|
27
|
+
import { patientFetch } from '../patient-fetch.js';
|
|
28
|
+
import { MockImageProvider } from './mock.js';
|
|
29
|
+
import { MODEL_SAMPLE_STEP_DEFAULTS, StableDiffusionCppProvider, sdServerLoadProfile, } from './sd-cpp.js';
|
|
30
|
+
/**
|
|
31
|
+
* Nothing to generate with: the person has to download a model first. Its
|
|
32
|
+
* own type so a route can answer with the fix, not a bare 500, which reads
|
|
33
|
+
* to an outside app as "something went wrong inside Gezel".
|
|
34
|
+
*/
|
|
35
|
+
export class NoLocalImageModelError extends Error {
|
|
36
|
+
code = 'no_local_image_model';
|
|
37
|
+
constructor() {
|
|
38
|
+
super('No image model is available locally. Download one from Settings → Image generation before generating.');
|
|
39
|
+
this.name = 'NoLocalImageModelError';
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* How sd-server is admitted to the device memory ledger: by bytes, like any
|
|
44
|
+
* other engine, never exclusively. The ledger already refuses co-residency
|
|
45
|
+
* where the byte or GPU budget cannot hold both, and `detectGpuPolicy` states
|
|
46
|
+
* the rule: automatic heuristics must not serialize models that fit.
|
|
47
|
+
* Exclusive admission did exactly that. On a DGX Spark with gemma4-31b
|
|
48
|
+
* resident (95 GB of a 128 GB machine), a 4-second SDXL render waited up to
|
|
49
|
+
* 89 s for the chat engine to go idle, then was evicted as soon as the next
|
|
50
|
+
* chat turn asked for memory, so every render paid a fresh load.
|
|
51
|
+
*/
|
|
52
|
+
export function sdServerCapacity(home) {
|
|
53
|
+
return { home };
|
|
54
|
+
}
|
|
55
|
+
export async function createLocalImageProvider(opts) {
|
|
56
|
+
const env = opts.env ?? process.env;
|
|
57
|
+
const storageRoots = modelStorageRoots({ home: opts.home, engine: 'sd-cpp', env });
|
|
58
|
+
const modelsRoot = storageRoots.writableRoot;
|
|
59
|
+
const defaultLocalModelId = opts.defaultModelId;
|
|
60
|
+
if (env.GEZEL_MOCK_PROVIDER === '1') {
|
|
61
|
+
return new MockImageProvider();
|
|
62
|
+
}
|
|
63
|
+
if (env.GEZEL_SD_SERVER_URL) {
|
|
64
|
+
return new StableDiffusionCppProvider({
|
|
65
|
+
baseUrl: env.GEZEL_SD_SERVER_URL,
|
|
66
|
+
modelsRoot,
|
|
67
|
+
storageRoots,
|
|
68
|
+
fetchImpl: patientFetch(),
|
|
69
|
+
...(defaultLocalModelId ? { defaultModelId: defaultLocalModelId } : {}),
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
if (env.GEZEL_SD_SERVER_BIN) {
|
|
73
|
+
const binary = env.GEZEL_SD_SERVER_BIN;
|
|
74
|
+
// Track the most-recently-resolved launch so the provider can point
|
|
75
|
+
// fetch() at the right port without the supervisor exposing state.
|
|
76
|
+
let cachedPort;
|
|
77
|
+
// Shared with the provider: which installed model to launch
|
|
78
|
+
// sd-server with. sd-server is single-model per process, so the
|
|
79
|
+
// provider sets this to the requested model and stops the server
|
|
80
|
+
// when it changes; resolveLaunch reads it on the next start.
|
|
81
|
+
const launchState = { modelId: undefined };
|
|
82
|
+
let launchedModel;
|
|
83
|
+
const supervisor = new NativeEngineSupervisor({
|
|
84
|
+
capacity: {
|
|
85
|
+
...sdServerCapacity(opts.home),
|
|
86
|
+
// Price the launched model from its own weight files — the same
|
|
87
|
+
// pricing the download page used — and claim what the render
|
|
88
|
+
// actually holds on the card (see sdServerLedgerClaim).
|
|
89
|
+
requirement: async () => {
|
|
90
|
+
if (!launchedModel)
|
|
91
|
+
return undefined;
|
|
92
|
+
const budget = await measuredCapacityBudget();
|
|
93
|
+
const requirement = sdServerRequirement({
|
|
94
|
+
modelId: launchedModel.id,
|
|
95
|
+
weightsKind: launchedModel.weightsKind,
|
|
96
|
+
files: await localSdCppWeightFiles(launchedModel),
|
|
97
|
+
pool: imageMediaPool(budget.kind),
|
|
98
|
+
});
|
|
99
|
+
return sdServerLedgerClaim(requirement, budget.fastBytes);
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
logPrefix: '[sd-server]',
|
|
103
|
+
// sd-server has no `/health`. It answers the root with an HTML file
|
|
104
|
+
// (with --serve-html-path) or a 404, and starts listening only once
|
|
105
|
+
// it has read the model's metadata — the weights themselves load
|
|
106
|
+
// inside the first request. Any HTTP response is the signal.
|
|
107
|
+
readinessPath: '/',
|
|
108
|
+
readyOnAnyResponse: true,
|
|
109
|
+
// Cold start can take a while: metadata for a 49 GB model, backend
|
|
110
|
+
// init, first kernel compile. Give the readiness probe generous
|
|
111
|
+
// headroom so first launch doesn't trip the restart-budget guard.
|
|
112
|
+
startupTimeoutMs: 5 * 60 * 1000,
|
|
113
|
+
// An idle image engine gives its memory back when a chat model is
|
|
114
|
+
// waiting for it, the same way the chat engines yield to a render.
|
|
115
|
+
...(opts.arbiter
|
|
116
|
+
? {
|
|
117
|
+
memoryPressure: () => opts.arbiter.getMemoryPressureStatus(),
|
|
118
|
+
pressureIdleTimeoutMs: pressureIdleGraceMs(),
|
|
119
|
+
}
|
|
120
|
+
: {}),
|
|
121
|
+
resolveLaunch: async () => {
|
|
122
|
+
const port = cachedPort ?? (await pickFreePort());
|
|
123
|
+
cachedPort = port;
|
|
124
|
+
// Prefer the model the provider asked for, then the configured
|
|
125
|
+
// default, then the first installed — covering a generate with no
|
|
126
|
+
// explicit model, and a requested (or configured) model having
|
|
127
|
+
// since been deleted.
|
|
128
|
+
const requestedId = launchState.modelId ?? defaultLocalModelId;
|
|
129
|
+
const model = (requestedId ? await findInstalledModel(storageRoots, requestedId) : undefined) ??
|
|
130
|
+
(await findFirstInstalledModel(storageRoots));
|
|
131
|
+
if (!model)
|
|
132
|
+
throw new NoLocalImageModelError();
|
|
133
|
+
launchedModel = model;
|
|
134
|
+
return {
|
|
135
|
+
command: binary,
|
|
136
|
+
args: buildSdServerArgs(model, port),
|
|
137
|
+
baseUrl: `http://127.0.0.1:${port}`,
|
|
138
|
+
};
|
|
139
|
+
},
|
|
140
|
+
});
|
|
141
|
+
return new StableDiffusionCppProvider({
|
|
142
|
+
// The baseUrl here is only the fallback when the supervisor is
|
|
143
|
+
// not yet resolved; generate() overrides from `ensureRunning()`.
|
|
144
|
+
baseUrl: 'http://127.0.0.1:9081',
|
|
145
|
+
modelsRoot,
|
|
146
|
+
storageRoots,
|
|
147
|
+
supervisor,
|
|
148
|
+
launchState,
|
|
149
|
+
fetchImpl: patientFetch(),
|
|
150
|
+
...(defaultLocalModelId ? { defaultModelId: defaultLocalModelId } : {}),
|
|
151
|
+
...(opts.arbiter ? { arbiter: opts.arbiter } : {}),
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
return new StableDiffusionCppProvider({
|
|
155
|
+
baseUrl: 'http://127.0.0.1:9081',
|
|
156
|
+
modelsRoot,
|
|
157
|
+
storageRoots,
|
|
158
|
+
...(defaultLocalModelId ? { defaultModelId: defaultLocalModelId } : {}),
|
|
159
|
+
// Nothing was wired up. Flag the provider so `health()` can return
|
|
160
|
+
// `not-configured` (distinct UI guidance) rather than making the user
|
|
161
|
+
// wait for a generic "unreachable" probe to fail.
|
|
162
|
+
configured: false,
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Read one installed model's resolved launch info from its on-disk
|
|
167
|
+
* `manifest.json`, or undefined if absent / malformed / incomplete.
|
|
168
|
+
*/
|
|
169
|
+
async function resolveInstalledModel(modelsRoot, id, storageRoots) {
|
|
170
|
+
try {
|
|
171
|
+
const itemDir = join(modelsRoot, id);
|
|
172
|
+
const raw = await readFile(join(itemDir, 'manifest.json'), 'utf8');
|
|
173
|
+
const parsed = JSON.parse(raw);
|
|
174
|
+
if (!parsed.id || !parsed.name || !parsed.weightsFilename)
|
|
175
|
+
return undefined;
|
|
176
|
+
if (storageRoots &&
|
|
177
|
+
!(await verifyReadOnlyModelPayload(storageRoots, modelsRoot, id, parsed.fileSha256))) {
|
|
178
|
+
return undefined;
|
|
179
|
+
}
|
|
180
|
+
const aux = [];
|
|
181
|
+
for (const a of parsed.auxiliaryFiles ?? []) {
|
|
182
|
+
if (a.role && a.filename)
|
|
183
|
+
aux.push({ role: a.role, path: join(itemDir, a.filename) });
|
|
184
|
+
}
|
|
185
|
+
return {
|
|
186
|
+
id: parsed.id,
|
|
187
|
+
name: parsed.name,
|
|
188
|
+
approxSizeBytes: parsed.approxSizeBytes ?? 0,
|
|
189
|
+
installedAt: parsed.installedAt ?? new Date().toISOString(),
|
|
190
|
+
weightsPath: join(itemDir, parsed.weightsFilename),
|
|
191
|
+
weightsKind: parsed.weightsKind ?? 'checkpoint',
|
|
192
|
+
auxiliaryFiles: aux,
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
return undefined;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/** Resolve a specific installed model by id (the one a request targets). */
|
|
200
|
+
async function findInstalledModel(storageRoots, id) {
|
|
201
|
+
const root = await findModelRoot(storageRoots, id);
|
|
202
|
+
return root ? resolveInstalledModel(root, id, storageRoots) : undefined;
|
|
203
|
+
}
|
|
204
|
+
async function findFirstInstalledModel(storageRoots) {
|
|
205
|
+
const entries = await listOverlayModelIds(storageRoots);
|
|
206
|
+
entries.sort();
|
|
207
|
+
for (const id of entries) {
|
|
208
|
+
const model = await findInstalledModel(storageRoots, id);
|
|
209
|
+
if (model)
|
|
210
|
+
return model;
|
|
211
|
+
}
|
|
212
|
+
return undefined;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Translate a resolved on-disk model into stable-diffusion.cpp's `sd-server`
|
|
216
|
+
* launch flags. `checkpoint` weights load via `--model`; `diffusion-model`
|
|
217
|
+
* weights are unet-only and ride alongside `--vae`/`--clip_l`/`--clip_g`/
|
|
218
|
+
* `--t5xxl` from the auxiliary files. Aux roles map 1:1 to flag names.
|
|
219
|
+
*
|
|
220
|
+
* Bind flags use `--listen-ip` / `--listen-port` (NOT `--host` / `--port`)
|
|
221
|
+
* — that's the upstream stable-diffusion.cpp server CLI as of master-587;
|
|
222
|
+
* earlier llama.cpp-style `--host` / `--port` will be rejected with
|
|
223
|
+
* `error: unknown argument: --host` and the supervisor restart-loops.
|
|
224
|
+
*
|
|
225
|
+
* `--vae-tiling` is the default because SDXL/Flux-class VAE decode at
|
|
226
|
+
* 800–1024px wants a single ~7 GB Vulkan buffer, which exceeds the
|
|
227
|
+
* per-allocation cap most consumer drivers enforce (~25–50% of total
|
|
228
|
+
* VRAM). The OOM lands at the very last step (sampling completes, then
|
|
229
|
+
* VAE blows up), wasting the entire generation. Tiled decode splits
|
|
230
|
+
* that buffer into chunks at ~10–20% decode-speed cost; on a 12 GB
|
|
231
|
+
* card it's the difference between succeeding and dying on `vae alloc
|
|
232
|
+
* compute buffer failed`. Quality difference is imperceptible for the
|
|
233
|
+
* tile sizes sd.cpp picks.
|
|
234
|
+
*
|
|
235
|
+
* The text encoder never takes flash attention. ggml's CPU kernel sums the
|
|
236
|
+
* attention values in FP16, which a Qwen-family encoder overflows: FLUX.2
|
|
237
|
+
* klein on a Windows-on-ARM CPU rendered unrelated shapes with a 200
|
|
238
|
+
* (native-v0.1.53). It bites wherever the encoder lands on the CPU, and the
|
|
239
|
+
* encoder runs once per image, so GPUs lose nothing. `--diffusion-fa` means
|
|
240
|
+
* `--flash-attn off,diffusion=on`, so that branch already leaves it off.
|
|
241
|
+
*/
|
|
242
|
+
export function buildSdServerArgs(model, port) {
|
|
243
|
+
const args = [];
|
|
244
|
+
if (model.weightsKind === 'diffusion-model') {
|
|
245
|
+
args.push('--diffusion-model', model.weightsPath);
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
args.push('--model', model.weightsPath);
|
|
249
|
+
}
|
|
250
|
+
for (const aux of model.auxiliaryFiles) {
|
|
251
|
+
args.push(`--${aux.role}`, aux.path);
|
|
252
|
+
}
|
|
253
|
+
const steps = MODEL_SAMPLE_STEP_DEFAULTS.get(model.id);
|
|
254
|
+
if (steps)
|
|
255
|
+
args.push('--steps', String(steps));
|
|
256
|
+
const profile = sdServerLoadProfile(model.id);
|
|
257
|
+
if (profile.diffusionFlashAttention)
|
|
258
|
+
args.push('--diffusion-fa');
|
|
259
|
+
else
|
|
260
|
+
args.push('--flash-attn', 'auto,te=off');
|
|
261
|
+
if (profile.bf16TensorPattern) {
|
|
262
|
+
args.push('--tensor-type-rules', `${profile.bf16TensorPattern}=bf16`);
|
|
263
|
+
}
|
|
264
|
+
if (profile.vaeDecode === 'tiled')
|
|
265
|
+
args.push('--vae-tiling');
|
|
266
|
+
if (profile.stepCache)
|
|
267
|
+
args.push('--cache-mode', profile.stepCache);
|
|
268
|
+
args.push('--listen-ip', '127.0.0.1', '--listen-port', String(port));
|
|
269
|
+
return args;
|
|
270
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MockImageProvider — deterministic image generation for tests and for the
|
|
3
|
+
* `GEZEL_MOCK_PROVIDER=1` flow. Produces a tiny, valid PNG whose pixel
|
|
4
|
+
* colour is derived from a hash of the prompt so different prompts yield
|
|
5
|
+
* visually different (but still trivially small) outputs.
|
|
6
|
+
*/
|
|
7
|
+
import type { ImageEngineHealth, ImageGenerationInput, ImageGenerationOutput, ImageModelPullEvent, ImageModelPullSpec, ImageProvider, InstalledImageModelInfo } from './types.js';
|
|
8
|
+
export declare class MockImageProvider implements ImageProvider {
|
|
9
|
+
readonly name = "mock";
|
|
10
|
+
private readonly installed;
|
|
11
|
+
/** Internal counter driving deterministic seed generation on "random" seeds. */
|
|
12
|
+
private nextSeed;
|
|
13
|
+
constructor(opts?: {
|
|
14
|
+
preinstalled?: InstalledImageModelInfo[];
|
|
15
|
+
});
|
|
16
|
+
/** Number of input images received on the most recent generate(). Tests use this. */
|
|
17
|
+
lastInputImageCount: number;
|
|
18
|
+
generate(input: ImageGenerationInput): Promise<ImageGenerationOutput>;
|
|
19
|
+
listInstalledModels(): Promise<InstalledImageModelInfo[]>;
|
|
20
|
+
pullModel(id: string, spec: ImageModelPullSpec, signal?: AbortSignal): AsyncIterable<ImageModelPullEvent>;
|
|
21
|
+
deleteModel(id: string): Promise<void>;
|
|
22
|
+
health(): Promise<ImageEngineHealth>;
|
|
23
|
+
}
|