@qvac/harness 1.0.0-rc.0 → 1.0.0-rc.2

package/dist-lib/lib/models/model-catalog.js

@@ -1,8 +1,8 @@
-import { AUDIOGEN_ACESTEP_5HZ_LM_0_6B_Q8_0, AUDIOGEN_ACESTEP_V15_SFT_Q8_0, AUDIOGEN_ACESTEP_V15_TURBO_Q4_K_M, AUDIOGEN_ACESTEP_V15_TURBO_Q8_0, AUDIOGEN_QWEN3_EMBEDDING_0_6B_Q8_0, AUDIOGEN_VAE_BF16, EMBEDDINGGEMMA_300M_Q4_0, FLUX_2_KLEIN_4B_Q4_0, FLUX_2_KLEIN_4B_VAE, HEALTHCARE_1_7B_MEDICAL_Q4_K_M, HEALTHCARE_4B_MEDICAL_Q4_K_M, MMPROJ_QWEN3_5_2B_MULTIMODAL_Q8_0, MMPROJ_QWEN3_5_4B_MULTIMODAL_Q8_0, MMPROJ_QWEN3_5_9B_MULTIMODAL_F16, MMPROJ_QWEN3_6_27B_MULTIMODAL_F16, MMPROJ_QWEN3_6_35B_A3B_MULTIMODAL_F16, MMPROJ_QWEN3_8_27B_MULTIMODAL_F16, QWEN3_5_2B_MULTIMODAL_Q4_K_M, QWEN3_5_4B_MULTIMODAL_Q4_K_M, QWEN3_5_9B_MULTIMODAL_Q4_K_M, PARAKEET_TDT_0_6B_V3_Q8_0, QWEN3_4B_Q4_K_M, QWEN3_6_27B_MULTIMODAL_Q4_K_XL, QWEN3_6_35B_A3B_MULTIMODAL_Q4_K_M, QWEN3_8_27B_MULTIMODAL_UD_Q4_K_XL, SD_V2_1_1B_Q8_0, SDXL_BASE_1_0_3B_Q4_0, TTS_MULTILINGUAL_SUPERTONIC3_Q4_0, TTS_MULTILINGUAL_SUPERTONIC3_Q8_0 } from '@qvac/sdk/models';
+import { AUDIOGEN_ACESTEP_5HZ_LM_0_6B_Q8_0, AUDIOGEN_ACESTEP_V15_BASE_Q4_K_M, AUDIOGEN_ACESTEP_V15_SFT_Q8_0, AUDIOGEN_ACESTEP_V15_TURBO_Q4_K_M, AUDIOGEN_ACESTEP_V15_TURBO_Q8_0, AUDIOGEN_QWEN3_EMBEDDING_0_6B_Q8_0, AUDIOGEN_VAE_BF16, EMBEDDINGGEMMA_300M_Q4_0, FLUX_2_KLEIN_4B_Q4_0, FLUX_2_KLEIN_4B_VAE, HEALTHCARE_1_7B_MEDICAL_Q4_K_M, HEALTHCARE_4B_MEDICAL_Q4_K_M, MMPROJ_QWEN3_5_2B_MULTIMODAL_Q8_0, MMPROJ_QWEN3_5_4B_MULTIMODAL_Q8_0, MMPROJ_QWEN3_5_9B_MULTIMODAL_F16, MMPROJ_QWEN3_6_27B_MULTIMODAL_F16, MMPROJ_QWEN3_6_35B_A3B_MULTIMODAL_F16, MMPROJ_QWEN3_8_27B_MULTIMODAL_F16, MMPROJ_TERNARY_BONSAI_2_27B_MULTIMODAL_Q8_0, QWEN3_5_2B_MULTIMODAL_Q4_K_M, QWEN3_5_4B_MULTIMODAL_Q4_K_M, QWEN3_5_9B_MULTIMODAL_Q4_K_M, PARAKEET_TDT_0_6B_V3_Q8_0, PARAKEET_TDT_1_1B_Q8_0, QWEN3_4B_Q4_K_M, QWEN3_6_27B_MULTIMODAL_Q4_K_XL, QWEN3_6_35B_A3B_MULTIMODAL_Q4_K_M, QWEN3_8_27B_MULTIMODAL_UD_Q4_K_XL, SD_V2_1_1B_Q8_0, SDXL_BASE_1_0_3B_Q4_0, TERNARY_BONSAI_2_27B_MULTIMODAL_PQ2_0, TERNARY_BONSAI_2_27B_MULTIMODAL_PTQ1_0, TTS_CODEC_DECODER_MOSS_TTS_F16, TTS_CODEC_ENCODER_MOSS_TTS_F16, TTS_DELAY_LLM_MOSS_TTS_F16, TTS_MULTILINGUAL_SUPERTONIC3_FP16, TTS_MULTILINGUAL_SUPERTONIC3_Q4_0, TTS_MULTILINGUAL_SUPERTONIC3_Q8_0 } from '@qvac/sdk/models';
 import { detectToolDialect } from './tool-dialect.js';
 import { defaultTtsModelName } from './tts-model-default.js';
-import { CAPABILITY_PROFILE_ORDER, QWEN_35_2B, QWEN_35_4B, QWEN_35_9B, QWEN_36_27B, QWEN_36_35B_A3B, QWEN_38_27B, MEDPSY_1_7B, MEDPSY_4B, isCapabilityProfile, meetsMinCapabilityProfile, modelNameToCapabilityProfile, capabilityProfileToModelName, isMultimodalLlm, AGENT_MODEL_NAMES, modelDisplayName, capabilityProfileModelLabel, DEFAULT_IMAGE_MODEL_CHOICE, IMAGE_CHOICE_CATALOG, isMobilePlatform, runnableModelName } from './model-catalog-constants.js';
-export { CAPABILITY_PROFILE_ORDER, QWEN_35_2B, QWEN_35_4B, QWEN_35_9B, QWEN_36_27B, QWEN_36_35B_A3B, QWEN_38_27B, MEDPSY_1_7B, MEDPSY_4B, isCapabilityProfile, meetsMinCapabilityProfile, modelNameToCapabilityProfile, capabilityProfileToModelName, isMultimodalLlm, AGENT_MODEL_NAMES, modelDisplayName, capabilityProfileModelLabel, DEFAULT_IMAGE_MODEL_CHOICE, IMAGE_CHOICE_CATALOG, isMobilePlatform, runnableModelName };
+import { CAPABILITY_PROFILE_ORDER, QWEN_35_2B, QWEN_35_4B, QWEN_35_9B, QWEN_36_27B, QWEN_36_35B_A3B, QWEN_38_27B, BONSAI_2_27B_PQ2, BONSAI_2_27B_PTQ1, MEDPSY_1_7B, MEDPSY_4B, isCapabilityProfile, meetsMinCapabilityProfile, modelNameToCapabilityProfile, capabilityProfileToModelName, isMultimodalLlm, AGENT_MODEL_NAMES, modelDisplayName, capabilityProfileModelLabel, DEFAULT_IMAGE_MODEL_CHOICE, IMAGE_CHOICE_CATALOG, isMobilePlatform, runnableModelName } from './model-catalog-constants.js';
+export { CAPABILITY_PROFILE_ORDER, QWEN_35_2B, QWEN_35_4B, QWEN_35_9B, QWEN_36_27B, QWEN_36_35B_A3B, QWEN_38_27B, BONSAI_2_27B_PQ2, BONSAI_2_27B_PTQ1, MEDPSY_1_7B, MEDPSY_4B, isCapabilityProfile, meetsMinCapabilityProfile, modelNameToCapabilityProfile, capabilityProfileToModelName, isMultimodalLlm, AGENT_MODEL_NAMES, modelDisplayName, capabilityProfileModelLabel, DEFAULT_IMAGE_MODEL_CHOICE, IMAGE_CHOICE_CATALOG, isMobilePlatform, runnableModelName };
 export const LLM_CATALOG = [
     {
         name: QWEN_35_2B,
@@ -51,6 +51,20 @@ export const LLM_CATALOG = [
         weight: QWEN3_8_27B_MULTIMODAL_UD_Q4_K_XL,
         capabilityProfile: 'xhigh',
         projection: MMPROJ_QWEN3_8_27B_MULTIMODAL_F16
+    },
+    {
+        name: BONSAI_2_27B_PQ2,
+        weight: TERNARY_BONSAI_2_27B_MULTIMODAL_PQ2_0,
+        capabilityProfile: 'high',
+        projection: MMPROJ_TERNARY_BONSAI_2_27B_MULTIMODAL_Q8_0,
+        toolDialect: 'qwen35'
+    },
+    {
+        name: BONSAI_2_27B_PTQ1,
+        weight: TERNARY_BONSAI_2_27B_MULTIMODAL_PTQ1_0,
+        capabilityProfile: 'high',
+        projection: MMPROJ_TERNARY_BONSAI_2_27B_MULTIMODAL_Q8_0,
+        toolDialect: 'qwen35'
     }
 ];
 export function defaultProjectionMap() {
@@ -62,16 +76,63 @@ export function defaultProjectionMap() {
     }
     return map;
 }
+const TTS_FULL_STEPS = { ttsNumInferenceSteps: 16 };
+export const MOSS_TTS = {
+    name: 'MOSS-TTS-8B-F16',
+    category: 'tts',
+    asset: TTS_DELAY_LLM_MOSS_TTS_F16,
+    companions: [TTS_CODEC_DECODER_MOSS_TTS_F16],
+    modelConfig: {
+        ttsEngine: 'moss',
+        language: 'en',
+        mossCodecDecoderModelSrc: TTS_CODEC_DECODER_MOSS_TTS_F16.src,
+        streamChunkTokens: 25,
+        useGPU: true
+    }
+};
+export const MOSS_CODEC_ENCODER_SRC = TTS_CODEC_ENCODER_MOSS_TTS_F16.src;
 export const CATEGORY_DEFAULTS = [
     { name: 'Supertonic3-Q4_0', category: 'tts', asset: TTS_MULTILINGUAL_SUPERTONIC3_Q4_0 },
-    { name: 'Supertonic3-Q8_0', category: 'tts', asset: TTS_MULTILINGUAL_SUPERTONIC3_Q8_0 },
+    {
+        name: 'Supertonic3-Q8_0',
+        category: 'tts',
+        asset: TTS_MULTILINGUAL_SUPERTONIC3_Q8_0,
+        modelConfig: TTS_FULL_STEPS
+    },
+    {
+        name: 'Supertonic3-FP16',
+        category: 'tts',
+        asset: TTS_MULTILINGUAL_SUPERTONIC3_FP16,
+        modelConfig: TTS_FULL_STEPS
+    },
+    MOSS_TTS,
     {
         name: 'Parakeet-TDT-0.6B-v3-Q8_0',
         category: 'transcription',
         asset: PARAKEET_TDT_0_6B_V3_Q8_0
     },
+    {
+        name: 'Parakeet-TDT-1.1B-Q8_0',
+        category: 'transcription',
+        asset: PARAKEET_TDT_1_1B_Q8_0
+    },
     { name: 'EmbeddingGemma-300M-Q4_0', category: 'embedding', asset: EMBEDDINGGEMMA_300M_Q4_0 }
 ];
+export function categoryModelConfig(modelSrc) {
+    const src = typeof modelSrc === 'string' ? modelSrc : modelSrc.src;
+    return CATEGORY_DEFAULTS.find((entry) => entry.asset.src === src)?.modelConfig;
+}
+export function categoryDownloadMetadata(model, addon) {
+    const entry = CATEGORY_DEFAULTS.find((candidate) => candidate.asset.addon === addon && (candidate.name === model || candidate.asset.src === model));
+    if (!entry?.companions) {
+        return undefined;
+    }
+    const assets = [entry.asset, ...entry.companions];
+    return {
+        companions: entry.companions.map((asset) => asset.src),
+        assetSizes: Object.fromEntries(assets.map((asset) => [asset.src, asset.expectedSize]))
+    };
+}
 export function defaultVoiceModel(category) {
     const entry = CATEGORY_DEFAULTS.find((candidate) => candidate.category === category);
     if (!entry) {
@@ -88,9 +149,7 @@ export function defaultTtsFor(profile) {
     return entry;
 }
 export function infraModelsFor(profile) {
-    const tts = defaultTtsFor(profile);
-    const rest = CATEGORY_DEFAULTS.filter((entry) => entry.category !== 'tts');
-    return [tts, ...rest].map((entry) => ({ name: entry.name, addon: entry.asset.addon }));
+    return CATEGORY_DEFAULTS.filter((entry) => entry.name === mediaModelNameFor(entry.category, profile)).map((entry) => ({ name: entry.name, addon: entry.asset.addon }));
 }
 export function mediaModelNameFor(category, profile) {
     if (category === 'tts') {
@@ -198,6 +257,16 @@ export const AUDIOGEN_CATALOG = [
         vae: AUDIOGEN_VAE_BF16,
         requiresGpu: true,
         defaultSteps: 24
+    },
+    {
+        choice: 'base',
+        name: 'ACE-Step-1.5-Base-Q4_K_M',
+        dit: AUDIOGEN_ACESTEP_V15_BASE_Q4_K_M,
+        textEnc: AUDIOGEN_QWEN3_EMBEDDING_0_6B_Q8_0,
+        lm: AUDIOGEN_ACESTEP_5HZ_LM_0_6B_Q8_0,
+        vae: AUDIOGEN_VAE_BF16,
+        requiresGpu: true,
+        defaultSteps: 50
     }
 ];
 export function musicDefaultSteps(choice) {
@@ -329,7 +398,11 @@ export function curatedListables() {
             companions: [],
             projection: entry.projection
         })),
-        ...CATEGORY_DEFAULTS.map((entry) => ({ name: entry.name, asset: entry.asset, companions: [] })),
+        ...CATEGORY_DEFAULTS.map((entry) => ({
+            name: entry.name,
+            asset: entry.asset,
+            companions: [...(entry.companions ?? [])]
+        })),
         ...DIFFUSION_CATALOG.map((entry) => ({
             name: entry.name,
             asset: entry.model,
@@ -362,6 +435,9 @@ export function customModelSrc(name, addon) {
     const entry = CUSTOM_MODELS.get(name);
     return entry && (entry.taskType ?? 'llm') === addon ? entry.src : undefined;
 }
+export function toolDialectFor(name) {
+    return LLM_CATALOG.find((entry) => entry.name === name)?.toolDialect ?? customToolDialect(name);
+}
 export function customToolDialect(name) {
     const entry = CUSTOM_MODELS.get(name);
     return entry
@@ -390,6 +466,11 @@ export function companionRegistryPaths() {
             map.set(entry.model.registryPath, companions);
         }
     }
+    for (const entry of CATEGORY_DEFAULTS) {
+        if (entry.companions) {
+            map.set(entry.asset.registryPath, entry.companions.map((asset) => asset.registryPath));
+        }
+    }
     for (const entry of AUDIOGEN_CATALOG) {
         map.set(entry.dit.registryPath, [
             entry.textEnc.registryPath,

package/dist-lib/lib/models/model-fit.d.ts

@@ -0,0 +1,55 @@
+import type { AssessModelFitInput, AssessModelFitResult } from '@qvac/sdk';
+export type ModelFitVerdict = AssessModelFitResult['verdict'];
+export type ModelFitBasis = AssessModelFitResult['basis'];
+export type ModelFitEvidence = NonNullable<AssessModelFitResult['evidence']>;
+export type FitModelType = 'llm' | 'embeddings';
+export interface FitQuery {
+    src: string;
+    sha256Checksum: string;
+    name: string;
+    registryPath?: string;
+    registrySource?: string;
+    modelType?: FitModelType;
+    contextTokens: number;
+}
+export interface FitAssessment {
+    verdict: ModelFitVerdict;
+    basis: ModelFitBasis;
+    evidence?: ModelFitEvidence | undefined;
+    reasons: string[];
+}
+export interface FitAsset {
+    readonly src: string;
+    readonly sha256Checksum: string;
+    readonly name: string;
+    readonly registryPath: string;
+    readonly registrySource: string;
+}
+export interface ContextChoice {
+    ctxSize: number;
+    slots: number;
+    fits: boolean;
+}
+export interface FitQueryOptions {
+    readonly native?: boolean;
+    readonly modelType?: FitModelType;
+}
+export type AssessModelFit = (input: AssessModelFitInput) => Promise<AssessModelFitResult>;
+export declare class ModelFit {
+    private readonly _assess;
+    private readonly _chosen;
+    private _rows;
+    constructor(assess: AssessModelFit);
+    static query(asset: FitAsset, contextTokens: number, { native, modelType }?: FitQueryOptions): FitQuery;
+    static confirms(assessed: FitAssessment | null): boolean;
+    assess(query: FitQuery): Promise<FitAssessment>;
+    rows(queries: () => ReadonlyMap<string, FitQuery>): Promise<ReadonlyMap<string, FitAssessment>>;
+    clearRows(): void;
+    chosen(key: string): number | undefined;
+    chosenSlots(key: string): number | undefined;
+    keep(key: string, { ctxSize, slots }: ContextChoice): void;
+    ladder(asset: FitAsset, tier: number, slots?: number): Promise<ContextChoice>;
+    private _steps;
+    private _ask;
+    settle(query: FitQuery): Promise<FitAssessment | null>;
+}

package/dist-lib/lib/models/model-fit.js

@@ -0,0 +1,102 @@
+import { errorMessage } from '../guards.js';
+import { hlog } from '../log.js';
+const CONTEXT_RUNGS = [32_768, 16_384, 8_192];
+export class ModelFit {
+    _assess;
+    _chosen = new Map();
+    _rows = null;
+    constructor(assess) {
+        this._assess = assess;
+    }
+    static query(asset, contextTokens, { native = false, modelType } = {}) {
+        const { src, sha256Checksum, name, registryPath, registrySource } = asset;
+        return native
+            ? { src, sha256Checksum, name, registryPath, registrySource, modelType, contextTokens }
+            : { src, sha256Checksum, name, modelType, contextTokens };
+    }
+    static confirms(assessed) {
+        return (assessed?.verdict === 'likely-fits' &&
+            (assessed.evidence === 'native-fit' || assessed.evidence === 'calibration'));
+    }
+    async assess(query) {
+        const [assessed] = await this._ask([query]);
+        return assessed;
+    }
+    rows(queries) {
+        if (this._rows) {
+            return this._rows;
+        }
+        const keyed = [...queries()];
+        const pending = this._ask(keyed.map(([, query]) => query)).then((assessed) => new Map(keyed.map(([key], i) => [key, assessed[i]])));
+        this._rows = pending;
+        pending.catch(() => {
+            if (this._rows === pending) {
+                this._rows = null;
+            }
+        });
+        return pending;
+    }
+    clearRows() {
+        this._rows = null;
+    }
+    chosen(key) {
+        return this._chosen.get(key)?.ctxSize;
+    }
+    chosenSlots(key) {
+        return this._chosen.get(key)?.slots;
+    }
+    keep(key, { ctxSize, slots }) {
+        if (!this._chosen.has(key)) {
+            this._chosen.set(key, { ctxSize, slots });
+        }
+    }
+    async ladder(asset, tier, slots = 1) {
+        const choice = await this._steps(asset, tier, { slots, native: true });
+        if (choice.fits || slots === 1) {
+            return choice;
+        }
+        return this._steps(asset, tier, { slots: 1, native: false });
+    }
+    async _steps(asset, tier, { slots, native }) {
+        const top = await this.settle(ModelFit.query(asset, tier * slots, { native }));
+        if (top?.verdict !== 'likely-too-large') {
+            return { ctxSize: tier, slots, fits: true };
+        }
+        for (const rung of CONTEXT_RUNGS.filter((size) => size < tier)) {
+            const assessed = await this.settle(ModelFit.query(asset, rung * slots));
+            if (ModelFit.confirms(assessed)) {
+                return { ctxSize: rung, slots, fits: true };
+            }
+            if (assessed?.verdict !== 'likely-too-large') {
+                break;
+            }
+        }
+        return { ctxSize: tier, slots, fits: false };
+    }
+    async _ask(queries) {
+        const result = await this._assess({
+            models: queries.map(({ contextTokens, modelType = 'llm', ...modelSrc }) => ({
+                modelSrc,
+                modelType,
+                modelConfig: { ctx_size: contextTokens }
+            })),
+            execution: 'sequential',
+            policy: 'interactive-v1'
+        });
+        return queries.map((_, i) => {
+            const model = result.models[i];
+            return {
+                verdict: model?.verdict ?? result.verdict,
+                basis: result.basis,
+                evidence: model ? model.evidence : result.evidence,
+                reasons: [...(model?.reasons ?? result.reasons)]
+            };
+        });
+    }
+    settle(query) {
+        return this.assess(query).catch((err) => {
+            hlog('fit estimate failed', { model: query.name, err: errorMessage(err) });
+            return null;
+        });
+    }
+}

package/dist-lib/lib/models/model-handle.d.ts

@@ -1,7 +1,9 @@
+import type { LoadFit } from '../contract-types.ts';
 export interface ModelResource {
     model: string;
     modelId: string;
     ctxSize?: number;
+    fit?: LoadFit | undefined;
 }
 export interface ModelHandleOptions {
     load(): Promise<ModelResource>;
@@ -15,8 +17,10 @@ export declare class ModelHandle {
     private readonly _onFailed;
     private _tracked;
     private _loading;
+    private _resource;
     constructor({ load, onLoaded, onFailed }: ModelHandleOptions);
     load(): Promise<ModelResource>;
     get loading(): boolean;
+    get resource(): ModelResource | null;
     peek(): Promise<ModelResource> | null;
 }

package/dist-lib/lib/models/model-handle.js

@@ -6,6 +6,7 @@ export class ModelHandle {
     _onFailed;
     _tracked = null;
     _loading = false;
+    _resource = null;
     constructor({ load, onLoaded, onFailed }) {
         const memo = retryMemo(load);
         this._load = memo;
@@ -25,13 +26,19 @@ export class ModelHandle {
                     report();
                 }
             };
-            attempt.then(() => settle(() => this._onLoaded?.()), (err) => settle(() => this._onFailed?.(err)));
+            attempt.then((resource) => settle(() => {
+                this._resource = resource;
+                this._onLoaded?.();
+            }), (err) => settle(() => this._onFailed?.(err)));
         }
         return attempt;
     }
     get loading() {
         return this._loading;
     }
+    get resource() {
+        return this._resource;
+    }
     peek() {
         return this._peek();
     }

package/dist-lib/lib/models/model-list.d.ts

@@ -1,6 +1,7 @@
 import type * as QvacSdk from '@qvac/sdk';
 import type { ModelAddon, ModelRegistrySdk } from './model-source.ts';
 import { type ModelTaskType } from './model-task.ts';
+import type { ModelFitBasis, ModelFitEvidence, ModelFitVerdict } from './model-fit.ts';
 export type ModelType = ModelAddon;
 export interface StaticCatalogEntry {
     readonly name: string;
@@ -22,6 +23,9 @@ export interface ModelInfo {
     actualSize?: number;
     label?: string;
     taskType: ModelTaskType;
+    fit?: ModelFitVerdict;
+    fitBasis?: ModelFitBasis;
+    fitEvidence?: ModelFitEvidence;
 }
 export type CatalogRow = Omit<ModelInfo, 'isLoading'>;
 export interface ListModelsOptions {

package/dist-lib/lib/models/model-pool.d.ts

@@ -3,7 +3,14 @@ import { ModelHandle } from './model-handle.ts';
 import { type ModelSrc } from './context.ts';
 import { type ModelAddon } from './model-source.ts';
 import type { GpuFaultState } from './sdk-faults.ts';
+import type { LoadedModelInfo } from '@qvac/sdk';
+import type { LoadedModel } from '../contract-types.ts';
 type SdkModule = typeof import('@qvac/sdk');
+export interface FitProbeSdk {
+    getLoadedModelInfo?(params: {
+        modelId: string;
+    }): Promise<Pick<LoadedModelInfo, 'fitProbe'>>;
+}
 export interface ModelState {
     status: 'loading' | 'downloading' | 'downloaded' | 'ready' | 'error' | 'unloading' | 'unloaded';
     model: string;
@@ -14,6 +21,13 @@ export type NotifyFn = (state: ModelState) => void;
 export declare class ModelUnloadError extends Error {
     constructor(cause: unknown);
 }
+export interface LlamaLoadKnobs {
+    threads?: number;
+    'batch-size'?: number;
+    'ubatch-size'?: number;
+    'kv-offload'?: false;
+    'n-cpu-moe'?: number;
+}
 export interface LlmDeviceConfig {
     device?: 'cpu' | 'gpu';
     gpuLayers?: number;
@@ -22,6 +36,10 @@ export interface LlmDeviceConfig {
     kvCacheV?: string;
     mainGpu?: () => Promise<string | undefined>;
     openclCacheDir?: () => string | undefined;
+    llama?: LlamaLoadKnobs;
+    slots?: number;
+    contextFor?: (key: string) => number | undefined;
+    slotsFor?: (key: string) => number | undefined;
 }
 export type TrackedLoad = Promise<string> & {
     requestId?: string;
@@ -38,15 +56,17 @@ export interface LoadTracker {
     undrain(): void;
 }
 export declare function createLoadTracker(): LoadTracker;
-export type ModelLifecycleSdk = Pick<SdkModule, 'unloadModel' | 'modelRegistrySearch'> & Partial<CancelSdk> & {
+export type ModelLifecycleSdk = Pick<SdkModule, 'unloadModel' | 'modelRegistrySearch'> & Partial<CancelSdk> & FitProbeSdk & {
     loadModel(params: {
         modelSrc: ModelSrc;
         modelType: 'llamacpp-completion';
-        modelConfig?: {
+        modelConfig?: LlamaLoadKnobs & {
             verbosity?: 0 | 1 | 2 | 3;
+            system_prompt?: string;
             tools?: boolean;
             stop_sequences?: readonly string[];
             ctx_size?: number;
+            parallel?: number;
             'cache-type-k'?: string;
             'cache-type-v'?: string;
             projectionModelSrc?: ModelSrc;
@@ -63,13 +83,11 @@ export interface ModelLoadOptions {
     kvCacheK?: string;
     kvCacheV?: string;
     projectionModel?: string;
+    llama?: LlamaLoadKnobs;
+    slots?: number;
 }
 export declare function resolveLoadOptions(opts?: ModelLoadOptions, deviceConfig?: LlmDeviceConfig): ModelLoadOptions;
 export declare function modelKey(model: string, opts?: ModelLoadOptions): string;
-export interface LoadedModel {
-    model: string;
-    refs: number;
-}
 interface PuddleEntry {
     readonly id: string;
     readonly refs: number;
@@ -109,7 +127,7 @@ export declare function awaitModelCapacity(pool: PoolLike, { cap, deadlineMs, ti
 export declare function evictModelWithin(pool: PoolLike, key: string, options: DeadlineOptions): Promise<boolean>;
 export declare function evictModel(pool: PoolLike, model: string, options: DeadlineOptions): Promise<boolean>;
 export type LoadSdk = () => Promise<ModelLifecycleSdk>;
-export type EmbedLifecycleSdk = Pick<SdkModule, 'unloadModel' | 'modelRegistrySearch'> & Partial<CancelSdk> & {
+export type EmbedLifecycleSdk = Pick<SdkModule, 'unloadModel' | 'modelRegistrySearch'> & Partial<CancelSdk> & FitProbeSdk & {
     loadModel(params: {
         modelSrc: ModelSrc;
         modelType: 'llamacpp-embedding';
@@ -134,7 +152,13 @@ interface LoadClockGuard {
     race: (pending: TrackedLoad) => Promise<string>;
 }
 export declare function loadClockGuard(model: string, sdk: Partial<CancelSdk>, { now, monoMs, sampleMs, jumpMs }?: LoadClockGuardOptions): LoadClockGuard;
-export declare function resolveCtxSize(loadSdk: LoadSdk, model: string, opts: ModelLoadOptions): Promise<number>;
+export interface ResolvedLoad {
+    opts: ModelLoadOptions;
+    deviceConfig: LlmDeviceConfig;
+}
+export declare function slotCount(model: string, { opts, deviceConfig }: ResolvedLoad): number;
+export declare function contextSize(model: string, modelSrc: ModelSrc, { opts, deviceConfig }: ResolvedLoad): number;
+export declare function resolveCtxSize(loadSdk: LoadSdk, model: string, load: ResolvedLoad): Promise<number>;
 type UnloadFailureFn = (modelId: string, error: unknown) => void;
 export interface ModelPoolOptions {
     deviceConfig?: LlmDeviceConfig;
@@ -150,7 +174,7 @@ export interface EmbedPoolOptions {
     onUnloadFailure?: UnloadFailureFn | undefined;
 }
 export declare function createEmbedPool(loadSdk: LoadEmbedSdk, notify: NotifyFn, { faults, batchSize, openclCacheDir, loads, onUnloadFailure }: EmbedPoolOptions): ModelPuddle;
-export type VoiceLifecycleSdk = Pick<SdkModule, 'unloadModel' | 'modelRegistrySearch'> & Partial<CancelSdk> & {
+export type VoiceLifecycleSdk = Pick<SdkModule, 'unloadModel' | 'modelRegistrySearch'> & Partial<CancelSdk> & FitProbeSdk & {
     loadModel(params: {
         modelSrc: ModelSrc;
         modelType: string;
@@ -163,9 +187,10 @@ export interface VoicePoolOptions {
     modelType: string;
     addon: ModelAddon;
     modelConfig?: Record<string, unknown> | undefined;
+    modelConfigFor?: (modelSrc: ModelSrc) => Readonly<Record<string, unknown>> | undefined;
     openclCacheDir?: (() => string | undefined) | undefined;
     loads?: LoadTracker;
     onUnloadFailure?: UnloadFailureFn | undefined;
 }
-export declare function createVoicePool(loadSdk: LoadVoiceSdk, notify: NotifyFn, { modelType, addon, modelConfig, openclCacheDir, loads, onUnloadFailure }: VoicePoolOptions): ModelPuddle;
+export declare function createVoicePool(loadSdk: LoadVoiceSdk, notify: NotifyFn, { modelType, addon, modelConfig, modelConfigFor, openclCacheDir, loads, onUnloadFailure }: VoicePoolOptions): ModelPuddle;
 export {};

package/dist-lib/lib/models/model-pool.js

@@ -73,7 +73,9 @@ export function resolveLoadOptions(opts = {}, deviceConfig = {}) {
         ctxSize: opts.ctxSize || deviceConfig.contextSize,
         kvCacheK: opts.kvCacheK ?? deviceConfig.kvCacheK,
         kvCacheV: opts.kvCacheV ?? deviceConfig.kvCacheV,
-        projectionModel: opts.projectionModel
+        projectionModel: opts.projectionModel,
+        llama: deviceConfig.llama,
+        slots: opts.slots ?? deviceConfig.slots
     };
 }
 export function modelKey(model, opts = {}) {
@@ -82,7 +84,13 @@ export function modelKey(model, opts = {}) {
         opts.ctxSize ?? '',
         opts.kvCacheK ?? '',
         opts.kvCacheV ?? '',
-        opts.projectionModel ?? ''
+        opts.projectionModel ?? '',
+        opts.llama?.threads ?? '',
+        opts.llama?.['batch-size'] ?? '',
+        opts.llama?.['ubatch-size'] ?? '',
+        opts.llama?.['kv-offload'] ?? '',
+        opts.llama?.['n-cpu-moe'] ?? '',
+        opts.slots && opts.slots > 1 ? opts.slots : ''
     ].join('\0');
 }
 class ModelPuddle extends ResourcePuddle {
@@ -119,7 +127,9 @@ export function loadedModels(pool) {
     for (const entry of pool.resources.values()) {
         if (entry.value.peek() === null)
             continue;
-        out.push({ model: entry.id.split('\0')[0], refs: entry.refs });
+        const model = modelOf(entry.id);
+        const fit = entry.value.resource?.fit;
+        out.push(fit ? { model, refs: entry.refs, fit } : { model, refs: entry.refs });
     }
     return out;
 }
@@ -339,19 +349,28 @@ export function loadClockGuard(model, sdk, { now = () => Date.now(), monoMs = mo
     };
     return { race };
 }
-export async function resolveCtxSize(loadSdk, model, opts) {
-    if (opts.ctxSize) {
-        return opts.ctxSize;
+export function slotCount(model, { opts, deviceConfig }) {
+    return deviceConfig.slotsFor?.(modelKey(model, opts)) ?? opts.slots ?? 1;
+}
+export function contextSize(model, modelSrc, { opts, deviceConfig }) {
+    return (opts.ctxSize ||
+        deviceConfig.contextFor?.(modelKey(model, opts)) ||
+        defaultContextSize(modelSrc, customContextLength(model)));
+}
+export async function resolveCtxSize(loadSdk, model, load) {
+    if (load.opts.ctxSize) {
+        return load.opts.ctxSize;
     }
     const sdk = await loadSdk();
-    return defaultContextSize(await resolveModelSrc(sdk, model));
+    return contextSize(model, await resolveModelSrc(sdk, model), load);
 }
 export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = createLoadTracker(), onUnloadFailure } = {}) {
     const load = async (model, opts) => {
         const sdk = await loadSdk();
         const modelSrc = await resolveModelSrc(sdk, model);
         const resolved = resolveLoadOptions(opts, deviceConfig);
-        const ctxSize = resolved.ctxSize || defaultContextSize(modelSrc, customContextLength(model));
+        const ctxSize = contextSize(model, modelSrc, { opts: resolved, deviceConfig });
+        const slots = slotCount(model, { opts: resolved, deviceConfig });
         const projectionModelSrc = resolved.projectionModel
             ? await resolveModelSrc(sdk, resolved.projectionModel)
             : undefined;
@@ -366,10 +385,13 @@ export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = cr
             ? deviceConfig.openclCacheDir()
             : undefined;
         const modelConfig = definedFields({
+            ...resolved.llama,
             verbosity: llmVerbosity(),
+            system_prompt: '',
             tools: true,
             stop_sequences: RUNAWAY_STOP_SEQUENCES,
-            ctx_size: ctxSize,
+            ctx_size: ctxSize * slots,
+            parallel: slots > 1 ? slots : undefined,
             'cache-type-k': kvCache.k,
             'cache-type-v': kvCache.v,
             projectionModelSrc,
@@ -388,7 +410,7 @@ export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = cr
             onProgress: (progress) => notify(progressState(model, progress))
         }), sdk));
         hlog('llm loaded', { model, loadMs: Date.now() - loadStarted });
-        return { model, modelId, ctxSize };
+        return { model, modelId, ctxSize, fit: await loadFit(sdk, model, modelId) };
     };
     return new ModelPuddle({
         open: (key, model = key, opts = {}) => reportedHandle(notify, model, () => load(model, opts)),
@@ -412,7 +434,7 @@ export function createEmbedPool(loadSdk, notify, { faults, batchSize = DEFAULT_E
             },
             onProgress: (progress) => notify(progressState(model, progress))
         }), sdk));
-        return { model, modelId };
+        return { model, modelId, fit: await loadFit(sdk, model, modelId) };
     };
     return new ModelPuddle({
         open: (key, model = key) => reportedHandle(notify, model, () => load(model)),
@@ -420,14 +442,16 @@ export function createEmbedPool(loadSdk, notify, { faults, batchSize = DEFAULT_E
         ...NO_IDLE_GC
     });
 }
-export function createVoicePool(loadSdk, notify, { modelType, addon, modelConfig, openclCacheDir, loads = createLoadTracker(), onUnloadFailure }) {
+export function createVoicePool(loadSdk, notify, { modelType, addon, modelConfig, modelConfigFor, openclCacheDir, loads = createLoadTracker(), onUnloadFailure }) {
     const wantsGpu = modelConfig?.['useGPU'] === true;
     let gpuRefused = false;
     const load = async (model) => {
         const sdk = await loadSdk();
         const modelSrc = await resolveModelSrc(sdk, model, addon);
+        const tuned = modelConfigFor?.(modelSrc);
+        const loadConfig = tuned ? { ...modelConfig, ...tuned } : modelConfig;
         const attempt = () => {
-            const config = gpuRefused ? { ...modelConfig, useGPU: false } : modelConfig;
+            const config = gpuRefused ? { ...loadConfig, useGPU: false } : loadConfig;
             const openclCache = gpuRefused ? undefined : openclCacheDir?.();
             return loads.track(sdk.loadModel({
                 modelSrc,
@@ -443,7 +467,7 @@ export function createVoicePool(loadSdk, notify, { modelType, addon, modelConfig
             gpuRefused = true;
             return attempt();
         }));
-        return { model, modelId };
+        return { model, modelId, fit: await loadFit(sdk, model, modelId) };
     };
     return new ModelPuddle({
         open: (key, model = key) => reportedHandle(notify, model, () => load(model)),
@@ -451,6 +475,31 @@ export function createVoicePool(loadSdk, notify, { modelType, addon, modelConfig
         ...NO_IDLE_GC
     });
 }
+async function loadFit(sdk, model, modelId) {
+    let probe;
+    try {
+        probe = (await sdk.getLoadedModelInfo?.({ modelId }))?.fitProbe;
+    }
+    catch (err) {
+        hlog('fit probe read failed', { model, err: errorMessage(err) });
+        return undefined;
+    }
+    if (!probe) {
+        return undefined;
+    }
+    const { verdict, reason, plan, projection } = probe;
+    const fit = {
+        verdict,
+        reason,
+        plan: plan && { nCtx: plan.nCtx, nGpuLayers: plan.nGpuLayers, nGpuDevices: plan.nGpuDevices },
+        deviceBytes: projection?.deviceBytes,
+        hostBytes: projection?.hostBytes
+    };
+    if (verdict === 'does-not-fit') {
+        hlog('loaded past a does-not-fit probe', { model, ...fit });
+    }
+    return fit;
+}
 function unloadOnGc(loadSdk, notify, onUnloadFailure) {
     return async (handles) => {
         const loadedHandles = [];

package/dist-lib/lib/models/music-catalog.d.ts

@@ -1,4 +1,4 @@
-export declare const MUSIC_MODEL_CHOICES: readonly ['turbo', 'turbo-hq', 'sft'];
+export declare const MUSIC_MODEL_CHOICES: readonly ['turbo', 'turbo-hq', 'sft', 'base'];
 export type MusicModelChoice = (typeof MUSIC_MODEL_CHOICES)[number];
 export declare const GENERATE_MUSIC_TOOL = "generate_music";
 export declare const DEFAULT_MUSIC_MODEL_CHOICE: MusicModelChoice;