@qvac/harness 0.2.5 → 0.2.6

package/dist/prebuilds-manifest.json

@@ -272,14 +272,6 @@
       "win32-arm64",
       "win32-x64"
     ],
-    "bare-tty/node_modules/bare-signals": [
-      "darwin-arm64",
-      "darwin-x64",
-      "linux-arm64",
-      "linux-x64",
-      "win32-arm64",
-      "win32-x64"
-    ],
     "bare-tty": [
       "darwin-arm64",
       "darwin-x64",

package/dist-lib/index.d.ts

@@ -82,6 +82,7 @@ export interface HarnessOptions {
     transcriptionDevice?: 'cpu' | 'gpu';
     credentialStore?: CredentialStoreFn;
     platform?: string;
+    completionModelCap?: number;
 }
 export interface CompletionContext {
     model: string;
@@ -133,6 +134,7 @@ export declare class Harness extends ReadyResource {
     private readonly _kvCacheV?;
     private readonly _transcriptionDevice?;
     private readonly _gpuStats?;
+    private readonly _completionModelCap;
     private _allSkills;
     private _credentialStore?;
     private _credentials;
@@ -145,7 +147,7 @@ export declare class Harness extends ReadyResource {
     private _capability;
     private _capabilityInflight;
     private readonly _modelFaults;
-    constructor({ tools, sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, credentialStore, platform, gpuStats }?: HarnessOptions);
+    constructor({ tools, sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats }?: HarnessOptions);
     _open(): Promise<void>;
     get suspended(): boolean;
     get skillNames(): string[];
@@ -238,6 +240,7 @@ export declare class Harness extends ReadyResource {
         signal?: BareAbortSignal;
     }): Promise<void>;
     completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }: CompletionContext): AsyncGenerator<CompletionEvent>;
+    private _makeRoomForCompletion;
     private _evictWedgedModel;
     private _evictAfterLeases;
 }

package/dist-lib/index.js

@@ -17,7 +17,7 @@ import { AbortedError, raceAbort } from "./lib/helpers/race-abort.js";
 import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, infraModelsFor, modelNameToCapabilityProfile, musicModelSet, setCustomModels } from "./lib/models/model-catalog.js";
 import { createModelStatusCache, invalidateModelStatusCache, listModels } from "./lib/models/model-list.js";
 import { deleteModelFiles, summarizeDelete } from "./lib/models/model-delete.js";
-import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, evictAllWithin, evictIdle, evictModelWithin, isResident, leaseCount, loadedModels, modelKey, releaseModel, resolveLoadOptions, progressState } from "./lib/models/model-pool.js";
+import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictModelWithin, isResident, leaseCount, loadedModels, modelKey, releaseModel, resolveLoadOptions, progressState } from "./lib/models/model-pool.js";
 import { resolveMainGpu } from "./lib/models/gpu-selection.js";
 import { openclCacheDir } from "./lib/models/opencl-cache.js";
 import { assertEmbedDevice, createEmbedConfig } from "./lib/models/embed-config.js";
@@ -68,6 +68,9 @@ const TRANSCRIPTION_MODEL_LABEL = 'transcription model';
 const DIFFUSION_EVICT_WAIT_MS = 15_000;
 const MUSIC_EVICT_WAIT_MS = 15_000;
 const WEDGED_EVICT_WAIT_MS = 10_000;
+const MODEL_ROOM_WAIT_MS = 300_000;
+const WARM_ROOM_WAIT_MS = 2_000;
+const DEFAULT_COMPLETION_MODEL_CAP = 1;
 const WEDGED_EVICT_LEASED_MS = 120_000;
 const TTS_MODEL_CONFIG = {
     ttsEngine: 'supertonic',
@@ -104,6 +107,7 @@ export class Harness extends ReadyResource {
     _kvCacheV;
     _transcriptionDevice;
     _gpuStats;
+    _completionModelCap;
     _allSkills;
     _credentialStore;
     _credentials = new Map();
@@ -116,7 +120,7 @@ export class Harness extends ReadyResource {
     _capability = null;
     _capabilityInflight = null;
     _modelFaults = new Map();
-    constructor({ tools = [], sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, credentialStore, platform, gpuStats } = {}) {
+    constructor({ tools = [], sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats } = {}) {
         super();
         this._sdk = sdkLoader ?? createSdkLoader();
         this._llmDevice = llmDevice;
@@ -126,6 +130,7 @@ export class Harness extends ReadyResource {
         this._kvCacheK = kvCacheK;
         this._kvCacheV = kvCacheV;
         this._transcriptionDevice = transcriptionDevice;
+        this._completionModelCap = completionModelCap ?? DEFAULT_COMPLETION_MODEL_CAP;
         const embedConfig = createEmbedConfig({
             device: embedDevice,
             gpuLayers: embedGpuLayers,
@@ -168,7 +173,7 @@ export class Harness extends ReadyResource {
                 onBeforeLoad: async (signal) => {
                     if (this._models.destroyed)
                         return;
-                    const evicted = await evictAllWithin(this._models, DIFFUSION_EVICT_WAIT_MS, undefined, signal);
+                    const evicted = await awaitModelCapacity(this._models, 1, DIFFUSION_EVICT_WAIT_MS, undefined, signal);
                     if (signal?.aborted)
                         throw new AbortedError();
                     if (!evicted) {
@@ -187,7 +192,7 @@ export class Harness extends ReadyResource {
                 onBeforeLoad: async (signal) => {
                     if (this._models.destroyed)
                         return;
-                    const evicted = await evictAllWithin(this._models, MUSIC_EVICT_WAIT_MS, undefined, signal);
+                    const evicted = await awaitModelCapacity(this._models, 1, MUSIC_EVICT_WAIT_MS, undefined, signal);
                     if (signal?.aborted)
                         throw new AbortedError();
                     if (!evicted) {
@@ -927,7 +932,8 @@ export class Harness extends ReadyResource {
         const pool = this._models;
         hlog('warm model', { model, resident: isResident(pool, key) });
         if (!isResident(pool, key)) {
-            await evictIdle(pool);
+            if (!(await this._makeRoomForCompletion(signal, WARM_ROOM_WAIT_MS)))
+                return;
             await this._imageRuntime?.unloadIdle().catch(noop);
         }
         try {
@@ -953,13 +959,13 @@ export class Harness extends ReadyResource {
         const resident = isResident(pool, key);
         hlog('completion start', { model, resident, projection: Boolean(projectionModel) });
         if (!resident) {
-            await evictIdle(pool);
+            await this._makeRoomForCompletion(signal);
             await this._imageRuntime?.unloadIdle().catch(noop);
             await this._musicRuntime?.unloadIdle().catch(noop);
         }
         const acquireRound = async () => {
             if (!isResident(pool, key)) {
-                await evictIdle(pool);
+                await this._makeRoomForCompletion(signal);
                 await this._imageRuntime?.unloadIdle().catch(noop);
                 await this._musicRuntime?.unloadIdle().catch(noop);
             }
@@ -1052,6 +1058,16 @@ export class Harness extends ReadyResource {
             void this._musicRuntime?.unloadIdle().catch(noop);
         }
     }
+    async _makeRoomForCompletion(signal, deadlineMs = MODEL_ROOM_WAIT_MS) {
+        const roomed = await awaitModelCapacity(this._models, this._completionModelCap, deadlineMs, 250, signal);
+        if (!roomed) {
+            hlog('model room wait gave up', {
+                cap: this._completionModelCap,
+                loaded: loadedModels(this._models).length
+            });
+        }
+        return roomed;
+    }
     async _evictWedgedModel(model, key) {
         const faults = (this._modelFaults.get(model) ?? 0) + 1;
         this._modelFaults.set(model, faults);

package/dist-lib/lib/models/model-pool.d.ts

@@ -72,7 +72,7 @@ export declare function loadedModels(pool: PoolLike): LoadedModel[];
 export declare function isResident(pool: PoolLike, key: string): boolean;
 export declare function leaseCount(pool: PoolLike, key: string): number;
 export declare function evictIdle(pool: PoolLike): Promise<void>;
-export declare function evictAllWithin(pool: PoolLike, deadlineMs: number, tickMs?: number, signal?: {
+export declare function awaitModelCapacity(pool: PoolLike, cap: number, deadlineMs: number, tickMs?: number, signal?: {
     aborted?: boolean;
 }): Promise<boolean>;
 export declare function evictModelWithin(pool: PoolLike, key: string, deadlineMs: number, tickMs?: number): Promise<boolean>;

package/dist-lib/lib/models/model-pool.js

@@ -103,7 +103,8 @@ export async function evictIdle(pool) {
         await pool.gc();
     }
 }
-export async function evictAllWithin(pool, deadlineMs, tickMs = 250, signal) {
+export async function awaitModelCapacity(pool, cap, deadlineMs, tickMs = 250, signal) {
+    const roomed = () => loadedModels(pool).length < cap;
     let done = false;
     let timer = null;
     const expired = new Promise((resolve) => {
@@ -114,7 +115,7 @@ export async function evictAllWithin(pool, deadlineMs, tickMs = 250, signal) {
     });
     const evicting = (async () => {
         await evictIdle(pool);
-        while (loadedModels(pool).length > 0) {
+        while (!roomed()) {
             if (done || signal?.aborted)
                 return false;
             await new Promise((resolve) => setTimeout(resolve, tickMs));

package/dist-lib/lib/spawn.d.ts

@@ -20,6 +20,7 @@ export interface HarnessLlmConfig {
     contextSize?: number;
     kvCacheK?: string;
     kvCacheV?: string;
+    completionModelCap?: number;
 }
 export interface HarnessChildOptions {
     runner: StowRunner;

package/dist-lib/schema/generate.d.ts

@@ -13,8 +13,12 @@ interface HRPCHandler {
 interface HRPCBuilderLike {
     handlers: ReadonlyArray<HRPCHandler>;
 }
+interface CallSiteOptions {
+    readonly optionalRequests?: ReadonlySet<string>;
+    readonly exportName?: string;
+}
 export declare function writeBind(builder: HRPCBuilderLike, outFile: string): void;
-export declare function writeClient(builder: HRPCBuilderLike, outFile: string, exportName?: string): void;
-export declare function writeCapabilities(builder: HRPCBuilderLike, outFile: string): void;
+export declare function writeClient(builder: HRPCBuilderLike, outFile: string, options?: CallSiteOptions | string): void;
+export declare function writeCapabilities(builder: HRPCBuilderLike, outFile: string, options?: CallSiteOptions): void;
 export declare function writeConfigTypes(outFile: string): void;
 export {};

package/dist-lib/schema/generate.js

@@ -13,6 +13,7 @@ function camel(name, prefix = '') {
     return prefix + cased.join('');
 }
 const HEADER = '// This file is autogenerated by schema/build.js\n// Do not edit manually.\n\n';
+const DEFAULT_OPTIONAL_REQUESTS = new Set(['@assistant/empty']);
 export function writeBind(builder, outFile) {
     let s = `${HEADER}import { pipeline, Readable } from "streamx";\n\nfunction noop() {}\n\nexport function bindApi(rpc, api) {\n`;
     for (const handler of builder.handlers) {
@@ -56,24 +57,28 @@ export function writeBind(builder, outFile) {
     s += '}\n';
     fs.writeFileSync(outFile, s);
 }
-export function writeClient(builder, outFile, exportName = 'engineClient') {
+export function writeClient(builder, outFile, options = {}) {
+    const callSite = typeof options === 'string' ? { exportName: options } : options;
+    const { optionalRequests = DEFAULT_OPTIONAL_REQUESTS, exportName = 'engineClient' } = callSite;
     let s = `${HEADER}function noop() {}\n\nfunction armed(stream) {\n  stream.on("error", noop);\n  return stream;\n}\n\nexport function ${exportName}(rpc) {\n  return {\n`;
     for (const handler of builder.handlers) {
         const method = camel(handler.name);
+        const input = optionalRequests.has(handler.request?.name ?? '') ? 'input ?? {}' : 'input';
         if (handler.request?.stream && handler.response?.stream) {
             s += `    ${method}: () => armed(rpc.${method}()),\n`;
         }
         else if (handler.response?.stream) {
-            s += `    ${method}: (input) => armed(rpc.${method}(input)),\n`;
+            s += `    ${method}: (input) => armed(rpc.${method}(${input})),\n`;
         }
         else {
-            s += `    ${method}: (input) => rpc.${method}(input),\n`;
+            s += `    ${method}: (input) => rpc.${method}(${input}),\n`;
         }
     }
     s += '  };\n}\n';
     fs.writeFileSync(outFile, s);
 }
-export function writeCapabilities(builder, outFile) {
+export function writeCapabilities(builder, outFile, options = {}) {
+    const { optionalRequests = DEFAULT_OPTIONAL_REQUESTS } = options;
     let s = HEADER;
     s += "import type * as T from './hyperschema/types.d.ts'\n\n";
     s +=
@@ -112,7 +117,7 @@ export function writeCapabilities(builder, outFile) {
         const method = camel(handler.name);
         const req = `T.${tsName(handler.request.name)}`;
         const res = `T.${tsName(handler.response.name)}`;
-        const input = handler.request.name === '@assistant/empty' ? `input?: ${req}` : `input: ${req}`;
+        const input = optionalRequests.has(handler.request.name) ? `input?: ${req}` : `input: ${req}`;
         if (handler.request?.stream && handler.response?.stream) {
             client += `  ${method}(): UploadStream<${req}, ${res}>\n`;
             server += `  ${method}(stream: AsyncIterable<${req}> | HandlerFrames<${req}>): AsyncIterable<${res}>\n`;