@qvac/harness 0.8.6 → 0.8.8

package/dist-lib/index.d.ts

@@ -99,7 +99,9 @@ export interface CompletionContext {
     model: string;
     agent?: Agent;
     messages: Message[];
+    typedText?: string;
     cacheKey?: string;
+    session?: string;
     thinkingDefault?: string;
     toolConfigs?: Record<string, Record<string, unknown>>;
     generationParams?: LlmGenerationParams;
@@ -189,6 +191,7 @@ export declare class Harness extends ReadyResource {
     private _sizeLoads;
     private _capBytes;
     private readonly _reserved;
+    private readonly _contexts;
     private readonly _admission;
     private readonly _byHand;
     constructor({ tools, sdkLoader, skillsDir, modelsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats }?: HarnessOptions);
@@ -290,14 +293,22 @@ export declare class Harness extends ReadyResource {
     private _curatedFit;
     private _admitByHand;
     private _replacedBy;
+    private _replacedEntries;
+    private _alone;
     private _sizeByHand;
     private _loadByHand;
     private _admit;
     private _makeRoomForGenerator;
     private _yieldGenerators;
     private _beforeLoad;
+    private _reserveByHand;
     private _reserve;
     private _sizeLoad;
+    private _chooseContext;
+    private _capRoom;
+    private _contextProfile;
+    private _trainedContext;
+    private _ctxSizeOf;
     private _makeRoom;
     private _capVictims;
     private _victimRank;

package/dist-lib/index.js

@@ -28,6 +28,8 @@ import { deleteModelFiles, summarizeDelete } from './lib/models/model-delete.js'
 import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictIdleWhere, evictIdleWithin, evictModel, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, loadingModels, settledEntries, settledModels, modelKey, modelOf, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
 import { memoryCapBytes, nextVictim, VictimRank } from './lib/models/memory-cap.js';
 import { defaultContextSize } from './lib/models/context.js';
+import { ContextChoice } from './lib/models/context-choice.js';
+import { parseGgufHeader, readGgufHeaderBytes } from './lib/models/gguf-header.js';
 import { resolveMainGpu } from './lib/models/gpu-selection.js';
 import { createIdleReaper } from './lib/models/idle-unload.js';
 import { openclCacheDir } from './lib/models/opencl-cache.js';
@@ -163,8 +165,9 @@ export class Harness extends ReadyResource {
     _sizeLoads = false;
     _capBytes = null;
     _reserved = new Map();
+    _contexts = new ContextChoice(() => this._contextProfile());
     _admission = new ScopeLock();
-    _byHand = new Set();
+    _byHand = new Map();
     constructor({ tools = [], sdkLoader, skillsDir, modelsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats } = {}) {
         super();
         this._sdk = sdkLoader ?? createSdkLoader();
@@ -682,7 +685,7 @@ export class Harness extends ReadyResource {
         const key = modelKey(model, loadOpts);
         const pool = this._models;
         if (!isResident(pool, key)) {
-            await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
+            await this._makeRoomForCompletion(model, { ctxSize: this._ctxSizeOf(key, loadOpts), signal });
         }
         try {
             const { modelId } = await raceAbort(pool.open(key, model, loadOpts).load(), signal);
@@ -721,6 +724,7 @@ export class Harness extends ReadyResource {
             deviceConfig: this._deviceConfig(),
             loads: this._loads,
             beforeLoad: (load) => this._beforeLoad(load),
+            chooseContext: (load, tier) => this._chooseContext(load, tier),
             hold: this._hold
         });
     }
@@ -1346,7 +1350,9 @@ export class Harness extends ReadyResource {
     }
     async _admitByHand(model, target) {
         if (this._capBytes === null) {
-            return target.pool !== this._models || this._fits(model, target.ctxSize);
+            return (target.pool !== this._models ||
+                this._alone(model, target) ||
+                this._fits(model, target.ctxSize));
         }
         return this._admit(async () => {
             if (isResident(target.pool, target.key) || this._byHand.has(target.key)) {
@@ -1354,18 +1360,14 @@ export class Harness extends ReadyResource {
             }
             const sized = await this._sizeByHand(model, target);
             const capBytes = this._capBytes;
-            const settled = this._pools().flatMap((pool) => settledEntries(pool));
-            const replacedBy = this._replacedBy(model);
-            const replaced = target.pool === this._models
-                ? settledEntries(this._models).filter(({ key, refs }) => refs === 0 && replacedBy(key))
-                : [];
+            const replaced = this._replacedEntries(model, target);
             const used = this._usedBytes() - replaced.reduce((sum, { bytes }) => sum + (bytes ?? 0), 0);
-            const alone = this._reserved.size === 0 && settled.length === replaced.length;
+            const alone = this._alone(model, target);
             const fits = capBytes === null || alone || sized.bytes === undefined || used + sized.bytes <= capBytes;
-            if (sized.tooLarge || !fits) {
+            if ((sized.tooLarge && !alone) || !fits) {
                 return false;
             }
-            this._byHand.add(target.key);
+            this._byHand.set(target.key, sized.ctxSize);
             if (sized.bytes !== undefined) {
                 this._reserved.set(target.key, sized.bytes);
             }
@@ -1375,19 +1377,35 @@ export class Harness extends ReadyResource {
     _replacedBy(model) {
         return (key) => modelOf(key) === model && !this._replying.includes(key);
     }
-    _sizeByHand(model, target) {
+    _replacedEntries(model, target) {
+        const replacedBy = this._replacedBy(model);
+        return target.pool === this._models
+            ? settledEntries(this._models).filter(({ key, refs }) => refs === 0 && replacedBy(key))
+            : [];
+    }
+    _alone(model, target) {
+        const settled = this._pools().flatMap((pool) => settledEntries(pool));
+        const loading = this._pools().some((pool) => loadingModels(pool).length > 0);
+        return (!loading &&
+            this._reserved.size === 0 &&
+            settled.length === this._replacedEntries(model, target).length);
+    }
+    async _sizeByHand(model, target) {
         const addon = modelAddonOf(model);
         const modelSrc = curatedModelAsset(model, addon) ?? customModelSrc(model, addon) ?? model;
-        const contextTokens = target.ctxSize || defaultContextSize(modelSrc, customContextLength(model));
-        return this._sizeLoad({
+        const ctxSize = target.pool === this._models
+            ? target.ctxSize || defaultContextSize(modelSrc, customContextLength(model))
+            : undefined;
+        const sized = await this._sizeLoad({
             key: target.key,
             model,
             modelSrc,
             modelType: target.modelType,
-            modelConfig: target.pool === this._models
-                ? { ctx_size: contextTokens, projectionModelSrc: this._projectionFor(model) }
-                : undefined
+            modelConfig: ctxSize === undefined
+                ? undefined
+                : { ctx_size: ctxSize, projectionModelSrc: this._projectionFor(model) }
         });
+        return { ...sized, ctxSize };
     }
     async _loadByHand(target, signal) {
         const loading = target.open().load();
@@ -1429,11 +1447,22 @@ export class Harness extends ReadyResource {
     async _beforeLoad(load) {
         if (!this._sizeLoads)
             return undefined;
-        const bytes = this._byHand.delete(load.key)
-            ? this._reserved.get(load.key)
+        const bytes = this._byHand.has(load.key)
+            ? await this._admit(() => this._reserveByHand(load))
             : await this._admit(() => this._reserve(load));
         return { bytes, release: () => this._reserved.delete(load.key) };
     }
+    async _reserveByHand(load) {
+        const admittedAt = this._byHand.get(load.key);
+        this._byHand.delete(load.key);
+        if (load.modelConfig?.['ctx_size'] !== admittedAt) {
+            const { bytes } = await this._sizeLoad(load);
+            if (bytes !== undefined) {
+                this._reserved.set(load.key, bytes);
+            }
+        }
+        return this._reserved.get(load.key);
+    }
     async _reserve(load) {
         const sized = await this._sizeLoad(load);
         if (this._capBytes !== null) {
@@ -1461,17 +1490,60 @@ export class Harness extends ReadyResource {
             const bytes = fit?.estimate?.upperBoundBytes ?? fit?.floorBytes ?? downloaded;
             return {
                 bytes: bytes === undefined ? undefined : bytes + projection,
-                tooLarge: fit?.verdict === 'likely-too-large'
+                tooLarge: fit?.verdict === 'likely-too-large',
+                confirmed: fit?.verdict === 'likely-fits' &&
+                    (fit.evidence === 'calibration' || fit.evidence === 'native-fit')
             };
         }
         catch (err) {
             hlog('fit estimate failed', { model, err: errorMessage(err) });
             return {
                 bytes: downloaded === undefined ? undefined : downloaded + projection,
-                tooLarge: false
+                tooLarge: false,
+                confirmed: false
             };
         }
     }
+    async _chooseContext(load, tier) {
+        if (isTranslationModel(load.model)) {
+            return tier;
+        }
+        const ctxSize = await this._contexts.choose(load.key, tier, {
+            trained: () => this._trainedContext(load),
+            fits: async (rung) => {
+                const sized = await this._sizeLoad({
+                    ...load,
+                    modelConfig: { ...load.modelConfig, ctx_size: rung }
+                });
+                const room = this._capRoom(load.key);
+                return sized.confirmed && (room === null || (sized.bytes ?? Infinity) <= room);
+            }
+        });
+        hlog('context chosen', { model: load.model, tier, ctxSize });
+        return ctxSize;
+    }
+    _capRoom(key) {
+        const cap = this._capBytes;
+        return cap === null ? null : cap - (this._usedBytes() - (this._reserved.get(key) ?? 0));
+    }
+    async _contextProfile() {
+        const { capabilityProfile, memoryBudget } = await this.capability();
+        return memoryBudget ? capabilityProfile : undefined;
+    }
+    async _trainedContext({ model, modelSrc }) {
+        const custom = customContextLength(model);
+        if (custom || typeof modelSrc === 'string') {
+            return custom;
+        }
+        const sdk = await this._sdk.load();
+        const info = await sdk.getModelInfo({ name: modelSrc.name }).catch(() => null);
+        const file = info?.cacheFiles[0]?.path;
+        const bytes = file ? await readGgufHeaderBytes(file) : null;
+        return (bytes && parseGgufHeader(bytes)?.contextLength) || undefined;
+    }
+    _ctxSizeOf(key, loadOpts) {
+        return loadOpts.ctxSize || this._contexts.chosen(key);
+    }
     async _makeRoom(need, room) {
         const capBytes = this._capBytes;
         if (capBytes === null) {
@@ -1563,7 +1635,7 @@ export class Harness extends ReadyResource {
                 pool: this._models,
                 key,
                 modelType: 'llamacpp-completion',
-                ctxSize: loadOpts.ctxSize,
+                ctxSize: this._ctxSizeOf(key, loadOpts),
                 open: () => this._models.open(key, model, loadOpts)
             };
         }
@@ -1625,7 +1697,7 @@ export class Harness extends ReadyResource {
     completion(context) {
         return this._busyStream(() => this._completion(context));
     }
-    async *_completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, captureInferenceCalls, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }) {
+    async *_completion({ model, agent, messages, typedText, cacheKey, session, thinkingDefault, toolConfigs, generationParams, responseFormat, captureInferenceCalls, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }) {
         if (!this.opened)
             await this.ready();
         if (this.suspended)
@@ -1637,12 +1709,15 @@ export class Harness extends ReadyResource {
         const resident = isResident(pool, key);
         hlog('completion start', { model, resident, projection: Boolean(projectionModel) });
         if (!resident) {
-            await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
+            await this._makeRoomForCompletion(model, { ctxSize: this._ctxSizeOf(key, loadOpts), signal });
             await this._yieldGenerators();
         }
         const acquireRound = async () => {
             if (!isResident(pool, key)) {
-                await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
+                await this._makeRoomForCompletion(model, {
+                    ctxSize: this._ctxSizeOf(key, loadOpts),
+                    signal
+                });
                 await this._yieldGenerators();
             }
             try {
@@ -1690,11 +1765,13 @@ export class Harness extends ReadyResource {
                 agent,
                 ctxSize,
                 cacheKey,
+                session,
                 thinkingDefault,
                 generationParams,
                 responseFormat,
                 captureInferenceCalls,
                 messages: turns,
+                typedText,
                 tools: this._tools,
                 skillCatalog: await this._completionSkills(agent?.id ?? null, model),
                 toolConfigs,

package/dist-lib/lib/capabilities/completion.d.ts

@@ -125,6 +125,7 @@ export interface ToolExecuteContext {
     skills?: readonly string[] | undefined;
     skillsLoadedThisTurn?: readonly string[] | undefined;
     ctxSize?: number | undefined;
+    session?: string | undefined;
     image?: ImageSettings | undefined;
     settings?: Record<string, unknown> | undefined;
     turnImages?: {
@@ -184,6 +185,7 @@ export interface ToolDefinition {
     toolset: Toolset | null;
     validate?: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => string | null;
     effect: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => ToolEffect;
+    stableRead?: true;
     resource?: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => ResourceDescriptor | null;
     compiled?: true;
     execute: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => Promise<unknown>;
@@ -296,9 +298,6 @@ export interface CompletionSdk {
     cancel?(params: {
         requestId: string;
     }): Promise<void>;
-    deleteCache?(params: {
-        kvCacheKey: string;
-    }): Promise<unknown>;
 }
 export interface RoundModelLease {
     modelId: string;
@@ -311,10 +310,12 @@ interface CompletionArgs {
     agent?: Agent;
     ctxSize?: number;
     cacheKey?: string;
+    session?: string;
     thinkingDefault?: string;
     generationParams?: LlmGenerationParams;
     responseFormat?: ResponseFormat;
     messages: Message[];
+    typedText?: string;
     tools?: ToolDefinition[];
     skillCatalog?: readonly CompletionSkillEntry[];
     approve?: ApproveFn;
@@ -335,5 +336,5 @@ export declare function toSdkTool(tool: ToolDefinition): SdkTool;
 export type CompletionSkillEntry = SkillCatalogEntry & {
     readonly status?: SkillState;
 };
-export declare function completion(sdk: CompletionSdk, { modelId, acquireRound, model, agent, ctxSize, cacheKey, thinkingDefault, generationParams: generationParamsOverride, responseFormat, messages: storedMessages, tools: allTools, skillCatalog, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal, toolConfigs, fsRoots, fsWriteRoots, captureInferenceCalls }: CompletionArgs): AsyncGenerator<CompletionEvent>;
+export declare function completion(sdk: CompletionSdk, { modelId, acquireRound, model, agent, ctxSize, cacheKey, session, thinkingDefault, generationParams: generationParamsOverride, responseFormat, messages: storedMessages, typedText, tools: allTools, skillCatalog, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal, toolConfigs, fsRoots, fsWriteRoots, captureInferenceCalls }: CompletionArgs): AsyncGenerator<CompletionEvent>;
 export {};