@qvac/harness 0.8.5 → 0.8.7

package/README.md

@@ -268,6 +268,55 @@ unloaded, and the next call loads what it needs again. The clock covers the
 whole device: any call resets it, whichever model it uses. `0`, the default, is
 Never.
 
+## Keeping several models in memory
+
+```js
+await harness.setModelMemoryCap(85)
+```
+
+Loaded models may fill that percent of the device's `memoryBudget`, so two chat
+models that fit stay loaded and a chat switching between them loads neither
+again. A model is unloaded only when the next load would not fit, one at a time:
+idle image and music models first, then embedding, speech and transcription
+models, then chat models a request loaded, then chat models loaded by hand, the
+least recently used first within each. A model a request holds stays, and a
+load past the cap goes ahead once nothing else can give way, so a model larger
+than the cap loads alone. A manual `loadModel`, of any kind, still never unloads
+another model: beside what is loaded it must fit the cap, or it is refused with
+`reason: 'memory'`. Lowering the cap unloads idle models past it at once. Loads
+under way count by their estimate, so two at once never count on the same room.
+
+Each load is sized by the SDK's `assessModelFit` for exactly the load it runs,
+falling back to the download size, and `list()` rows of loaded models carry that
+estimate as `memoryBytes`. `memoryUsage()` answers what every loaded model holds,
+whichever list rows a host reads. `0`, and any cap on a mobile device, keeps one chat
+model a request loaded, as without a cap; a host that never calls it keeps that
+behaviour and sizes nothing.
+
+## Holding model downloads
+
+```js
+await harness.setDownloadsAllowed({ allowed: false })
+```
+
+With downloads not allowed, a load whose model is not on the device does not
+download it. This covers every load the harness makes: chat, embedding, speech,
+transcription, image, music and translation. The load emits one `model` event
+with `status: 'held'`, the model's download name and its `addon`, then waits.
+Each `download()` that completes re-checks every waiting load, and a load goes
+ahead once its model is on the device, or once
+`setDownloadsAllowed({ allowed: true })` arrives, with no retry from the caller.
+Aborting the caller's signal ends the wait with `AbortedError`, and a suspend or
+close ends every wait. A wait that ends without loading reports `error` with
+`message: 'aborted'`. A released wait reports nothing of its own: the `loading`
+and `ready` after it name the model the way the pool does, which for embedding,
+speech and transcription is not the row name. So a host clears `held` when its
+`download()` of the row completes, when it allows downloads again, or on that
+`error`. A model already on the device loads with no wait. The
+check is the one `list({ cachedOnly: true })` makes, so a model counts as on the
+device only when its companions and projection are too. `download()` itself is
+never held. Downloads are allowed by default.
+
 ## Office extraction
 
 Hosts can supply the bundled Python extractor to their document pipeline:

package/dist-lib/index.d.ts

@@ -10,7 +10,7 @@ import { type EmbedContext } from './lib/capabilities/embed.ts';
 import { type KvCacheDeletionResult } from './lib/capabilities/kv-cache.ts';
 import type { LlmGenerationParams } from './lib/models/llm-presets.ts';
 import { type SpeakContext, type SpeakResult, type SpeakStreamContext, type TranscribeContext } from './lib/capabilities/voice.ts';
-import type { CapabilityInfo, CredentialEntry, HarnessDiagnostics, InfraModel, LoadModelResult, SkillsOptions, SkillWriteResult, UnloadModelContext, UnloadModelResult } from './lib/contract-types.ts';
+import type { CapabilityInfo, CredentialEntry, HarnessDiagnostics, InfraModel, LoadModelResult, MemoryUsage, SkillsOptions, SkillWriteResult, UnloadModelContext, UnloadModelResult } from './lib/contract-types.ts';
 import { type CustomModelEntry } from './lib/models/model-catalog.ts';
 import { type ListModelsOptions, type ModelInfo } from './lib/models/model-list.ts';
 import { type DeleteModelSummary } from './lib/models/model-delete.ts';
@@ -54,7 +54,7 @@ export { isAvailableState, SKILL_STATE, type SkillState } from './lib/tools/skil
 export { connectHarness, HARNESS_DIED, RemoteHarness } from './lib/connect.ts';
 export type { KvCacheDeletionResult } from './lib/capabilities/kv-cache.ts';
 export type { AgentKnobs, ApprovalCall, ChatMessage, CredentialEntry, DownloadContext, HarnessContract, HarnessDiagnostics, HarnessEvent, HarnessInfo, HarnessStatus, InfraModel, ModelState, SkillsOptions, SkillWriteResult } from './lib/contract-types.ts';
-import type { HarnessInfo } from './lib/contract-types.ts';
+import type { DownloadsAllowedOptions, HarnessInfo } from './lib/contract-types.ts';
 export { serveHarness } from './lib/serve.ts';
 export { hlog, setHarnessLog, setHarnessLogSink } from './lib/log.ts';
 export { harnessHooks } from './lib/hooks/registry.ts';
@@ -99,7 +99,9 @@ export interface CompletionContext {
     model: string;
     agent?: Agent;
     messages: Message[];
+    typedText?: string;
     cacheKey?: string;
+    session?: string;
     thinkingDefault?: string;
     toolConfigs?: Record<string, Record<string, unknown>>;
     generationParams?: LlmGenerationParams;
@@ -142,6 +144,8 @@ export declare class Harness extends ReadyResource {
     private readonly _pinned;
     private readonly _replying;
     private readonly _modelStatus;
+    private readonly _downloads;
+    private readonly _hold;
     private _models;
     private _embedModels;
     private readonly _retiredEmbedModels;
@@ -184,6 +188,12 @@ export declare class Harness extends ReadyResource {
     private _capabilityInflight;
     private readonly _modelFaults;
     private readonly _idle;
+    private _sizeLoads;
+    private _capBytes;
+    private readonly _reserved;
+    private readonly _contexts;
+    private readonly _admission;
+    private readonly _byHand;
     constructor({ tools, sdkLoader, skillsDir, modelsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats }?: HarnessOptions);
     _open(): Promise<void>;
     get suspended(): boolean;
@@ -213,6 +223,10 @@ export declare class Harness extends ReadyResource {
     private _projectionFor;
     setCustomModels(entries: ReadonlyArray<CustomModelEntry>): void;
     setModelIdleTimeout(ms: number): void;
+    setModelMemoryCap(percent: number): Promise<void>;
+    memoryUsage(): Promise<MemoryUsage>;
+    setDownloadsAllowed({ allowed }: DownloadsAllowedOptions): void;
+    private _isModelCached;
     previewModelSource(context: PreviewModelSourceContext): Promise<ModelSourcePreview>;
     private _assessFit;
     describe(): Promise<HarnessInfo>;
@@ -234,6 +248,7 @@ export declare class Harness extends ReadyResource {
     private _notifyModel;
     private _collectRetiredEmbedPool;
     private _releaseEmbedResource;
+    private _collectRetiredEmbedPoolAfter;
     private _releaseEmbedResourceAfter;
     private _destroyEmbedPools;
     private _enqueueEmbedControl;
@@ -275,6 +290,31 @@ export declare class Harness extends ReadyResource {
     private _loadAndRelease;
     private _unloadIdleGenerators;
     private _fits;
+    private _curatedFit;
+    private _admitByHand;
+    private _replacedBy;
+    private _replacedEntries;
+    private _alone;
+    private _sizeByHand;
+    private _loadByHand;
+    private _admit;
+    private _makeRoomForGenerator;
+    private _yieldGenerators;
+    private _beforeLoad;
+    private _reserveByHand;
+    private _reserve;
+    private _sizeLoad;
+    private _chooseContext;
+    private _capRoom;
+    private _contextProfile;
+    private _trainedContext;
+    private _ctxSizeOf;
+    private _makeRoom;
+    private _capVictims;
+    private _victimRank;
+    private _usedBytes;
+    private _settledBytes;
+    private _memoryByModel;
     private _yieldPins;
     private _residency;
     private _completionLoadOptions;