@qvac/harness 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/asana/SKILL.md +86 -0
- package/.claude/skills/gh-stack/SKILL.md +181 -0
- package/.claude/skills/gh-stack/references/commands.md +177 -0
- package/.claude/skills/gh-stack/references/stack-design.md +97 -0
- package/.claude/skills/gh-stack/references/troubleshooting.md +159 -0
- package/.claude/skills/js-house-style/SKILL.md +671 -0
- package/.claude/skills/kvcache/SKILL.md +171 -0
- package/.claude/skills/pr-create/SKILL.md +180 -0
- package/.claude/skills/pr-review/SKILL.md +217 -0
- package/CLAUDE.md +39 -0
- package/README.md +26 -4
- package/dist/harness.bundle +8 -60
- package/dist/prebuilds-manifest.json +0 -8
- package/dist-lib/index.d.ts +4 -1
- package/dist-lib/index.js +23 -7
- package/dist-lib/lib/docx/index.d.ts +5 -0
- package/dist-lib/lib/docx/index.js +49 -0
- package/dist-lib/lib/models/model-pool.d.ts +1 -1
- package/dist-lib/lib/models/model-pool.js +3 -2
- package/dist-lib/lib/spawn.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +2 -0
- package/dist-lib/lib/tools/exec/adapters/python/wheels.js +2 -1
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.js +1 -1
- package/dist-lib/schema/generate.d.ts +6 -2
- package/dist-lib/schema/generate.js +10 -5
- package/docs/decisions/2026-05-29-kvcache-tool-turns.md +75 -0
- package/docs/decisions/2026-06-02-kvcache-tool-turns-reenabled.md +127 -0
- package/docs/decisions/2026-06-23-android-cpu-llm-defaults.md +126 -0
- package/docs/decisions/2026-07-29-assistant-owned-storage-layout.md +88 -0
- package/docs/decisions/2026-08-04-sidecar-orphan-reaping.md +79 -0
- package/docs/decisions/2026-08-11-pyodide-worker-thread.md +68 -0
- package/docs/decisions/2026-08-17-harness-hook-pipeline.md +84 -0
- package/docs/decisions/2026-08-17-harness-outbound-egress-guard.md +176 -0
- package/docs/decisions/2026-08-20-chat-prompt-prefix-is-stable.md +95 -0
- package/docs/decisions/2026-08-20-kvcache-static-placement.md +109 -0
- package/docs/decisions/2026-08-21-vendored-python-wheels.md +85 -0
- package/docs/decisions/2026-08-26-exec-is-not-gated-by-skills.md +64 -0
- package/docs/decisions/2026-08-27-writable-skills-overlay.md +56 -0
- package/docs/decisions/2026-09-01-skill-operation-manifests.md +53 -0
- package/docs/decisions/2026-09-01-usage-breakdown-is-measured.md +78 -0
- package/docs/decisions/2026-09-03-usage-breakdown-reported-by-the-harness.md +136 -0
- package/docs/decisions/INDEX.md +38 -0
- package/docs/decisions/TEMPLATE.md +32 -0
- package/docs/kvcache-reference.md +655 -0
- package/package.json +6 -2
- package/worker/harness-sidecar.mjs +3 -2
|
@@ -272,14 +272,6 @@
|
|
|
272
272
|
"win32-arm64",
|
|
273
273
|
"win32-x64"
|
|
274
274
|
],
|
|
275
|
-
"bare-tty/node_modules/bare-signals": [
|
|
276
|
-
"darwin-arm64",
|
|
277
|
-
"darwin-x64",
|
|
278
|
-
"linux-arm64",
|
|
279
|
-
"linux-x64",
|
|
280
|
-
"win32-arm64",
|
|
281
|
-
"win32-x64"
|
|
282
|
-
],
|
|
283
275
|
"bare-tty": [
|
|
284
276
|
"darwin-arm64",
|
|
285
277
|
"darwin-x64",
|
package/dist-lib/index.d.ts
CHANGED
|
@@ -82,6 +82,7 @@ export interface HarnessOptions {
|
|
|
82
82
|
transcriptionDevice?: 'cpu' | 'gpu';
|
|
83
83
|
credentialStore?: CredentialStoreFn;
|
|
84
84
|
platform?: string;
|
|
85
|
+
completionModelCap?: number;
|
|
85
86
|
}
|
|
86
87
|
export interface CompletionContext {
|
|
87
88
|
model: string;
|
|
@@ -133,6 +134,7 @@ export declare class Harness extends ReadyResource {
|
|
|
133
134
|
private readonly _kvCacheV?;
|
|
134
135
|
private readonly _transcriptionDevice?;
|
|
135
136
|
private readonly _gpuStats?;
|
|
137
|
+
private readonly _completionModelCap;
|
|
136
138
|
private _allSkills;
|
|
137
139
|
private _credentialStore?;
|
|
138
140
|
private _credentials;
|
|
@@ -145,7 +147,7 @@ export declare class Harness extends ReadyResource {
|
|
|
145
147
|
private _capability;
|
|
146
148
|
private _capabilityInflight;
|
|
147
149
|
private readonly _modelFaults;
|
|
148
|
-
constructor({ tools, sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, credentialStore, platform, gpuStats }?: HarnessOptions);
|
|
150
|
+
constructor({ tools, sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats }?: HarnessOptions);
|
|
149
151
|
_open(): Promise<void>;
|
|
150
152
|
get suspended(): boolean;
|
|
151
153
|
get skillNames(): string[];
|
|
@@ -238,6 +240,7 @@ export declare class Harness extends ReadyResource {
|
|
|
238
240
|
signal?: BareAbortSignal;
|
|
239
241
|
}): Promise<void>;
|
|
240
242
|
completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }: CompletionContext): AsyncGenerator<CompletionEvent>;
|
|
243
|
+
private _makeRoomForCompletion;
|
|
241
244
|
private _evictWedgedModel;
|
|
242
245
|
private _evictAfterLeases;
|
|
243
246
|
}
|
package/dist-lib/index.js
CHANGED
|
@@ -17,7 +17,7 @@ import { AbortedError, raceAbort } from "./lib/helpers/race-abort.js";
|
|
|
17
17
|
import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, infraModelsFor, modelNameToCapabilityProfile, musicModelSet, setCustomModels } from "./lib/models/model-catalog.js";
|
|
18
18
|
import { createModelStatusCache, invalidateModelStatusCache, listModels } from "./lib/models/model-list.js";
|
|
19
19
|
import { deleteModelFiles, summarizeDelete } from "./lib/models/model-delete.js";
|
|
20
|
-
import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool,
|
|
20
|
+
import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictModelWithin, isResident, leaseCount, loadedModels, modelKey, releaseModel, resolveLoadOptions, progressState } from "./lib/models/model-pool.js";
|
|
21
21
|
import { resolveMainGpu } from "./lib/models/gpu-selection.js";
|
|
22
22
|
import { openclCacheDir } from "./lib/models/opencl-cache.js";
|
|
23
23
|
import { assertEmbedDevice, createEmbedConfig } from "./lib/models/embed-config.js";
|
|
@@ -68,6 +68,9 @@ const TRANSCRIPTION_MODEL_LABEL = 'transcription model';
|
|
|
68
68
|
const DIFFUSION_EVICT_WAIT_MS = 15_000;
|
|
69
69
|
const MUSIC_EVICT_WAIT_MS = 15_000;
|
|
70
70
|
const WEDGED_EVICT_WAIT_MS = 10_000;
|
|
71
|
+
const MODEL_ROOM_WAIT_MS = 300_000;
|
|
72
|
+
const WARM_ROOM_WAIT_MS = 2_000;
|
|
73
|
+
const DEFAULT_COMPLETION_MODEL_CAP = 1;
|
|
71
74
|
const WEDGED_EVICT_LEASED_MS = 120_000;
|
|
72
75
|
const TTS_MODEL_CONFIG = {
|
|
73
76
|
ttsEngine: 'supertonic',
|
|
@@ -104,6 +107,7 @@ export class Harness extends ReadyResource {
|
|
|
104
107
|
_kvCacheV;
|
|
105
108
|
_transcriptionDevice;
|
|
106
109
|
_gpuStats;
|
|
110
|
+
_completionModelCap;
|
|
107
111
|
_allSkills;
|
|
108
112
|
_credentialStore;
|
|
109
113
|
_credentials = new Map();
|
|
@@ -116,7 +120,7 @@ export class Harness extends ReadyResource {
|
|
|
116
120
|
_capability = null;
|
|
117
121
|
_capabilityInflight = null;
|
|
118
122
|
_modelFaults = new Map();
|
|
119
|
-
constructor({ tools = [], sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, credentialStore, platform, gpuStats } = {}) {
|
|
123
|
+
constructor({ tools = [], sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats } = {}) {
|
|
120
124
|
super();
|
|
121
125
|
this._sdk = sdkLoader ?? createSdkLoader();
|
|
122
126
|
this._llmDevice = llmDevice;
|
|
@@ -126,6 +130,7 @@ export class Harness extends ReadyResource {
|
|
|
126
130
|
this._kvCacheK = kvCacheK;
|
|
127
131
|
this._kvCacheV = kvCacheV;
|
|
128
132
|
this._transcriptionDevice = transcriptionDevice;
|
|
133
|
+
this._completionModelCap = completionModelCap ?? DEFAULT_COMPLETION_MODEL_CAP;
|
|
129
134
|
const embedConfig = createEmbedConfig({
|
|
130
135
|
device: embedDevice,
|
|
131
136
|
gpuLayers: embedGpuLayers,
|
|
@@ -168,7 +173,7 @@ export class Harness extends ReadyResource {
|
|
|
168
173
|
onBeforeLoad: async (signal) => {
|
|
169
174
|
if (this._models.destroyed)
|
|
170
175
|
return;
|
|
171
|
-
const evicted = await
|
|
176
|
+
const evicted = await awaitModelCapacity(this._models, 1, DIFFUSION_EVICT_WAIT_MS, undefined, signal);
|
|
172
177
|
if (signal?.aborted)
|
|
173
178
|
throw new AbortedError();
|
|
174
179
|
if (!evicted) {
|
|
@@ -187,7 +192,7 @@ export class Harness extends ReadyResource {
|
|
|
187
192
|
onBeforeLoad: async (signal) => {
|
|
188
193
|
if (this._models.destroyed)
|
|
189
194
|
return;
|
|
190
|
-
const evicted = await
|
|
195
|
+
const evicted = await awaitModelCapacity(this._models, 1, MUSIC_EVICT_WAIT_MS, undefined, signal);
|
|
191
196
|
if (signal?.aborted)
|
|
192
197
|
throw new AbortedError();
|
|
193
198
|
if (!evicted) {
|
|
@@ -927,7 +932,8 @@ export class Harness extends ReadyResource {
|
|
|
927
932
|
const pool = this._models;
|
|
928
933
|
hlog('warm model', { model, resident: isResident(pool, key) });
|
|
929
934
|
if (!isResident(pool, key)) {
|
|
930
|
-
await
|
|
935
|
+
if (!(await this._makeRoomForCompletion(signal, WARM_ROOM_WAIT_MS)))
|
|
936
|
+
return;
|
|
931
937
|
await this._imageRuntime?.unloadIdle().catch(noop);
|
|
932
938
|
}
|
|
933
939
|
try {
|
|
@@ -953,13 +959,13 @@ export class Harness extends ReadyResource {
|
|
|
953
959
|
const resident = isResident(pool, key);
|
|
954
960
|
hlog('completion start', { model, resident, projection: Boolean(projectionModel) });
|
|
955
961
|
if (!resident) {
|
|
956
|
-
await
|
|
962
|
+
await this._makeRoomForCompletion(signal);
|
|
957
963
|
await this._imageRuntime?.unloadIdle().catch(noop);
|
|
958
964
|
await this._musicRuntime?.unloadIdle().catch(noop);
|
|
959
965
|
}
|
|
960
966
|
const acquireRound = async () => {
|
|
961
967
|
if (!isResident(pool, key)) {
|
|
962
|
-
await
|
|
968
|
+
await this._makeRoomForCompletion(signal);
|
|
963
969
|
await this._imageRuntime?.unloadIdle().catch(noop);
|
|
964
970
|
await this._musicRuntime?.unloadIdle().catch(noop);
|
|
965
971
|
}
|
|
@@ -1052,6 +1058,16 @@ export class Harness extends ReadyResource {
|
|
|
1052
1058
|
void this._musicRuntime?.unloadIdle().catch(noop);
|
|
1053
1059
|
}
|
|
1054
1060
|
}
|
|
1061
|
+
async _makeRoomForCompletion(signal, deadlineMs = MODEL_ROOM_WAIT_MS) {
|
|
1062
|
+
const roomed = await awaitModelCapacity(this._models, this._completionModelCap, deadlineMs, 250, signal);
|
|
1063
|
+
if (!roomed) {
|
|
1064
|
+
hlog('model room wait gave up', {
|
|
1065
|
+
cap: this._completionModelCap,
|
|
1066
|
+
loaded: loadedModels(this._models).length
|
|
1067
|
+
});
|
|
1068
|
+
}
|
|
1069
|
+
return roomed;
|
|
1070
|
+
}
|
|
1055
1071
|
async _evictWedgedModel(model, key) {
|
|
1056
1072
|
const faults = (this._modelFaults.get(model) ?? 0) + 1;
|
|
1057
1073
|
this._modelFaults.set(model, faults);
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
export { closePythonWorker as closeDocxWorker } from '../tools/exec/adapters/python/worker/client.ts';
|
|
3
|
+
export declare function extractDocxText(bytes: Uint8Array, fileName: string, options?: {
|
|
4
|
+
readonly signal?: AbortSignal;
|
|
5
|
+
}): Promise<string>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import { runPython } from "../tools/exec/adapters/python/pyodide.js";
|
|
4
|
+
import { planVendoredInstall, vendoredManifest, wheelsDir } from "../tools/exec/adapters/python/wheels.js";
|
|
5
|
+
export { closePythonWorker as closeDocxWorker } from "../tools/exec/adapters/python/worker/client.js";
|
|
6
|
+
const MAX_OFFICE_BYTES = 20 * 1024 * 1024;
|
|
7
|
+
const PACKAGES = ['python-docx==1.2.0'];
|
|
8
|
+
const EXTRACT = `from docx import Document
|
|
9
|
+
from docx.text.paragraph import Paragraph
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
def text(container):
|
|
13
|
+
for block in container.iter_inner_content():
|
|
14
|
+
if isinstance(block, Paragraph):
|
|
15
|
+
if block.text.strip():
|
|
16
|
+
yield block.text
|
|
17
|
+
else:
|
|
18
|
+
for row in block.rows:
|
|
19
|
+
yield " | ".join(" ".join(text(cell)) for cell in row.cells)
|
|
20
|
+
|
|
21
|
+
Path("extracted.txt").write_text("\\n".join(text(Document("input.docx"))), encoding="utf-8")
|
|
22
|
+
`;
|
|
23
|
+
export async function extractDocxText(bytes, fileName, options = {}) {
|
|
24
|
+
if (bytes.byteLength > MAX_OFFICE_BYTES) {
|
|
25
|
+
throw new Error(`document is too large to extract: ${bytes.byteLength} bytes, limit ${MAX_OFFICE_BYTES}`);
|
|
26
|
+
}
|
|
27
|
+
const dir = wheelsDir();
|
|
28
|
+
const manifest = vendoredManifest(dir);
|
|
29
|
+
const install = planVendoredInstall(PACKAGES, { dir, manifest });
|
|
30
|
+
if (!dir ||
|
|
31
|
+
!manifest ||
|
|
32
|
+
!install ||
|
|
33
|
+
!['micropip', ...install.packages].every((name) => Object.keys(manifest.files).some((file) => file.replaceAll('_', '-').startsWith(`${name}-`) && fs.existsSync(`${dir}/${file}`)))) {
|
|
34
|
+
throw new Error('bundled DOCX extraction packages are unavailable');
|
|
35
|
+
}
|
|
36
|
+
const result = await runPython(EXTRACT, {
|
|
37
|
+
packages: PACKAGES,
|
|
38
|
+
inputs: [{ path: 'input.docx', bytes }],
|
|
39
|
+
outputs: ['extracted.txt'],
|
|
40
|
+
signal: options.signal
|
|
41
|
+
});
|
|
42
|
+
if (result.exitCode !== 0)
|
|
43
|
+
throw new Error(`${fileName}: ${result.stderr.trim()}`);
|
|
44
|
+
const output = result.files?.get('extracted.txt');
|
|
45
|
+
if (!output) {
|
|
46
|
+
throw new Error(result.rejected?.[0]?.reason ?? 'DOCX extraction produced no text file');
|
|
47
|
+
}
|
|
48
|
+
return b4a.toString(output, 'utf8');
|
|
49
|
+
}
|
|
@@ -72,7 +72,7 @@ export declare function loadedModels(pool: PoolLike): LoadedModel[];
|
|
|
72
72
|
export declare function isResident(pool: PoolLike, key: string): boolean;
|
|
73
73
|
export declare function leaseCount(pool: PoolLike, key: string): number;
|
|
74
74
|
export declare function evictIdle(pool: PoolLike): Promise<void>;
|
|
75
|
-
export declare function
|
|
75
|
+
export declare function awaitModelCapacity(pool: PoolLike, cap: number, deadlineMs: number, tickMs?: number, signal?: {
|
|
76
76
|
aborted?: boolean;
|
|
77
77
|
}): Promise<boolean>;
|
|
78
78
|
export declare function evictModelWithin(pool: PoolLike, key: string, deadlineMs: number, tickMs?: number): Promise<boolean>;
|
|
@@ -103,7 +103,8 @@ export async function evictIdle(pool) {
|
|
|
103
103
|
await pool.gc();
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
|
-
export async function
|
|
106
|
+
export async function awaitModelCapacity(pool, cap, deadlineMs, tickMs = 250, signal) {
|
|
107
|
+
const roomed = () => loadedModels(pool).length < cap;
|
|
107
108
|
let done = false;
|
|
108
109
|
let timer = null;
|
|
109
110
|
const expired = new Promise((resolve) => {
|
|
@@ -114,7 +115,7 @@ export async function evictAllWithin(pool, deadlineMs, tickMs = 250, signal) {
|
|
|
114
115
|
});
|
|
115
116
|
const evicting = (async () => {
|
|
116
117
|
await evictIdle(pool);
|
|
117
|
-
while (
|
|
118
|
+
while (!roomed()) {
|
|
118
119
|
if (done || signal?.aborted)
|
|
119
120
|
return false;
|
|
120
121
|
await new Promise((resolve) => setTimeout(resolve, tickMs));
|
package/dist-lib/lib/spawn.d.ts
CHANGED
|
@@ -99,6 +99,8 @@ export async function runPython(code, options = {}) {
|
|
|
99
99
|
}
|
|
100
100
|
await runLock.lock();
|
|
101
101
|
try {
|
|
102
|
+
if (options.signal?.aborted)
|
|
103
|
+
throw new AbortedError();
|
|
102
104
|
return await runPythonInWorker(spec, { timeoutMs, signal: options.signal });
|
|
103
105
|
}
|
|
104
106
|
finally {
|
|
@@ -17,7 +17,8 @@ export function wheelsDir() {
|
|
|
17
17
|
const fromHost = process.env.QVAC_PYTHON_WHEELS_DIR;
|
|
18
18
|
if (fromHost)
|
|
19
19
|
return fromHost;
|
|
20
|
-
const
|
|
20
|
+
const root = new URL('../../../../../', import.meta.url);
|
|
21
|
+
const source = new URL(root.pathname.endsWith('/dist-lib/') ? '../wheels' : 'wheels', root);
|
|
21
22
|
if (source.href.includes('.bundle/'))
|
|
22
23
|
return undefined;
|
|
23
24
|
return fileURLToPath(source.href);
|
|
@@ -13,7 +13,7 @@ const SHUTDOWN_GRACE_MS = 3_000;
|
|
|
13
13
|
let active = null;
|
|
14
14
|
function workerEntry() {
|
|
15
15
|
return (process.env.QVAC_PYODIDE_WORKER_ENTRY ??
|
|
16
|
-
fileURLToPath(new URL('./entry.ts', import.meta.url).href));
|
|
16
|
+
fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './entry.ts' : './entry.js', import.meta.url).href));
|
|
17
17
|
}
|
|
18
18
|
async function readMessage(link) {
|
|
19
19
|
for (;;) {
|
|
@@ -13,8 +13,12 @@ interface HRPCHandler {
|
|
|
13
13
|
interface HRPCBuilderLike {
|
|
14
14
|
handlers: ReadonlyArray<HRPCHandler>;
|
|
15
15
|
}
|
|
16
|
+
interface CallSiteOptions {
|
|
17
|
+
readonly optionalRequests?: ReadonlySet<string>;
|
|
18
|
+
readonly exportName?: string;
|
|
19
|
+
}
|
|
16
20
|
export declare function writeBind(builder: HRPCBuilderLike, outFile: string): void;
|
|
17
|
-
export declare function writeClient(builder: HRPCBuilderLike, outFile: string,
|
|
18
|
-
export declare function writeCapabilities(builder: HRPCBuilderLike, outFile: string): void;
|
|
21
|
+
export declare function writeClient(builder: HRPCBuilderLike, outFile: string, options?: CallSiteOptions | string): void;
|
|
22
|
+
export declare function writeCapabilities(builder: HRPCBuilderLike, outFile: string, options?: CallSiteOptions): void;
|
|
19
23
|
export declare function writeConfigTypes(outFile: string): void;
|
|
20
24
|
export {};
|
|
@@ -13,6 +13,7 @@ function camel(name, prefix = '') {
|
|
|
13
13
|
return prefix + cased.join('');
|
|
14
14
|
}
|
|
15
15
|
const HEADER = '// This file is autogenerated by schema/build.js\n// Do not edit manually.\n\n';
|
|
16
|
+
const DEFAULT_OPTIONAL_REQUESTS = new Set(['@assistant/empty']);
|
|
16
17
|
export function writeBind(builder, outFile) {
|
|
17
18
|
let s = `${HEADER}import { pipeline, Readable } from "streamx";\n\nfunction noop() {}\n\nexport function bindApi(rpc, api) {\n`;
|
|
18
19
|
for (const handler of builder.handlers) {
|
|
@@ -56,24 +57,28 @@ export function writeBind(builder, outFile) {
|
|
|
56
57
|
s += '}\n';
|
|
57
58
|
fs.writeFileSync(outFile, s);
|
|
58
59
|
}
|
|
59
|
-
export function writeClient(builder, outFile,
|
|
60
|
+
export function writeClient(builder, outFile, options = {}) {
|
|
61
|
+
const callSite = typeof options === 'string' ? { exportName: options } : options;
|
|
62
|
+
const { optionalRequests = DEFAULT_OPTIONAL_REQUESTS, exportName = 'engineClient' } = callSite;
|
|
60
63
|
let s = `${HEADER}function noop() {}\n\nfunction armed(stream) {\n stream.on("error", noop);\n return stream;\n}\n\nexport function ${exportName}(rpc) {\n return {\n`;
|
|
61
64
|
for (const handler of builder.handlers) {
|
|
62
65
|
const method = camel(handler.name);
|
|
66
|
+
const input = optionalRequests.has(handler.request?.name ?? '') ? 'input ?? {}' : 'input';
|
|
63
67
|
if (handler.request?.stream && handler.response?.stream) {
|
|
64
68
|
s += ` ${method}: () => armed(rpc.${method}()),\n`;
|
|
65
69
|
}
|
|
66
70
|
else if (handler.response?.stream) {
|
|
67
|
-
s += ` ${method}: (input) => armed(rpc.${method}(input)),\n`;
|
|
71
|
+
s += ` ${method}: (input) => armed(rpc.${method}(${input})),\n`;
|
|
68
72
|
}
|
|
69
73
|
else {
|
|
70
|
-
s += ` ${method}: (input) => rpc.${method}(input),\n`;
|
|
74
|
+
s += ` ${method}: (input) => rpc.${method}(${input}),\n`;
|
|
71
75
|
}
|
|
72
76
|
}
|
|
73
77
|
s += ' };\n}\n';
|
|
74
78
|
fs.writeFileSync(outFile, s);
|
|
75
79
|
}
|
|
76
|
-
export function writeCapabilities(builder, outFile) {
|
|
80
|
+
export function writeCapabilities(builder, outFile, options = {}) {
|
|
81
|
+
const { optionalRequests = DEFAULT_OPTIONAL_REQUESTS } = options;
|
|
77
82
|
let s = HEADER;
|
|
78
83
|
s += "import type * as T from './hyperschema/types.d.ts'\n\n";
|
|
79
84
|
s +=
|
|
@@ -112,7 +117,7 @@ export function writeCapabilities(builder, outFile) {
|
|
|
112
117
|
const method = camel(handler.name);
|
|
113
118
|
const req = `T.${tsName(handler.request.name)}`;
|
|
114
119
|
const res = `T.${tsName(handler.response.name)}`;
|
|
115
|
-
const input = handler.request.name
|
|
120
|
+
const input = optionalRequests.has(handler.request.name) ? `input?: ${req}` : `input: ${req}`;
|
|
116
121
|
if (handler.request?.stream && handler.response?.stream) {
|
|
117
122
|
client += ` ${method}(): UploadStream<${req}, ${res}>\n`;
|
|
118
123
|
server += ` ${method}(stream: AsyncIterable<${req}> | HandlerFrames<${req}>): AsyncIterable<${res}>\n`;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
---
|
|
2
|
+
id: ADR-2026-05-29-kvcache-tool-turns
|
|
3
|
+
status: superseded
|
|
4
|
+
introduced_by: PR-228
|
|
5
|
+
supersedes: []
|
|
6
|
+
superseded_by: [ADR-2026-06-02-kvcache-tool-turns-reenabled]
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# KV-cache session is disabled for tool-using completions
|
|
10
|
+
|
|
11
|
+
> **Superseded** by
|
|
12
|
+
> [ADR-2026-06-02-kvcache-tool-turns-reenabled](./2026-06-02-kvcache-tool-turns-reenabled.md):
|
|
13
|
+
> the cache is re-enabled for tool turns now that models load with
|
|
14
|
+
> `toolsMode: "dynamic"` (the SDK's DYNAMIC path is coherent, including across
|
|
15
|
+
> reload). Kept for history.
|
|
16
|
+
|
|
17
|
+
## Decision
|
|
18
|
+
|
|
19
|
+
The `@qvac/sdk` session prompt cache (`completion({ kvCache })`) is passed
|
|
20
|
+
**only for tool-free completions** — `kvCache` is set only when
|
|
21
|
+
`tools.length === 0`. Agentic / tool-using turns always run without the session
|
|
22
|
+
cache.
|
|
23
|
+
|
|
24
|
+
## Context
|
|
25
|
+
|
|
26
|
+
The SDK session cache has two paths (see
|
|
27
|
+
`@qvac/sdk → server/bare/plugins/llamacpp-completion/ops/completion-stream.ts` in the SDK):
|
|
28
|
+
|
|
29
|
+
- **STATIC** (no `tools`): on a cache hit it sends `history.slice(savedCount)`
|
|
30
|
+
and self-heals a stale boundary by re-sending the full history. Robust against
|
|
31
|
+
our reconstructed history.
|
|
32
|
+
- **DYNAMIC** (`tools` present): on a cache hit it sends only a role-based
|
|
33
|
+
_fragment_ of the history and trusts the SDK's own tool-chain flow to have
|
|
34
|
+
anchored the rest of the cache.
|
|
35
|
+
|
|
36
|
+
Our agentic loop builds (and on reload reconstructs from a persisted transcript)
|
|
37
|
+
the full history its own way. That does not satisfy the DYNAMIC path's
|
|
38
|
+
assumptions, so on a tool-cache hit the model receives an incoherent prompt and
|
|
39
|
+
emits `<tool_call>` markup as plain text instead of a parseable tool call —
|
|
40
|
+
silently breaking tool calling (no parse, no approval, no execution). Tools are
|
|
41
|
+
text-parsed (`toolCalling: "textParse"`), so a corrupted prompt degrades to a
|
|
42
|
+
plain assistant message rather than erroring.
|
|
43
|
+
|
|
44
|
+
## Alternatives Considered
|
|
45
|
+
|
|
46
|
+
- **Always-on per-chat `kvCache`** (the state that introduced the regression):
|
|
47
|
+
breaks tool calling as above.
|
|
48
|
+
- **Auto-cache (`kvCache: true`)**: tool-call turns are never cached, and a
|
|
49
|
+
verbatim `cacheableAssistantContent` echo is required for a hit —
|
|
50
|
+
incompatible with our thinking-stripped reconstruction.
|
|
51
|
+
- **Conform to the SDK's tool-chain contract** (collapse consecutive messages +
|
|
52
|
+
`deleteCache` on divergence, like the legacy desktop worker): viable but a
|
|
53
|
+
separate project, tracked in `.ai/cursor/plans/kvcache-tool-turn-reuse.md`.
|
|
54
|
+
|
|
55
|
+
## Consequences
|
|
56
|
+
|
|
57
|
+
- Tool turns re-prefill the full history each call (no session-cache reuse).
|
|
58
|
+
Tool calling stays correct; this is the prior known-good behavior.
|
|
59
|
+
- Tool-free turns (plain chat / title / summary) still benefit from the STATIC
|
|
60
|
+
cache path.
|
|
61
|
+
- Regression guards are GPU-free and mock-based:
|
|
62
|
+
`qvac-app → core/src/services/message/agentic-completion.test.ts` (argument-contract:
|
|
63
|
+
never send `kvCache` with `tools`) and the spec-fake
|
|
64
|
+
`qvac-app → core/src/services/message/__fixtures__/fake-completion-sdk.ts` (reproduces the
|
|
65
|
+
cache-hit tool-parse failure).
|
|
66
|
+
- Revisit if the SDK adds a "send full history, you slice it" mode for tools, or
|
|
67
|
+
when the conformance project lands.
|
|
68
|
+
|
|
69
|
+
## When Agents Should Read This
|
|
70
|
+
|
|
71
|
+
Read this before changing how `kvCache` is passed to `sdk.completion`, the
|
|
72
|
+
agentic loop's history construction, or transcript reconstruction —
|
|
73
|
+
`qvac-app → core/src/services/message/agentic-completion.ts`,
|
|
74
|
+
`qvac-app → core/src/services/message/agentic-loop.ts`,
|
|
75
|
+
`qvac-app → core/src/services/common/transcript.ts`.
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
---
|
|
2
|
+
id: ADR-2026-06-02-kvcache-tool-turns-reenabled
|
|
3
|
+
status: superseded
|
|
4
|
+
introduced_by: PR-QVAC-19770
|
|
5
|
+
supersedes: [ADR-2026-05-29-kvcache-tool-turns]
|
|
6
|
+
superseded_by: [ADR-2026-08-20-kvcache-static-placement]
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# KV-cache session is re-enabled for tool-using completions
|
|
10
|
+
|
|
11
|
+
> **Superseded** by
|
|
12
|
+
> [ADR-2026-08-20-kvcache-static-placement](./2026-08-20-kvcache-static-placement.md):
|
|
13
|
+
> the half of this record that still holds is that `kvCache: 'chat-<id>'` rides
|
|
14
|
+
> every completion. The dynamic-mode half was built on a false premise — the
|
|
15
|
+
> addon's compact-tools anchoring never ran on any shipped model (its arch gate
|
|
16
|
+
> is exact `qwen3`; ours are `qwen35`/`qwen36`), so the coherence this record
|
|
17
|
+
> credits to dynamic mode was never exercised, and the "static never surfaced
|
|
18
|
+
> the tool-call format" claim did not survive re-testing. core-v2 loads static
|
|
19
|
+
> with no sliding window since 2026-08-20; mechanics in
|
|
20
|
+
> `docs/kvcache-reference.md`. The `QVAC_PROMPT_CACHE` kill-switch and
|
|
21
|
+
> `deleteChatPromptCache` described below were core-v1 machinery and do not
|
|
22
|
+
> exist in the current runtime.
|
|
23
|
+
|
|
24
|
+
## Decision
|
|
25
|
+
|
|
26
|
+
The `@qvac/sdk` session prompt cache (`completion({ kvCache: "chat-<id>" })`) is
|
|
27
|
+
passed for **all** completions with a chatId — tool-using and tool-free alike —
|
|
28
|
+
gated only by the `QVAC_PROMPT_CACHE` kill-switch. This reverses
|
|
29
|
+
ADR-2026-05-29, which disabled the cache whenever tools were present.
|
|
30
|
+
|
|
31
|
+
## Context
|
|
32
|
+
|
|
33
|
+
The prior decision disabled tool-turn caching because the SDK's DYNAMIC cache
|
|
34
|
+
path broke tool-call parsing: it ships a role-based history fragment and trusts
|
|
35
|
+
the addon's tool-chain anchoring, which our reconstructed history did not
|
|
36
|
+
satisfy → `<tool_call>` markup leaked as text. Two things changed:
|
|
37
|
+
|
|
38
|
+
- Models loaded with `modelConfig.toolsMode: "dynamic"` (QVAC-19660). In
|
|
39
|
+
dynamic mode the addon anchors tools per-turn (compact-tools), excludes the
|
|
40
|
+
tool set from the cache config hash, and the fragment slicing keeps
|
|
41
|
+
multi-round tool turns coherent — including across a model evict/reload.
|
|
42
|
+
**No longer true as of `@qvac/sdk` 0.18 (QVAC-23936):** the key was removed
|
|
43
|
+
and the parser rejects it, so static is the only placement. Dynamic was in
|
|
44
|
+
any case a silent no-op on every arch we ship — the addon gates
|
|
45
|
+
`tools_compact` on exact-arch `qwen3` while our catalog is `qwen35`/`qwen36` —
|
|
46
|
+
and the SDK still sliced as-if-dynamic, which made the cache the chat's
|
|
47
|
+
_memory_ rather than an accelerator: after any cache reset it sent only the
|
|
48
|
+
last exchange. Under static placement the tool block sits in the addon's
|
|
49
|
+
protected prefix instead, and 0.18 keeps it out of the primed prefix
|
|
50
|
+
(QVAC-23251) so the qwen3.5 template no longer raises 'No user query found in
|
|
51
|
+
messages.'
|
|
52
|
+
- An SDK e2e guard (`kv-cache-tools-dynamic-reuse`, plus a qwen35-dialect
|
|
53
|
+
variant) verifies a custom-key multi-round tool chain reuses the on-disk cache
|
|
54
|
+
(`cacheTokens > 0`) and keeps tool calls parseable across an evict/reload.
|
|
55
|
+
|
|
56
|
+
With the DYNAMIC path proven coherent across reload, the heavyweight
|
|
57
|
+
"reconstruction invalidation / byte-identical history" work the prior ADR
|
|
58
|
+
deferred is unnecessary; divergence is handled with targeted invalidation.
|
|
59
|
+
|
|
60
|
+
## Consequences
|
|
61
|
+
|
|
62
|
+
- **Do not pass `toolsMode` at model load.** SDK 0.18 removed the key and its
|
|
63
|
+
parser rejects an unrecognised one, so passing it fails the load outright and
|
|
64
|
+
the app parks on the splash. Static placement is the default and the only mode.
|
|
65
|
+
- Dropping a key from `modelConfig` re-keys `modelId`, which is identity-by-config,
|
|
66
|
+
so expect one cold prime per chat on the first run after such a change.
|
|
67
|
+
- Tool turns reuse the session KV cache: lower TTFT/prefill on multi-round tool
|
|
68
|
+
turns and follow-ups (manually: round-2 TTFT ~1.1s → ~0.3s; cacheTokens grows
|
|
69
|
+
monotonically across a conversation).
|
|
70
|
+
- `QVAC_PROMPT_CACHE` (default on; `0`/`false`/`off`) disables all
|
|
71
|
+
prompt caching as a kill-switch.
|
|
72
|
+
- The per-chat cache is explicitly invalidated on chat delete and agent delete
|
|
73
|
+
(which cascades to its chats) via `deleteChatPromptCache`, plus a defensive
|
|
74
|
+
drop on retry (see below). Edit / branch / truncate flows do not exist yet and
|
|
75
|
+
need no hook.
|
|
76
|
+
- **Why stop / failed / new-message-after-failure need no invalidation.** The
|
|
77
|
+
cache only ever holds _successful_ turns: the SDK rolls back uncommitted turns
|
|
78
|
+
on abort (the custom-key path commits only when not aborted and tokens were
|
|
79
|
+
produced), and failed / canceled / not_delivered replies are excluded from
|
|
80
|
+
model-visible history (`isModelVisibleMessage`, `qvac-app → core/src/services/common/history.ts`).
|
|
81
|
+
So a stopped or failed turn leaves neither a committed cache entry nor a
|
|
82
|
+
visible history row — the reconstructed history and the cached prefix stay
|
|
83
|
+
aligned, and the next turn (new message or retry) does not slice against a
|
|
84
|
+
stale prefix. The SDK's own `kv-cache-cancel-then-new-prompt` e2e covers this
|
|
85
|
+
cancel-then-continue coherence. The **retry** `deleteChatPromptCache` is
|
|
86
|
+
therefore belt-and-suspenders, not load-bearing: it costs one re-prime on a
|
|
87
|
+
rare, user-initiated path and removes any dependence on the rollback timing.
|
|
88
|
+
- **Mobile scope.** Mobile now runs the same assistant and harness completion path
|
|
89
|
+
inside its supervised Bare worklets, so the per-chat cache key reaches the SDK
|
|
90
|
+
without the retired worker-to-host SDK bridge.
|
|
91
|
+
- `chatPromptCacheKey` is the single source of truth for the `chat-<id>` key
|
|
92
|
+
shared by the completion path and the invalidation callers.
|
|
93
|
+
- Known SDK limitation: on a multimodal (M-RoPE, `n_pos_per_embd > 1`) model such
|
|
94
|
+
as Qwen3.5, the SDK's KV sliding-window position shift (`seq_add`/`seq_div` via
|
|
95
|
+
`trySlidePrefill`) GGML_ASSERT-crashes the worker when the context fills. A
|
|
96
|
+
standalone SDK repro confirmed this is **independent of `toolsMode` (static and
|
|
97
|
+
dynamic both crash) and of whether tools are passed**, and is not caused by the
|
|
98
|
+
prompt cache — it is the sliding window itself, unsupported for multi-pos
|
|
99
|
+
models. Interim mitigation (separate from the kill-switch): we temporarily
|
|
100
|
+
stopped setting `n_discarded` at model load so on overflow the SDK took the
|
|
101
|
+
catchable `CONTEXT_OVERFLOW` path instead of the crashing slide.
|
|
102
|
+
**Resolved (2026-06-23):** `@qvac/sdk` 0.13.5 (llm-llamacpp 0.24.0) ships
|
|
103
|
+
M-RoPE/iM-RoPE sliding-context support, so `n_discarded` is set again in
|
|
104
|
+
`@qvac/sdk → model/model-load-config.ts` and the workbench no longer trims history. See
|
|
105
|
+
[`2026-06-23-append-only-history.md`](./2026-06-23-append-only-history.md).
|
|
106
|
+
- Regression guards updated: the contract test now asserts `kvCache` IS sent
|
|
107
|
+
with tools; the spec-fake models dynamic mode (healthy on a tools cache hit);
|
|
108
|
+
invalidation hooks are covered.
|
|
109
|
+
- **The in-repo suite cannot prove real dynamic-mode coherence.** The spec-fake
|
|
110
|
+
(`@qvac/sdk → __fixtures__/fake-completion-sdk.ts`) is parameterized to treat dynamic mode
|
|
111
|
+
as healthy, so the in-repo tests guard against regressing the model load back
|
|
112
|
+
to static mode and against dropping the `kvCache` key — but they do not prove
|
|
113
|
+
the SDK's real DYNAMIC path stays coherent against our reconstructed history.
|
|
114
|
+
That proof lives in the out-of-repo SDK e2e (`kv-cache-tools-dynamic-reuse`
|
|
115
|
+
and its qwen35 variant), which this repo's CI cannot run.
|
|
116
|
+
|
|
117
|
+
## When Agents Should Read This
|
|
118
|
+
|
|
119
|
+
Read this before changing how `kvCache` is passed to `sdk.completion`,
|
|
120
|
+
`toolsMode` at model load, the agentic loop's history construction, transcript
|
|
121
|
+
reconstruction, or the cache invalidation hooks —
|
|
122
|
+
`qvac-app → core/src/services/message/agentic-completion.ts`,
|
|
123
|
+
`qvac-app → core/src/services/model/service.ts`,
|
|
124
|
+
`qvac-app → core/src/services/execution/service.ts`,
|
|
125
|
+
`qvac-app → core/src/services/api/domain-apis.ts`,
|
|
126
|
+
`@qvac/core → lib/assistant.ts`, and
|
|
127
|
+
`lib/capabilities/completion.ts`.
|