@qvac/harness 0.2.5 → 0.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/.claude/skills/asana/SKILL.md +86 -0
  2. package/.claude/skills/gh-stack/SKILL.md +181 -0
  3. package/.claude/skills/gh-stack/references/commands.md +177 -0
  4. package/.claude/skills/gh-stack/references/stack-design.md +97 -0
  5. package/.claude/skills/gh-stack/references/troubleshooting.md +159 -0
  6. package/.claude/skills/js-house-style/SKILL.md +671 -0
  7. package/.claude/skills/kvcache/SKILL.md +171 -0
  8. package/.claude/skills/pr-create/SKILL.md +180 -0
  9. package/.claude/skills/pr-review/SKILL.md +217 -0
  10. package/CLAUDE.md +39 -0
  11. package/README.md +26 -4
  12. package/dist/harness.bundle +8 -60
  13. package/dist/prebuilds-manifest.json +0 -8
  14. package/dist-lib/index.d.ts +4 -1
  15. package/dist-lib/index.js +23 -7
  16. package/dist-lib/lib/docx/index.d.ts +5 -0
  17. package/dist-lib/lib/docx/index.js +49 -0
  18. package/dist-lib/lib/models/model-pool.d.ts +1 -1
  19. package/dist-lib/lib/models/model-pool.js +3 -2
  20. package/dist-lib/lib/spawn.d.ts +1 -0
  21. package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +2 -0
  22. package/dist-lib/lib/tools/exec/adapters/python/wheels.js +2 -1
  23. package/dist-lib/lib/tools/exec/adapters/python/worker/client.js +1 -1
  24. package/dist-lib/schema/generate.d.ts +6 -2
  25. package/dist-lib/schema/generate.js +10 -5
  26. package/docs/decisions/2026-05-29-kvcache-tool-turns.md +75 -0
  27. package/docs/decisions/2026-06-02-kvcache-tool-turns-reenabled.md +127 -0
  28. package/docs/decisions/2026-06-23-android-cpu-llm-defaults.md +126 -0
  29. package/docs/decisions/2026-07-29-assistant-owned-storage-layout.md +88 -0
  30. package/docs/decisions/2026-08-04-sidecar-orphan-reaping.md +79 -0
  31. package/docs/decisions/2026-08-11-pyodide-worker-thread.md +68 -0
  32. package/docs/decisions/2026-08-17-harness-hook-pipeline.md +84 -0
  33. package/docs/decisions/2026-08-17-harness-outbound-egress-guard.md +176 -0
  34. package/docs/decisions/2026-08-20-chat-prompt-prefix-is-stable.md +95 -0
  35. package/docs/decisions/2026-08-20-kvcache-static-placement.md +109 -0
  36. package/docs/decisions/2026-08-21-vendored-python-wheels.md +85 -0
  37. package/docs/decisions/2026-08-26-exec-is-not-gated-by-skills.md +64 -0
  38. package/docs/decisions/2026-08-27-writable-skills-overlay.md +56 -0
  39. package/docs/decisions/2026-09-01-skill-operation-manifests.md +53 -0
  40. package/docs/decisions/2026-09-01-usage-breakdown-is-measured.md +78 -0
  41. package/docs/decisions/2026-09-03-usage-breakdown-reported-by-the-harness.md +136 -0
  42. package/docs/decisions/INDEX.md +38 -0
  43. package/docs/decisions/TEMPLATE.md +32 -0
  44. package/docs/kvcache-reference.md +655 -0
  45. package/package.json +6 -2
  46. package/worker/harness-sidecar.mjs +3 -2
@@ -272,14 +272,6 @@
272
272
  "win32-arm64",
273
273
  "win32-x64"
274
274
  ],
275
- "bare-tty/node_modules/bare-signals": [
276
- "darwin-arm64",
277
- "darwin-x64",
278
- "linux-arm64",
279
- "linux-x64",
280
- "win32-arm64",
281
- "win32-x64"
282
- ],
283
275
  "bare-tty": [
284
276
  "darwin-arm64",
285
277
  "darwin-x64",
@@ -82,6 +82,7 @@ export interface HarnessOptions {
82
82
  transcriptionDevice?: 'cpu' | 'gpu';
83
83
  credentialStore?: CredentialStoreFn;
84
84
  platform?: string;
85
+ completionModelCap?: number;
85
86
  }
86
87
  export interface CompletionContext {
87
88
  model: string;
@@ -133,6 +134,7 @@ export declare class Harness extends ReadyResource {
133
134
  private readonly _kvCacheV?;
134
135
  private readonly _transcriptionDevice?;
135
136
  private readonly _gpuStats?;
137
+ private readonly _completionModelCap;
136
138
  private _allSkills;
137
139
  private _credentialStore?;
138
140
  private _credentials;
@@ -145,7 +147,7 @@ export declare class Harness extends ReadyResource {
145
147
  private _capability;
146
148
  private _capabilityInflight;
147
149
  private readonly _modelFaults;
148
- constructor({ tools, sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, credentialStore, platform, gpuStats }?: HarnessOptions);
150
+ constructor({ tools, sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats }?: HarnessOptions);
149
151
  _open(): Promise<void>;
150
152
  get suspended(): boolean;
151
153
  get skillNames(): string[];
@@ -238,6 +240,7 @@ export declare class Harness extends ReadyResource {
238
240
  signal?: BareAbortSignal;
239
241
  }): Promise<void>;
240
242
  completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }: CompletionContext): AsyncGenerator<CompletionEvent>;
243
+ private _makeRoomForCompletion;
241
244
  private _evictWedgedModel;
242
245
  private _evictAfterLeases;
243
246
  }
package/dist-lib/index.js CHANGED
@@ -17,7 +17,7 @@ import { AbortedError, raceAbort } from "./lib/helpers/race-abort.js";
17
17
  import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, infraModelsFor, modelNameToCapabilityProfile, musicModelSet, setCustomModels } from "./lib/models/model-catalog.js";
18
18
  import { createModelStatusCache, invalidateModelStatusCache, listModels } from "./lib/models/model-list.js";
19
19
  import { deleteModelFiles, summarizeDelete } from "./lib/models/model-delete.js";
20
- import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, evictAllWithin, evictIdle, evictModelWithin, isResident, leaseCount, loadedModels, modelKey, releaseModel, resolveLoadOptions, progressState } from "./lib/models/model-pool.js";
20
+ import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictModelWithin, isResident, leaseCount, loadedModels, modelKey, releaseModel, resolveLoadOptions, progressState } from "./lib/models/model-pool.js";
21
21
  import { resolveMainGpu } from "./lib/models/gpu-selection.js";
22
22
  import { openclCacheDir } from "./lib/models/opencl-cache.js";
23
23
  import { assertEmbedDevice, createEmbedConfig } from "./lib/models/embed-config.js";
@@ -68,6 +68,9 @@ const TRANSCRIPTION_MODEL_LABEL = 'transcription model';
68
68
  const DIFFUSION_EVICT_WAIT_MS = 15_000;
69
69
  const MUSIC_EVICT_WAIT_MS = 15_000;
70
70
  const WEDGED_EVICT_WAIT_MS = 10_000;
71
+ const MODEL_ROOM_WAIT_MS = 300_000;
72
+ const WARM_ROOM_WAIT_MS = 2_000;
73
+ const DEFAULT_COMPLETION_MODEL_CAP = 1;
71
74
  const WEDGED_EVICT_LEASED_MS = 120_000;
72
75
  const TTS_MODEL_CONFIG = {
73
76
  ttsEngine: 'supertonic',
@@ -104,6 +107,7 @@ export class Harness extends ReadyResource {
104
107
  _kvCacheV;
105
108
  _transcriptionDevice;
106
109
  _gpuStats;
110
+ _completionModelCap;
107
111
  _allSkills;
108
112
  _credentialStore;
109
113
  _credentials = new Map();
@@ -116,7 +120,7 @@ export class Harness extends ReadyResource {
116
120
  _capability = null;
117
121
  _capabilityInflight = null;
118
122
  _modelFaults = new Map();
119
- constructor({ tools = [], sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, credentialStore, platform, gpuStats } = {}) {
123
+ constructor({ tools = [], sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats } = {}) {
120
124
  super();
121
125
  this._sdk = sdkLoader ?? createSdkLoader();
122
126
  this._llmDevice = llmDevice;
@@ -126,6 +130,7 @@ export class Harness extends ReadyResource {
126
130
  this._kvCacheK = kvCacheK;
127
131
  this._kvCacheV = kvCacheV;
128
132
  this._transcriptionDevice = transcriptionDevice;
133
+ this._completionModelCap = completionModelCap ?? DEFAULT_COMPLETION_MODEL_CAP;
129
134
  const embedConfig = createEmbedConfig({
130
135
  device: embedDevice,
131
136
  gpuLayers: embedGpuLayers,
@@ -168,7 +173,7 @@ export class Harness extends ReadyResource {
168
173
  onBeforeLoad: async (signal) => {
169
174
  if (this._models.destroyed)
170
175
  return;
171
- const evicted = await evictAllWithin(this._models, DIFFUSION_EVICT_WAIT_MS, undefined, signal);
176
+ const evicted = await awaitModelCapacity(this._models, 1, DIFFUSION_EVICT_WAIT_MS, undefined, signal);
172
177
  if (signal?.aborted)
173
178
  throw new AbortedError();
174
179
  if (!evicted) {
@@ -187,7 +192,7 @@ export class Harness extends ReadyResource {
187
192
  onBeforeLoad: async (signal) => {
188
193
  if (this._models.destroyed)
189
194
  return;
190
- const evicted = await evictAllWithin(this._models, MUSIC_EVICT_WAIT_MS, undefined, signal);
195
+ const evicted = await awaitModelCapacity(this._models, 1, MUSIC_EVICT_WAIT_MS, undefined, signal);
191
196
  if (signal?.aborted)
192
197
  throw new AbortedError();
193
198
  if (!evicted) {
@@ -927,7 +932,8 @@ export class Harness extends ReadyResource {
927
932
  const pool = this._models;
928
933
  hlog('warm model', { model, resident: isResident(pool, key) });
929
934
  if (!isResident(pool, key)) {
930
- await evictIdle(pool);
935
+ if (!(await this._makeRoomForCompletion(signal, WARM_ROOM_WAIT_MS)))
936
+ return;
931
937
  await this._imageRuntime?.unloadIdle().catch(noop);
932
938
  }
933
939
  try {
@@ -953,13 +959,13 @@ export class Harness extends ReadyResource {
953
959
  const resident = isResident(pool, key);
954
960
  hlog('completion start', { model, resident, projection: Boolean(projectionModel) });
955
961
  if (!resident) {
956
- await evictIdle(pool);
962
+ await this._makeRoomForCompletion(signal);
957
963
  await this._imageRuntime?.unloadIdle().catch(noop);
958
964
  await this._musicRuntime?.unloadIdle().catch(noop);
959
965
  }
960
966
  const acquireRound = async () => {
961
967
  if (!isResident(pool, key)) {
962
- await evictIdle(pool);
968
+ await this._makeRoomForCompletion(signal);
963
969
  await this._imageRuntime?.unloadIdle().catch(noop);
964
970
  await this._musicRuntime?.unloadIdle().catch(noop);
965
971
  }
@@ -1052,6 +1058,16 @@ export class Harness extends ReadyResource {
1052
1058
  void this._musicRuntime?.unloadIdle().catch(noop);
1053
1059
  }
1054
1060
  }
1061
+ async _makeRoomForCompletion(signal, deadlineMs = MODEL_ROOM_WAIT_MS) {
1062
+ const roomed = await awaitModelCapacity(this._models, this._completionModelCap, deadlineMs, 250, signal);
1063
+ if (!roomed) {
1064
+ hlog('model room wait gave up', {
1065
+ cap: this._completionModelCap,
1066
+ loaded: loadedModels(this._models).length
1067
+ });
1068
+ }
1069
+ return roomed;
1070
+ }
1055
1071
  async _evictWedgedModel(model, key) {
1056
1072
  const faults = (this._modelFaults.get(model) ?? 0) + 1;
1057
1073
  this._modelFaults.set(model, faults);
@@ -0,0 +1,5 @@
1
+ import type { AbortSignal } from 'bare-abort-controller';
2
+ export { closePythonWorker as closeDocxWorker } from '../tools/exec/adapters/python/worker/client.ts';
3
+ export declare function extractDocxText(bytes: Uint8Array, fileName: string, options?: {
4
+ readonly signal?: AbortSignal;
5
+ }): Promise<string>;
@@ -0,0 +1,49 @@
1
+ import b4a from 'b4a';
2
+ import fs from 'fs';
3
+ import { runPython } from "../tools/exec/adapters/python/pyodide.js";
4
+ import { planVendoredInstall, vendoredManifest, wheelsDir } from "../tools/exec/adapters/python/wheels.js";
5
+ export { closePythonWorker as closeDocxWorker } from "../tools/exec/adapters/python/worker/client.js";
6
+ const MAX_OFFICE_BYTES = 20 * 1024 * 1024;
7
+ const PACKAGES = ['python-docx==1.2.0'];
8
+ const EXTRACT = `from docx import Document
9
+ from docx.text.paragraph import Paragraph
10
+ from pathlib import Path
11
+
12
+ def text(container):
13
+ for block in container.iter_inner_content():
14
+ if isinstance(block, Paragraph):
15
+ if block.text.strip():
16
+ yield block.text
17
+ else:
18
+ for row in block.rows:
19
+ yield " | ".join(" ".join(text(cell)) for cell in row.cells)
20
+
21
+ Path("extracted.txt").write_text("\\n".join(text(Document("input.docx"))), encoding="utf-8")
22
+ `;
23
+ export async function extractDocxText(bytes, fileName, options = {}) {
24
+ if (bytes.byteLength > MAX_OFFICE_BYTES) {
25
+ throw new Error(`document is too large to extract: ${bytes.byteLength} bytes, limit ${MAX_OFFICE_BYTES}`);
26
+ }
27
+ const dir = wheelsDir();
28
+ const manifest = vendoredManifest(dir);
29
+ const install = planVendoredInstall(PACKAGES, { dir, manifest });
30
+ if (!dir ||
31
+ !manifest ||
32
+ !install ||
33
+ !['micropip', ...install.packages].every((name) => Object.keys(manifest.files).some((file) => file.replaceAll('_', '-').startsWith(`${name}-`) && fs.existsSync(`${dir}/${file}`)))) {
34
+ throw new Error('bundled DOCX extraction packages are unavailable');
35
+ }
36
+ const result = await runPython(EXTRACT, {
37
+ packages: PACKAGES,
38
+ inputs: [{ path: 'input.docx', bytes }],
39
+ outputs: ['extracted.txt'],
40
+ signal: options.signal
41
+ });
42
+ if (result.exitCode !== 0)
43
+ throw new Error(`${fileName}: ${result.stderr.trim()}`);
44
+ const output = result.files?.get('extracted.txt');
45
+ if (!output) {
46
+ throw new Error(result.rejected?.[0]?.reason ?? 'DOCX extraction produced no text file');
47
+ }
48
+ return b4a.toString(output, 'utf8');
49
+ }
@@ -72,7 +72,7 @@ export declare function loadedModels(pool: PoolLike): LoadedModel[];
72
72
  export declare function isResident(pool: PoolLike, key: string): boolean;
73
73
  export declare function leaseCount(pool: PoolLike, key: string): number;
74
74
  export declare function evictIdle(pool: PoolLike): Promise<void>;
75
- export declare function evictAllWithin(pool: PoolLike, deadlineMs: number, tickMs?: number, signal?: {
75
+ export declare function awaitModelCapacity(pool: PoolLike, cap: number, deadlineMs: number, tickMs?: number, signal?: {
76
76
  aborted?: boolean;
77
77
  }): Promise<boolean>;
78
78
  export declare function evictModelWithin(pool: PoolLike, key: string, deadlineMs: number, tickMs?: number): Promise<boolean>;
@@ -103,7 +103,8 @@ export async function evictIdle(pool) {
103
103
  await pool.gc();
104
104
  }
105
105
  }
106
- export async function evictAllWithin(pool, deadlineMs, tickMs = 250, signal) {
106
+ export async function awaitModelCapacity(pool, cap, deadlineMs, tickMs = 250, signal) {
107
+ const roomed = () => loadedModels(pool).length < cap;
107
108
  let done = false;
108
109
  let timer = null;
109
110
  const expired = new Promise((resolve) => {
@@ -114,7 +115,7 @@ export async function evictAllWithin(pool, deadlineMs, tickMs = 250, signal) {
114
115
  });
115
116
  const evicting = (async () => {
116
117
  await evictIdle(pool);
117
- while (loadedModels(pool).length > 0) {
118
+ while (!roomed()) {
118
119
  if (done || signal?.aborted)
119
120
  return false;
120
121
  await new Promise((resolve) => setTimeout(resolve, tickMs));
@@ -20,6 +20,7 @@ export interface HarnessLlmConfig {
20
20
  contextSize?: number;
21
21
  kvCacheK?: string;
22
22
  kvCacheV?: string;
23
+ completionModelCap?: number;
23
24
  }
24
25
  export interface HarnessChildOptions {
25
26
  runner: StowRunner;
@@ -99,6 +99,8 @@ export async function runPython(code, options = {}) {
99
99
  }
100
100
  await runLock.lock();
101
101
  try {
102
+ if (options.signal?.aborted)
103
+ throw new AbortedError();
102
104
  return await runPythonInWorker(spec, { timeoutMs, signal: options.signal });
103
105
  }
104
106
  finally {
@@ -17,7 +17,8 @@ export function wheelsDir() {
17
17
  const fromHost = process.env.QVAC_PYTHON_WHEELS_DIR;
18
18
  if (fromHost)
19
19
  return fromHost;
20
- const source = new URL('../../../../../wheels', import.meta.url);
20
+ const root = new URL('../../../../../', import.meta.url);
21
+ const source = new URL(root.pathname.endsWith('/dist-lib/') ? '../wheels' : 'wheels', root);
21
22
  if (source.href.includes('.bundle/'))
22
23
  return undefined;
23
24
  return fileURLToPath(source.href);
@@ -13,7 +13,7 @@ const SHUTDOWN_GRACE_MS = 3_000;
13
13
  let active = null;
14
14
  function workerEntry() {
15
15
  return (process.env.QVAC_PYODIDE_WORKER_ENTRY ??
16
- fileURLToPath(new URL('./entry.ts', import.meta.url).href));
16
+ fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './entry.ts' : './entry.js', import.meta.url).href));
17
17
  }
18
18
  async function readMessage(link) {
19
19
  for (;;) {
@@ -13,8 +13,12 @@ interface HRPCHandler {
13
13
  interface HRPCBuilderLike {
14
14
  handlers: ReadonlyArray<HRPCHandler>;
15
15
  }
16
+ interface CallSiteOptions {
17
+ readonly optionalRequests?: ReadonlySet<string>;
18
+ readonly exportName?: string;
19
+ }
16
20
  export declare function writeBind(builder: HRPCBuilderLike, outFile: string): void;
17
- export declare function writeClient(builder: HRPCBuilderLike, outFile: string, exportName?: string): void;
18
- export declare function writeCapabilities(builder: HRPCBuilderLike, outFile: string): void;
21
+ export declare function writeClient(builder: HRPCBuilderLike, outFile: string, options?: CallSiteOptions | string): void;
22
+ export declare function writeCapabilities(builder: HRPCBuilderLike, outFile: string, options?: CallSiteOptions): void;
19
23
  export declare function writeConfigTypes(outFile: string): void;
20
24
  export {};
@@ -13,6 +13,7 @@ function camel(name, prefix = '') {
13
13
  return prefix + cased.join('');
14
14
  }
15
15
  const HEADER = '// This file is autogenerated by schema/build.js\n// Do not edit manually.\n\n';
16
+ const DEFAULT_OPTIONAL_REQUESTS = new Set(['@assistant/empty']);
16
17
  export function writeBind(builder, outFile) {
17
18
  let s = `${HEADER}import { pipeline, Readable } from "streamx";\n\nfunction noop() {}\n\nexport function bindApi(rpc, api) {\n`;
18
19
  for (const handler of builder.handlers) {
@@ -56,24 +57,28 @@ export function writeBind(builder, outFile) {
56
57
  s += '}\n';
57
58
  fs.writeFileSync(outFile, s);
58
59
  }
59
- export function writeClient(builder, outFile, exportName = 'engineClient') {
60
+ export function writeClient(builder, outFile, options = {}) {
61
+ const callSite = typeof options === 'string' ? { exportName: options } : options;
62
+ const { optionalRequests = DEFAULT_OPTIONAL_REQUESTS, exportName = 'engineClient' } = callSite;
60
63
  let s = `${HEADER}function noop() {}\n\nfunction armed(stream) {\n stream.on("error", noop);\n return stream;\n}\n\nexport function ${exportName}(rpc) {\n return {\n`;
61
64
  for (const handler of builder.handlers) {
62
65
  const method = camel(handler.name);
66
+ const input = optionalRequests.has(handler.request?.name ?? '') ? 'input ?? {}' : 'input';
63
67
  if (handler.request?.stream && handler.response?.stream) {
64
68
  s += ` ${method}: () => armed(rpc.${method}()),\n`;
65
69
  }
66
70
  else if (handler.response?.stream) {
67
- s += ` ${method}: (input) => armed(rpc.${method}(input)),\n`;
71
+ s += ` ${method}: (input) => armed(rpc.${method}(${input})),\n`;
68
72
  }
69
73
  else {
70
- s += ` ${method}: (input) => rpc.${method}(input),\n`;
74
+ s += ` ${method}: (input) => rpc.${method}(${input}),\n`;
71
75
  }
72
76
  }
73
77
  s += ' };\n}\n';
74
78
  fs.writeFileSync(outFile, s);
75
79
  }
76
- export function writeCapabilities(builder, outFile) {
80
+ export function writeCapabilities(builder, outFile, options = {}) {
81
+ const { optionalRequests = DEFAULT_OPTIONAL_REQUESTS } = options;
77
82
  let s = HEADER;
78
83
  s += "import type * as T from './hyperschema/types.d.ts'\n\n";
79
84
  s +=
@@ -112,7 +117,7 @@ export function writeCapabilities(builder, outFile) {
112
117
  const method = camel(handler.name);
113
118
  const req = `T.${tsName(handler.request.name)}`;
114
119
  const res = `T.${tsName(handler.response.name)}`;
115
- const input = handler.request.name === '@assistant/empty' ? `input?: ${req}` : `input: ${req}`;
120
+ const input = optionalRequests.has(handler.request.name) ? `input?: ${req}` : `input: ${req}`;
116
121
  if (handler.request?.stream && handler.response?.stream) {
117
122
  client += ` ${method}(): UploadStream<${req}, ${res}>\n`;
118
123
  server += ` ${method}(stream: AsyncIterable<${req}> | HandlerFrames<${req}>): AsyncIterable<${res}>\n`;
@@ -0,0 +1,75 @@
1
+ ---
2
+ id: ADR-2026-05-29-kvcache-tool-turns
3
+ status: superseded
4
+ introduced_by: PR-228
5
+ supersedes: []
6
+ superseded_by: [ADR-2026-06-02-kvcache-tool-turns-reenabled]
7
+ ---
8
+
9
+ # KV-cache session is disabled for tool-using completions
10
+
11
+ > **Superseded** by
12
+ > [ADR-2026-06-02-kvcache-tool-turns-reenabled](./2026-06-02-kvcache-tool-turns-reenabled.md):
13
+ > the cache is re-enabled for tool turns now that models load with
14
+ > `toolsMode: "dynamic"` (the SDK's DYNAMIC path is coherent, including across
15
+ > reload). Kept for history.
16
+
17
+ ## Decision
18
+
19
+ The `@qvac/sdk` session prompt cache (`completion({ kvCache })`) is passed
20
+ **only for tool-free completions** — `kvCache` is set only when
21
+ `tools.length === 0`. Agentic / tool-using turns always run without the session
22
+ cache.
23
+
24
+ ## Context
25
+
26
+ The SDK session cache has two paths (see
27
+ `@qvac/sdk → server/bare/plugins/llamacpp-completion/ops/completion-stream.ts` in the SDK):
28
+
29
+ - **STATIC** (no `tools`): on a cache hit it sends `history.slice(savedCount)`
30
+ and self-heals a stale boundary by re-sending the full history. Robust against
31
+ our reconstructed history.
32
+ - **DYNAMIC** (`tools` present): on a cache hit it sends only a role-based
33
+ _fragment_ of the history and trusts the SDK's own tool-chain flow to have
34
+ anchored the rest of the cache.
35
+
36
+ Our agentic loop builds (and on reload reconstructs from a persisted transcript)
37
+ the full history its own way. That does not satisfy the DYNAMIC path's
38
+ assumptions, so on a tool-cache hit the model receives an incoherent prompt and
39
+ emits `<tool_call>` markup as plain text instead of a parseable tool call —
40
+ silently breaking tool calling (no parse, no approval, no execution). Tools are
41
+ text-parsed (`toolCalling: "textParse"`), so a corrupted prompt degrades to a
42
+ plain assistant message rather than erroring.
43
+
44
+ ## Alternatives Considered
45
+
46
+ - **Always-on per-chat `kvCache`** (the state that introduced the regression):
47
+ breaks tool calling as above.
48
+ - **Auto-cache (`kvCache: true`)**: tool-call turns are never cached, and a
49
+ verbatim `cacheableAssistantContent` echo is required for a hit —
50
+ incompatible with our thinking-stripped reconstruction.
51
+ - **Conform to the SDK's tool-chain contract** (collapse consecutive messages +
52
+ `deleteCache` on divergence, like the legacy desktop worker): viable but a
53
+ separate project, tracked in `.ai/cursor/plans/kvcache-tool-turn-reuse.md`.
54
+
55
+ ## Consequences
56
+
57
+ - Tool turns re-prefill the full history each call (no session-cache reuse).
58
+ Tool calling stays correct; this is the prior known-good behavior.
59
+ - Tool-free turns (plain chat / title / summary) still benefit from the STATIC
60
+ cache path.
61
+ - Regression guards are GPU-free and mock-based:
62
+ `qvac-app → core/src/services/message/agentic-completion.test.ts` (argument-contract:
63
+ never send `kvCache` with `tools`) and the spec-fake
64
+ `qvac-app → core/src/services/message/__fixtures__/fake-completion-sdk.ts` (reproduces the
65
+ cache-hit tool-parse failure).
66
+ - Revisit if the SDK adds a "send full history, you slice it" mode for tools, or
67
+ when the conformance project lands.
68
+
69
+ ## When Agents Should Read This
70
+
71
+ Read this before changing how `kvCache` is passed to `sdk.completion`, the
72
+ agentic loop's history construction, or transcript reconstruction —
73
+ `qvac-app → core/src/services/message/agentic-completion.ts`,
74
+ `qvac-app → core/src/services/message/agentic-loop.ts`,
75
+ `qvac-app → core/src/services/common/transcript.ts`.
@@ -0,0 +1,127 @@
1
+ ---
2
+ id: ADR-2026-06-02-kvcache-tool-turns-reenabled
3
+ status: superseded
4
+ introduced_by: PR-QVAC-19770
5
+ supersedes: [ADR-2026-05-29-kvcache-tool-turns]
6
+ superseded_by: [ADR-2026-08-20-kvcache-static-placement]
7
+ ---
8
+
9
+ # KV-cache session is re-enabled for tool-using completions
10
+
11
+ > **Superseded** by
12
+ > [ADR-2026-08-20-kvcache-static-placement](./2026-08-20-kvcache-static-placement.md):
13
+ > the half of this record that still holds is that `kvCache: 'chat-<id>'` rides
14
+ > every completion. The dynamic-mode half was built on a false premise — the
15
+ > addon's compact-tools anchoring never ran on any shipped model (its arch gate
16
+ > is exact `qwen3`; ours are `qwen35`/`qwen36`), so the coherence this record
17
+ > credits to dynamic mode was never exercised, and the "static never surfaced
18
+ > the tool-call format" claim did not survive re-testing. core-v2 loads static
19
+ > with no sliding window since 2026-08-20; mechanics in
20
+ > `docs/kvcache-reference.md`. The `QVAC_PROMPT_CACHE` kill-switch and
21
+ > `deleteChatPromptCache` described below were core-v1 machinery and do not
22
+ > exist in the current runtime.
23
+
24
+ ## Decision
25
+
26
+ The `@qvac/sdk` session prompt cache (`completion({ kvCache: "chat-<id>" })`) is
27
+ passed for **all** completions with a chatId — tool-using and tool-free alike —
28
+ gated only by the `QVAC_PROMPT_CACHE` kill-switch. This reverses
29
+ ADR-2026-05-29, which disabled the cache whenever tools were present.
30
+
31
+ ## Context
32
+
33
+ The prior decision disabled tool-turn caching because the SDK's DYNAMIC cache
34
+ path broke tool-call parsing: it ships a role-based history fragment and trusts
35
+ the addon's tool-chain anchoring, which our reconstructed history did not
36
+ satisfy → `<tool_call>` markup leaked as text. Two things changed:
37
+
38
+ - Models loaded with `modelConfig.toolsMode: "dynamic"` (QVAC-19660). In
39
+ dynamic mode the addon anchors tools per-turn (compact-tools), excludes the
40
+ tool set from the cache config hash, and the fragment slicing keeps
41
+ multi-round tool turns coherent — including across a model evict/reload.
42
+ **No longer true as of `@qvac/sdk` 0.18 (QVAC-23936):** the key was removed
43
+ and the parser rejects it, so static is the only placement. Dynamic was in
44
+ any case a silent no-op on every arch we ship — the addon gates
45
+ `tools_compact` on exact-arch `qwen3` while our catalog is `qwen35`/`qwen36` —
46
+ and the SDK still sliced as-if-dynamic, which made the cache the chat's
47
+ _memory_ rather than an accelerator: after any cache reset it sent only the
48
+ last exchange. Under static placement the tool block sits in the addon's
49
+ protected prefix instead, and 0.18 keeps it out of the primed prefix
50
+ (QVAC-23251) so the qwen3.5 template no longer raises 'No user query found in
51
+ messages.'
52
+ - An SDK e2e guard (`kv-cache-tools-dynamic-reuse`, plus a qwen35-dialect
53
+ variant) verifies a custom-key multi-round tool chain reuses the on-disk cache
54
+ (`cacheTokens > 0`) and keeps tool calls parseable across an evict/reload.
55
+
56
+ With the DYNAMIC path proven coherent across reload, the heavyweight
57
+ "reconstruction invalidation / byte-identical history" work the prior ADR
58
+ deferred is unnecessary; divergence is handled with targeted invalidation.
59
+
60
+ ## Consequences
61
+
62
+ - **Do not pass `toolsMode` at model load.** SDK 0.18 removed the key and its
63
+ parser rejects an unrecognised one, so passing it fails the load outright and
64
+ the app parks on the splash. Static placement is the default and the only mode.
65
+ - Dropping a key from `modelConfig` re-keys `modelId`, which is identity-by-config,
66
+ so expect one cold prime per chat on the first run after such a change.
67
+ - Tool turns reuse the session KV cache: lower TTFT/prefill on multi-round tool
68
+ turns and follow-ups (manually: round-2 TTFT ~1.1s → ~0.3s; cacheTokens grows
69
+ monotonically across a conversation).
70
+ - `QVAC_PROMPT_CACHE` (default on; `0`/`false`/`off`) disables all
71
+ prompt caching as a kill-switch.
72
+ - The per-chat cache is explicitly invalidated on chat delete and agent delete
73
+ (which cascades to its chats) via `deleteChatPromptCache`, plus a defensive
74
+ drop on retry (see below). Edit / branch / truncate flows do not exist yet and
75
+ need no hook.
76
+ - **Why stop / failed / new-message-after-failure need no invalidation.** The
77
+ cache only ever holds _successful_ turns: the SDK rolls back uncommitted turns
78
+ on abort (the custom-key path commits only when not aborted and tokens were
79
+ produced), and failed / canceled / not_delivered replies are excluded from
80
+ model-visible history (`isModelVisibleMessage`, `qvac-app → core/src/services/common/history.ts`).
81
+ So a stopped or failed turn leaves neither a committed cache entry nor a
82
+ visible history row — the reconstructed history and the cached prefix stay
83
+ aligned, and the next turn (new message or retry) does not slice against a
84
+ stale prefix. The SDK's own `kv-cache-cancel-then-new-prompt` e2e covers this
85
+ cancel-then-continue coherence. The **retry** `deleteChatPromptCache` is
86
+ therefore belt-and-suspenders, not load-bearing: it costs one re-prime on a
87
+ rare, user-initiated path and removes any dependence on the rollback timing.
88
+ - **Mobile scope.** Mobile now runs the same assistant and harness completion path
89
+ inside its supervised Bare worklets, so the per-chat cache key reaches the SDK
90
+ without the retired worker-to-host SDK bridge.
91
+ - `chatPromptCacheKey` is the single source of truth for the `chat-<id>` key
92
+ shared by the completion path and the invalidation callers.
93
+ - Known SDK limitation: on a multimodal (M-RoPE, `n_pos_per_embd > 1`) model such
94
+ as Qwen3.5, the SDK's KV sliding-window position shift (`seq_add`/`seq_div` via
95
+ `trySlidePrefill`) GGML_ASSERT-crashes the worker when the context fills. A
96
+ standalone SDK repro confirmed this is **independent of `toolsMode` (static and
97
+ dynamic both crash) and of whether tools are passed**, and is not caused by the
98
+ prompt cache — it is the sliding window itself, unsupported for multi-pos
99
+ models. Interim mitigation (separate from the kill-switch): we temporarily
100
+ stopped setting `n_discarded` at model load so on overflow the SDK took the
101
+ catchable `CONTEXT_OVERFLOW` path instead of the crashing slide.
102
+ **Resolved (2026-06-23):** `@qvac/sdk` 0.13.5 (llm-llamacpp 0.24.0) ships
103
+ M-RoPE/iM-RoPE sliding-context support, so `n_discarded` is set again in
104
+ `@qvac/sdk → model/model-load-config.ts` and the workbench no longer trims history. See
105
+ [`2026-06-23-append-only-history.md`](./2026-06-23-append-only-history.md).
106
+ - Regression guards updated: the contract test now asserts `kvCache` IS sent
107
+ with tools; the spec-fake models dynamic mode (healthy on a tools cache hit);
108
+ invalidation hooks are covered.
109
+ - **The in-repo suite cannot prove real dynamic-mode coherence.** The spec-fake
110
+ (`@qvac/sdk → __fixtures__/fake-completion-sdk.ts`) is parameterized to treat dynamic mode
111
+ as healthy, so the in-repo tests guard against regressing the model load back
112
+ to static mode and against dropping the `kvCache` key — but they do not prove
113
+ the SDK's real DYNAMIC path stays coherent against our reconstructed history.
114
+ That proof lives in the out-of-repo SDK e2e (`kv-cache-tools-dynamic-reuse`
115
+ and its qwen35 variant), which this repo's CI cannot run.
116
+
117
+ ## When Agents Should Read This
118
+
119
+ Read this before changing how `kvCache` is passed to `sdk.completion`,
120
+ `toolsMode` at model load, the agentic loop's history construction, transcript
121
+ reconstruction, or the cache invalidation hooks —
122
+ `qvac-app → core/src/services/message/agentic-completion.ts`,
123
+ `qvac-app → core/src/services/model/service.ts`,
124
+ `qvac-app → core/src/services/execution/service.ts`,
125
+ `qvac-app → core/src/services/api/domain-apis.ts`,
126
+ `@qvac/core → lib/assistant.ts`, and
127
+ `lib/capabilities/completion.ts`.