@qvac/core 0.11.1 → 0.11.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +88 -1
  2. package/README.md +19 -16
  3. package/dist-lib/lib/api/chunks.js +4 -2
  4. package/dist-lib/lib/api/rag.js +7 -5
  5. package/dist-lib/lib/assistant/history.d.ts +1 -0
  6. package/dist-lib/lib/assistant/history.js +1 -17
  7. package/dist-lib/lib/assistant/message.d.ts +4 -1
  8. package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
  9. package/dist-lib/lib/assistant/pdf-budget.js +10 -11
  10. package/dist-lib/lib/assistant/pdf-pages.d.ts +18 -0
  11. package/dist-lib/lib/assistant/pdf-pages.js +75 -0
  12. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  13. package/dist-lib/lib/assistant-tree.js +4 -2
  14. package/dist-lib/lib/assistant.d.ts +10 -6
  15. package/dist-lib/lib/assistant.js +205 -140
  16. package/dist-lib/lib/compaction.d.ts +2 -1
  17. package/dist-lib/lib/compaction.js +14 -4
  18. package/dist-lib/lib/constants.d.ts +3 -0
  19. package/dist-lib/lib/constants.js +3 -0
  20. package/dist-lib/lib/core/chat-activity.js +12 -15
  21. package/dist-lib/lib/core/chunk.d.ts +9 -1
  22. package/dist-lib/lib/core/chunk.js +42 -8
  23. package/dist-lib/lib/core/document-attachment.d.ts +6 -3
  24. package/dist-lib/lib/core/document-attachment.js +10 -5
  25. package/dist-lib/lib/core/files/attachments.js +3 -3
  26. package/dist-lib/lib/core/files/index.d.ts +1 -1
  27. package/dist-lib/lib/core/files/index.js +3 -2
  28. package/dist-lib/lib/core/files/processing/index.d.ts +1 -0
  29. package/dist-lib/lib/core/files/processing/index.js +8 -0
  30. package/dist-lib/lib/core/files/processing/settle.d.ts +1 -0
  31. package/dist-lib/lib/core/files/processing/settle.js +20 -13
  32. package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
  33. package/dist-lib/lib/core/files/rag/index.js +1 -1
  34. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -0
  35. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +3 -0
  36. package/dist-lib/lib/core/hosted-agents.d.ts +3 -0
  37. package/dist-lib/lib/core/hosted-agents.js +19 -0
  38. package/dist-lib/lib/core/notification.js +16 -13
  39. package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
  40. package/dist-lib/lib/core/pdf/figure.js +64 -0
  41. package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
  42. package/dist-lib/lib/core/pdf/rasterize.js +72 -22
  43. package/dist-lib/lib/core.d.ts +6 -0
  44. package/dist-lib/lib/core.js +5 -0
  45. package/dist-lib/lib/credential-pusher.d.ts +11 -7
  46. package/dist-lib/lib/credential-pusher.js +40 -8
  47. package/dist-lib/lib/integrations/setup-routes.js +7 -5
  48. package/dist-lib/lib/kv-cache-retention.d.ts +34 -0
  49. package/dist-lib/lib/kv-cache-retention.js +98 -0
  50. package/dist-lib/schema/content.js +2 -1
  51. package/dist-lib/schema/qvac.js +4 -2
  52. package/dist-lib/spec/hrpc/messages.js +4 -10
  53. package/dist-lib/spec/hyperschema/index.js +4 -10
  54. package/dist-lib/spec/hyperschema/schema.json +2 -1
  55. package/dist-lib/spec/hyperschema/types.d.ts +1 -1
  56. package/dist-lib/spec/mesh/hyperdb/messages.js +34 -14
  57. package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
  58. package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
  59. package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
  60. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  61. package/package.json +6 -3
  62. package/tui/app.mjs +743 -266
  63. package/tui/boot.mjs +2 -2
  64. package/tui/lib/connect-sheet.mjs +350 -0
  65. package/tui/lib/setup.mjs +59 -25
  66. package/tui/lib/skill-connect.mjs +145 -0
@@ -21,8 +21,8 @@ import { meshScoped } from './mesh-scoped-engine.js';
21
21
  import { buildTitlePrompt, deriveChatTitle, isDefaultChatTitle, parseTitleResponse } from './chat-title.js';
22
22
  import { matchApprovalRule } from './core/approval-rules.js';
23
23
  import { isAudioAttachment } from './core/audio-attachment.js';
24
- import { attachTurnDocumentIds, formatRagContext, chooseRagNotice, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, createRagSourceTagFilter, citedByPhraseOverlap, formatFullDocumentContext, isFullTextHit, isTruncatedFullTextHit, sourcesFromHits, WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from './core/files/index.js';
25
- import { annotateDocumentAttachments, documentUnreadNotice, documentUnreadReason, pdfPagesCutNotice, pdfPagesSkippedNotice } from './core/document-attachment.js';
24
+ import { attachTurnDocumentIds, formatRagContext, chooseRagNotice, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, createRagSourceTagFilter, citedByPhraseOverlap, formatFullDocumentContext, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, sourcesFromHits, WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from './core/files/index.js';
25
+ import { annotateDocumentAttachments, documentUnreadNotice, documentUnreadReason, PDF_PAGES_SOURCE_LINE, pdfPagesCutNotice, pdfPagesShownNotice } from './core/document-attachment.js';
26
26
  import { imageCacheKey, imageCacheName, sweepImageCache } from './core/image/cache-sweep.js';
27
27
  import { IMAGE_MIMES, normalizeImageForModel } from './core/image/normalize.js';
28
28
  import { createPdfRasterizer } from './core/pdf/rasterize.js';
@@ -35,7 +35,9 @@ import { fsGrantsFrom, grantedDirsFrom } from './fs-grants.js';
35
35
  import { persistentDir } from './persistent-dir.js';
36
36
  import { ModelCatalogRevisionHub } from './model-catalog-revision.js';
37
37
  import { agentRekeyFingerprint, agentSwitched, chatKvCacheKey, chatRepairKvCacheKey, disappearedChatIds } from './kv-cache-reclaim.js';
38
- import { buildCompactionPrompt, canCompact, foldReducesPrompt, COMPACTION_KEEP_TURNS, COMPACTION_TURN_PREFIX, isContextOverflowFailure, nearContextLimit, compactionBoundary, lastCompactedThroughId, parseCompactionSummary } from './compaction.js';
38
+ import { KvCacheRetention } from './kv-cache-retention.js';
39
+ import { KV_CACHE_RETENTION_KEY } from './constants.js';
40
+ import { buildCompactionPrompt, canCompact, foldReducesPrompt, COMPACTION_KEEP_TURNS, COMPACTION_TURN_PREFIX, isContextOverflowFailure, nearContextLimit, compactionBoundary, foldsThrough, lastCompactedThroughId, parseCompactionSummary } from './compaction.js';
39
41
  import { ModelDownloadReconciler, musicModelChoiceOf } from './model-download.js';
40
42
  import { AbortedError, raceAbort } from '@qvac/harness/race-abort';
41
43
  import { RunTranscript } from './run-transcript.js';
@@ -49,7 +51,8 @@ import { frameTimes, probeVideo, sampleFrames } from './core/video/frames.js';
49
51
  import { packVideoDigest, parseVideoDigestHeader, unpackVideoDigest, VIDEO_DIGEST_MIME, videoDigestFileName } from './core/video/digest.js';
50
52
  import { VIDEO_FRAME_EDGE, videoFrameCount } from './assistant/video-budget.js';
51
53
  import { drainBody, raceInactivity } from './assistant/fetch-body.js';
52
- import { pdfPageBudget } from './assistant/pdf-budget.js';
54
+ import { pdfImageRoom } from './assistant/pdf-budget.js';
55
+ import { packPdfSelection, parsePdfSelection, PDF_PAGES_MIME, pdfSelectionFileName, selectPdfPages } from './assistant/pdf-pages.js';
53
56
  import { effectiveCtxSize } from './api/effective-ctx-size.js';
54
57
  import { GENERIC_ACKNOWLEDGEMENTS, HISTORY_TYPES, SKILL_TOOL, TERMINAL_STATES, couldNameSenders, historyFromChunks, inputAvailableToRun, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
55
58
  import { errorMessage, isRecord } from './guards.js';
@@ -112,19 +115,16 @@ const RunOccupancySchema = z.object({
112
115
  export const CAPTURE_INFERENCE_CALLS_KEY = 'inspector.captureInferenceCalls';
113
116
  export const CAPTURE_ON = 'on';
114
117
  export const CAPTURE_OFF = 'off';
115
- function pdfCut(doc, from) {
116
- const pageCount = doc.pageCount();
117
- for (let page = from; page < pageCount; page++) {
118
- try {
119
- const flags = doc.pageFlags(page);
120
- if (flags.hasImage || !flags.hasText) {
121
- return { lastPage: page, pageCount };
122
- }
123
- }
124
- catch {
125
- }
126
- }
127
- return undefined;
118
+ const NO_PDF_PAGES = { selection: { pageCount: 0, shots: [] }, names: [], labels: [] };
119
+ function pdfNotices(fileName, pages, wholeText) {
120
+ const { cutAt, pageCount } = pages.selection;
121
+ const shown = pages.names.length;
122
+ return [
123
+ ...(shown ? [pdfPagesShownNotice(fileName, pages.labels)] : []),
124
+ ...(cutAt === undefined
125
+ ? []
126
+ : [pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText })])
127
+ ];
128
128
  }
129
129
  function attachPages(turn, files) {
130
130
  if (!files.length)
@@ -152,8 +152,22 @@ function videoAudioBlock(fileName, { audio, transcript }) {
152
152
  }
153
153
  return unheardAudioNotice(`The audio of the attached video "${fileName}"`, 'transcription failed');
154
154
  }
155
- function pdfPageCacheName(pdf, page) {
156
- return imageCacheName({ blobId: pdf.blobId, fileName: `${pdf.fileName}.p${page}.jpg` });
155
+ async function cachedPdfPages(selection, { dir, pdf, pinned }) {
156
+ const names = selection.shots.map((shot) => pdfShotCacheName(pdf, shot));
157
+ for (const name of names) {
158
+ pinned.add(name);
159
+ }
160
+ const files = names.map((name) => path.join(dir, name));
161
+ const cached = await Promise.all(files.map((file) => cacheOrTouch(file, noBytes)));
162
+ return cached.every(Boolean)
163
+ ? { selection, names, labels: selection.shots.map((shot) => shot.label) }
164
+ : null;
165
+ }
166
+ function pdfShotCacheName(pdf, shot) {
167
+ const box = shot.clip
168
+ ? `.c${[shot.clip.x, shot.clip.y, shot.clip.width, shot.clip.height].map(Math.round).join('_')}`
169
+ : '';
170
+ return imageCacheName({ blobId: pdf.blobId, fileName: `${pdf.fileName}.p${shot.page}${box}.jpg` });
157
171
  }
158
172
  async function cacheOrTouch(file, produce) {
159
173
  const cached = await fs.promises.stat(file).catch(noop);
@@ -262,18 +276,20 @@ export class QvacAssistant extends ReadyResource {
262
276
  _backgroundReplays;
263
277
  _inferenceCalls;
264
278
  _runReplay;
279
+ _kvRetention;
265
280
  _lifecycle;
266
281
  _audioTranscripts;
267
282
  _videoProbes;
268
- _pdfPages;
283
+ _pdfSelections;
269
284
  _chats;
270
285
  _runImageCacheNames;
271
286
  _reconciledCompactions;
272
287
  _chatAttribution;
288
+ _knowledgeCaps = new Map();
273
289
  _modelCatalog = new ModelCatalogRevisionHub();
274
290
  _onHarnessStatus;
275
291
  _onModelCatalogEvent;
276
- constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, modelReloadDelayMs = MODEL_RELOAD_DELAY_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
292
+ constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, kvCacheRetentionMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, modelReloadDelayMs = MODEL_RELOAD_DELAY_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
277
293
  super();
278
294
  this._engine = engine;
279
295
  this._harness = harness;
@@ -314,10 +330,17 @@ export class QvacAssistant extends ReadyResource {
314
330
  this._backgroundReplays = new Set();
315
331
  this._inferenceCalls = new InferenceCallStore(inferenceCallsDir ?? persistentDir('inspector'));
316
332
  this._runReplay = new RunReplayStore(runReplayDir ?? persistentDir('run-replay'));
333
+ this._kvRetention = new KvCacheRetention(engine, {
334
+ active: () => !this.closing && !this.suspended,
335
+ running: (chatId) => [...this._runs.values()].some((run) => run.chatId === chatId),
336
+ deleteChatKvCache: (chatId) => this._deleteChatKvCache(chatId, 'idle past retention'),
337
+ defaultMs: kvCacheRetentionMs,
338
+ logger
339
+ });
317
340
  this._lifecycle = new AbortController();
318
341
  this._audioTranscripts = new Map();
319
342
  this._videoProbes = new Map();
320
- this._pdfPages = new Map();
343
+ this._pdfSelections = new Map();
321
344
  this._chats = new Map();
322
345
  this._runImageCacheNames = new Map();
323
346
  this._reconciledCompactions = new Map();
@@ -388,6 +411,7 @@ export class QvacAssistant extends ReadyResource {
388
411
  this._downloads.start();
389
412
  this._meshListWatch.start();
390
413
  this._accountWatch.start();
414
+ this._kvRetention.start(() => this._meshConsumers.keys());
391
415
  void this._bootOnline();
392
416
  this._triggerModelWarm();
393
417
  void this.onlineReady().then(() => this._triggerModelWarm(), noop);
@@ -780,6 +804,7 @@ export class QvacAssistant extends ReadyResource {
780
804
  }
781
805
  async _close() {
782
806
  this._stopMeshIntake();
807
+ this._kvRetention.stop();
783
808
  this._harness.off?.('status', this._onHarnessStatus);
784
809
  this._harness.off?.('model', this._onModelCatalogEvent);
785
810
  this._warmAbort.abort(RUNNER_CLOSED);
@@ -1150,6 +1175,13 @@ export class QvacAssistant extends ReadyResource {
1150
1175
  await this._sweepInferenceCalls();
1151
1176
  }
1152
1177
  }
1178
+ async _sweepIdleChatCaches(meshId, frames) {
1179
+ for await (const _frame of frames) {
1180
+ if (this.closing || this.suspended)
1181
+ return;
1182
+ void this._kvRetention.sweep(meshId);
1183
+ }
1184
+ }
1153
1185
  _dropDeletedChat(chatId) {
1154
1186
  for (const run of this._runs.values()) {
1155
1187
  if (run.chatId === chatId) {
@@ -1406,7 +1438,8 @@ export class QvacAssistant extends ReadyResource {
1406
1438
  const watches = [
1407
1439
  this._streamWatch(`run-requests:${meshId}`, () => this._engine.runRequestsWatch({ meshId }), (frames) => this._dispatchRunRequests(meshId, frames)),
1408
1440
  this._streamWatch(`chat-list:${meshId}`, () => this._engine.chatsWatch({ meshId }), (frames) => this._reclaimDeletedChats(frames)),
1409
- this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._followAgentEdits(meshId, frames))
1441
+ this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._followAgentEdits(meshId, frames)),
1442
+ this._streamWatch(`kv-cache-retention:${meshId}`, () => this._engine.configWatch({ key: KV_CACHE_RETENTION_KEY }), (frames) => this._sweepIdleChatCaches(meshId, frames))
1410
1443
  ];
1411
1444
  return {
1412
1445
  start: () => watches.forEach((watch) => watch.start()),
@@ -1723,7 +1756,8 @@ export class QvacAssistant extends ReadyResource {
1723
1756
  transcript,
1724
1757
  trigger: 'before-run',
1725
1758
  signal: controller.signal,
1726
- meshId
1759
+ meshId,
1760
+ requestSeq: request.seq
1727
1761
  });
1728
1762
  const toolConfigsPromise = this._loadToolConfigs(engine, agent.id);
1729
1763
  const fsGrantsPromise = lender ? Promise.resolve(NO_FS_GRANTS) : this._loadFsGrants(agent.id);
@@ -1786,7 +1820,8 @@ export class QvacAssistant extends ReadyResource {
1786
1820
  const seesImages = sendsFiles && carriesMedia && (await this._seesImages(model));
1787
1821
  let unreadableImages = new Map();
1788
1822
  const sampled = [];
1789
- let pdfPages = { rendered: new Set(), cuts: [] };
1823
+ let pdfPages = { rendered: new Set(), notices: [] };
1824
+ const selectedPdfs = [];
1790
1825
  if (!seesImages) {
1791
1826
  const dropped = messages.reduce((count, turn) => count + (turn.images?.length ?? 0), 0);
1792
1827
  if (dropped && (this._harness.supportsImages ?? this._info.supportsImages)) {
@@ -1891,6 +1926,7 @@ export class QvacAssistant extends ReadyResource {
1891
1926
  let sentRun = null;
1892
1927
  let inputLength = 0;
1893
1928
  let asked;
1929
+ let injectedHits = 0;
1894
1930
  const writeText = async (type, raw) => {
1895
1931
  const text = type === 'content' ? sourceTagFilter.push(raw) : raw;
1896
1932
  if (!text) {
@@ -1902,17 +1938,24 @@ export class QvacAssistant extends ReadyResource {
1902
1938
  await transcript.text(type, text);
1903
1939
  };
1904
1940
  const failRun = async (message) => {
1905
- const folded = isContextOverflowFailure(message) &&
1941
+ const overflowed = isContextOverflowFailure(message);
1942
+ const folded = overflowed &&
1906
1943
  (await this._runCompaction({
1907
1944
  chatId,
1908
1945
  transcript,
1909
1946
  trigger: 'overflow',
1910
1947
  signal: controller.signal,
1911
- meshId
1948
+ meshId,
1949
+ requestSeq: request.seq
1912
1950
  }));
1951
+ const shrunk = overflowed && !folded && injectedHits > 1;
1952
+ if (shrunk) {
1953
+ this._knowledgeCaps.set(chatId, Math.floor(injectedHits / 2));
1954
+ this._log?.info('runs', 'knowledge shrunk for retry', { runId, hits: injectedHits });
1955
+ }
1913
1956
  runFailure = runFailureNotice(message);
1914
1957
  await transcript.status({ state: 'failed', reason: runFailure, metrics });
1915
- if (folded) {
1958
+ if (folded || shrunk) {
1916
1959
  await engine
1917
1960
  .runAgent({ chatId, agentId: agent.id, parentRunId: runId })
1918
1961
  .catch((err) => this._log?.warn('runs', 'compaction retry not queued', { runId, err }));
@@ -1975,9 +2018,18 @@ export class QvacAssistant extends ReadyResource {
1975
2018
  await transcript.status({ state: 'failed', reason: outcome.reason, metrics });
1976
2019
  return;
1977
2020
  }
1978
- const injected = knowledge
1979
- ? await this._injectKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords })
2021
+ const cap = this._knowledgeCaps.get(chatId);
2022
+ this._knowledgeCaps.delete(chatId);
2023
+ const fitted = knowledge && cap ? { ...knowledge, hits: knowledge.hits.slice(0, cap) } : knowledge;
2024
+ const injected = fitted
2025
+ ? await this._injectKnowledge(fitted, {
2026
+ transcript,
2027
+ sourceTagFilter,
2028
+ retrievedRecords,
2029
+ pictured: seesImages && messages.some((turn) => turn.pdfs?.length)
2030
+ })
1980
2031
  : { documentsInjected: false, sourcesWritten: false, ragInjected: false };
2032
+ injectedHits = injected.ragInjected && fitted ? fitted.hits.length : 0;
1981
2033
  const hasAudio = messages.some((turn) => turn.audioAttachments?.length || turn.videos?.length);
1982
2034
  if (hasAudio) {
1983
2035
  modelWatch.transcribe(true);
@@ -1998,14 +2050,22 @@ export class QvacAssistant extends ReadyResource {
1998
2050
  }
1999
2051
  backfillImageOnlyTurns(messages);
2000
2052
  if (seesImages) {
2053
+ const hits = injected.documentsInjected && fitted ? fitted.hits : [];
2001
2054
  pdfPages = await this._hydratePdfPages(messages, {
2002
2055
  signal: controller.signal,
2003
2056
  runId,
2004
- window: { ctxSize: configuredAgent.ctxSize, modelName: model }
2057
+ window: { ctxSize: configuredAgent.ctxSize, modelName: model },
2058
+ reading: { chatId, meshId, folded },
2059
+ textChars: hits.reduce((chars, hit) => chars + hit.content.length, 0),
2060
+ wholeIds: new Set(hits
2061
+ .filter((hit) => !isTruncatedFullTextHit(hit))
2062
+ .map(fullTextAttachmentId)
2063
+ .filter((id) => id !== null)),
2064
+ selected: selectedPdfs
2005
2065
  });
2006
2066
  }
2007
2067
  annotateDocumentAttachments(messages);
2008
- for (const { turn, notice } of pdfPages.cuts) {
2068
+ for (const { turn, notice } of pdfPages.notices) {
2009
2069
  turn.content = `${turn.content}\n${notice}`;
2010
2070
  }
2011
2071
  const unreadIds = attachTurnIds.filter((id) => !pdfPages.rendered.has(id));
@@ -2127,6 +2187,9 @@ export class QvacAssistant extends ReadyResource {
2127
2187
  this._log?.info('runs', 'run metrics', { runId, ...runMetricsSummary(rest) });
2128
2188
  continue;
2129
2189
  }
2190
+ if (event.type === 'inference-sent' && event.outcome === 'overflowed') {
2191
+ this._log?.info('runs', 'context overflow', { runId, reason: event.reason });
2192
+ }
2130
2193
  if (event.type === 'inference-sent') {
2131
2194
  this._inspectSent(inspecting, chatId, {
2132
2195
  runId,
@@ -2210,6 +2273,7 @@ export class QvacAssistant extends ReadyResource {
2210
2273
  });
2211
2274
  titleOutcome = 'completed';
2212
2275
  await this._storeVideoDigests(sampled, { meshId, chatId, runId, deviceId });
2276
+ await this._storePdfSelections(selectedPdfs, { meshId, chatId, runId, deviceId });
2213
2277
  }
2214
2278
  catch (err) {
2215
2279
  const outcome = controller.signal.aborted
@@ -2254,6 +2318,7 @@ export class QvacAssistant extends ReadyResource {
2254
2318
  if (titleOutcome && !diagramRepair) {
2255
2319
  void this._autoTitle(engine, {
2256
2320
  chatId,
2321
+ history,
2257
2322
  messages,
2258
2323
  model,
2259
2324
  agent: runAgent,
@@ -2847,39 +2912,41 @@ export class QvacAssistant extends ReadyResource {
2847
2912
  });
2848
2913
  return file;
2849
2914
  }
2850
- async _hydratePdfPages(messages, { signal, runId, window }) {
2915
+ async _hydratePdfPages(messages, { signal, runId, window, reading, textChars, wholeIds, selected }) {
2851
2916
  const rendered = new Set();
2852
- const cuts = [];
2917
+ const notices = [];
2853
2918
  const turns = messages.filter((turn) => turn.pdfs?.length);
2854
2919
  if (!turns.length) {
2855
- return { rendered, cuts };
2920
+ return { rendered, notices };
2856
2921
  }
2857
2922
  const dir = this._imageCacheDir;
2858
2923
  await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
2859
2924
  const pinned = this._runImageCacheNames.get(runId) ?? new Set();
2860
2925
  this._runImageCacheNames.set(runId, pinned);
2861
2926
  const latestUser = messages.findLast((turn) => turn.role === 'user');
2862
- const maxPages = await this._pdfPageBudget(window);
2927
+ const measured = await this._pdfReading(window, reading);
2863
2928
  for (const turn of turns) {
2864
- const { shown, notices } = await this._hydrateTurnPdfs(turn, {
2865
- dir,
2866
- pinned,
2867
- signal,
2868
- maxPages
2869
- });
2870
- if (turn === latestUser) {
2871
- for (const id of shown) {
2872
- rendered.add(id);
2929
+ const room = pdfImageRoom(measured, turn === latestUser ? textChars : 0);
2930
+ for (const pdf of turn.pdfs ?? []) {
2931
+ const pages = await this._tryHydratePdf({ dir, pdf, pinned, room, signal, selected });
2932
+ attachPages(turn, pages.names.map((name) => path.join(dir, name)));
2933
+ if (pages.names.length && turn === latestUser) {
2934
+ rendered.add(pdf.id);
2935
+ }
2936
+ const whole = turn === latestUser && wholeIds.has(pdf.id);
2937
+ for (const notice of pdfNotices(pdf.fileName, pages, whole)) {
2938
+ notices.push({ turn, notice });
2873
2939
  }
2874
2940
  }
2875
- cuts.push(...notices.map((notice) => ({ turn, notice })));
2941
+ delete turn.pdfs;
2876
2942
  }
2877
- return { rendered, cuts };
2943
+ return { rendered, notices };
2878
2944
  }
2879
- async _pdfPageBudget({ ctxSize, modelName }) {
2945
+ async _pdfReading({ ctxSize, modelName }, options) {
2880
2946
  const custom = await this._customModel(modelName);
2881
2947
  const window = effectiveCtxSize({ ctxSize, modelName }, { appCtxSize: 0, platform: this._platform, custom });
2882
- return pdfPageBudget(window);
2948
+ const { cacheTokens } = await this._contextReading(options, window);
2949
+ return { ctxSize: window, cacheTokens };
2883
2950
  }
2884
2951
  async _customModel(name) {
2885
2952
  for await (const list of this._engine.customModelsWatch({})) {
@@ -2887,117 +2954,108 @@ export class QvacAssistant extends ReadyResource {
2887
2954
  }
2888
2955
  return null;
2889
2956
  }
2890
- async _hydrateTurnPdfs(turn, { dir, pinned, signal, maxPages }) {
2891
- const shown = [];
2892
- const notices = [];
2893
- let budget = maxPages;
2894
- for (const pdf of turn.pdfs ?? []) {
2895
- if (budget <= 0) {
2896
- notices.push(pdfPagesSkippedNotice(pdf.fileName));
2897
- continue;
2898
- }
2899
- const pages = await this._tryHydratePdf(dir, turn, { pdf, pinned, maxPages: budget, signal });
2900
- budget -= pages.tried;
2901
- if (pages.names.length) {
2902
- shown.push(pdf.id);
2903
- }
2904
- if (pages.cut) {
2905
- notices.push(pdfPagesCutNotice({ fileName: pdf.fileName, ...pages.cut }));
2906
- }
2907
- }
2908
- delete turn.pdfs;
2909
- return { shown, notices };
2910
- }
2911
- async _tryHydratePdf(dir, turn, opts) {
2957
+ async _tryHydratePdf(opts) {
2912
2958
  try {
2913
- return await this._hydratePdf(dir, turn, opts);
2959
+ return await this._hydratePdf(opts);
2914
2960
  }
2915
2961
  catch (err) {
2916
2962
  if (opts.signal.aborted) {
2917
2963
  throw err;
2918
2964
  }
2919
2965
  this._log?.warn('assistant', `pdf rasterization failed — ${opts.pdf.fileName}`, err);
2920
- return { names: [], tried: 0 };
2966
+ return NO_PDF_PAGES;
2921
2967
  }
2922
2968
  }
2923
- async _hydratePdf(dir, turn, { pdf, pinned, maxPages, signal }) {
2924
- const key = `${pdf.blobId}:${maxPages}`;
2925
- const memo = this._pdfPages.get(key);
2926
- if (memo) {
2927
- for (const name of memo.names) {
2928
- pinned.add(name);
2969
+ async _hydratePdf(hydration) {
2970
+ const { dir, pdf, pinned, room, signal, selected } = hydration;
2971
+ const key = pdf.pages?.blobId ?? pdf.id;
2972
+ const remember = (selection) => {
2973
+ this._pdfSelections.set(key, selection);
2974
+ if (!pdf.pages) {
2975
+ selected.push({ ref: pdf, selection });
2929
2976
  }
2930
- const files = memo.names.map((name) => path.join(dir, name));
2931
- if ((await Promise.all(files.map((file) => cacheOrTouch(file, noBytes)))).every(Boolean)) {
2932
- attachPages(turn, files);
2933
- return memo;
2977
+ };
2978
+ const known = this._pdfSelections.get(key) ?? (await this._storedPdfSelection(pdf, signal));
2979
+ if (known) {
2980
+ remember(known);
2981
+ const cached = await cachedPdfPages(known, { dir, pdf, pinned });
2982
+ if (cached) {
2983
+ return cached;
2934
2984
  }
2935
2985
  }
2936
- const pages = await this._renderPdf(dir, { pdf, pinned, maxPages, signal });
2937
- if (!pages) {
2938
- return { names: [], tried: 0 };
2939
- }
2940
- this._pdfPages.set(key, pages);
2941
- attachPages(turn, pages.names.map((name) => path.join(dir, name)));
2942
- return pages;
2943
- }
2944
- async _renderPdf(dir, { pdf, pinned, maxPages, signal }) {
2945
2986
  const bytes = await this._readBlob(pdf.url, signal);
2946
2987
  if (signal.aborted) {
2947
2988
  throw new Error('hydration aborted');
2948
2989
  }
2949
- if (!bytes) {
2950
- return null;
2951
- }
2952
- const doc = await this._pdfRasterizer.open(bytes);
2990
+ const doc = bytes ? await this._pdfRasterizer.open(bytes) : null;
2953
2991
  if (!doc) {
2954
- return { names: [], tried: 0 };
2992
+ return NO_PDF_PAGES;
2955
2993
  }
2956
2994
  try {
2957
- const names = [];
2958
- let tried = 0;
2959
- let page = 0;
2960
- const pageCount = doc.pageCount();
2961
- for (; page < pageCount && tried < maxPages; page++) {
2962
- if (signal.aborted)
2963
- throw new Error('hydration aborted');
2964
- const outcome = await this._renderPdfPage(doc, { dir, pdf, page, pinned, signal });
2965
- if (outcome.tried) {
2966
- tried++;
2967
- }
2968
- if (outcome.name) {
2969
- names.push(outcome.name);
2970
- }
2995
+ const selection = known ?? (await selectPdfPages(doc, room));
2996
+ if (!known) {
2997
+ remember(selection);
2971
2998
  }
2972
- return { names, tried, cut: pdfCut(doc, page) };
2999
+ return await this._renderPdfShots(doc, selection, hydration);
2973
3000
  }
2974
3001
  finally {
2975
3002
  doc.close();
2976
3003
  }
2977
3004
  }
2978
- async _renderPdfPage(doc, { dir, pdf, page, pinned, signal }) {
2979
- let tried = false;
3005
+ async _storedPdfSelection(pdf, signal) {
3006
+ const bytes = pdf.pages ? await this._readBlob(pdf.pages.url, signal) : null;
3007
+ if (!bytes) {
3008
+ return null;
3009
+ }
2980
3010
  try {
2981
- const flags = doc.pageFlags(page);
2982
- if (flags.hasText && !flags.hasImage) {
2983
- return { tried };
2984
- }
2985
- tried = true;
2986
- const name = pdfPageCacheName(pdf, page);
2987
- pinned.add(name);
2988
- const cached = await cacheOrTouch(path.join(dir, name), () => doc.renderPage(page));
2989
- return { tried, name: cached ? name : undefined };
3011
+ return parsePdfSelection(bytes);
2990
3012
  }
2991
3013
  catch (err) {
2992
- if (signal.aborted) {
2993
- throw err;
3014
+ this._log?.warn('assistant', `pdf page choice unreadable — ${pdf.fileName}`, err);
3015
+ return null;
3016
+ }
3017
+ }
3018
+ async _renderPdfShots(doc, selection, { dir, pdf, pinned, signal }) {
3019
+ const names = [];
3020
+ const labels = [];
3021
+ for (const shot of selection.shots) {
3022
+ if (signal.aborted)
3023
+ throw new Error('hydration aborted');
3024
+ const name = pdfShotCacheName(pdf, shot);
3025
+ pinned.add(name);
3026
+ try {
3027
+ const file = path.join(dir, name);
3028
+ if (await cacheOrTouch(file, () => doc.renderPage(shot.page, shot.clip))) {
3029
+ names.push(name);
3030
+ labels.push(shot.label);
3031
+ }
3032
+ }
3033
+ catch (err) {
3034
+ if (signal.aborted) {
3035
+ throw err;
3036
+ }
3037
+ this._log?.warn('assistant', `pdf page skipped — ${pdf.fileName} p${shot.page}`, err);
3038
+ }
3039
+ }
3040
+ return { selection, names, labels };
3041
+ }
3042
+ async _storePdfSelections(selected, target) {
3043
+ for (const { ref, selection } of selected) {
3044
+ try {
3045
+ await this._uploadGenerated({
3046
+ fileName: pdfSelectionFileName(ref.fileName),
3047
+ mimeType: PDF_PAGES_MIME,
3048
+ bytes: packPdfSelection(selection),
3049
+ raw: true
3050
+ }, { ...target, sourceId: ref.id });
3051
+ }
3052
+ catch (err) {
3053
+ this._log?.warn('assistant', `pdf page choice not stored — ${ref.fileName}`, err);
2994
3054
  }
2995
- this._log?.warn('assistant', `pdf page skipped — ${pdf.fileName} p${page}`, err);
2996
- return { tried };
2997
3055
  }
2998
3056
  }
2999
3057
  async _compactIfNearFull(params) {
3000
- const { chatId, transcript, trigger, signal, meshId } = params;
3058
+ const { chatId, transcript, trigger, signal, meshId, requestSeq } = params;
3001
3059
  const metrics = await this._lastRunMetrics(chatId, meshId);
3002
3060
  if (!nearContextLimit(metrics.cacheTokens, metrics.ctxSize)) {
3003
3061
  return false;
@@ -3015,11 +3073,12 @@ export class QvacAssistant extends ReadyResource {
3015
3073
  trigger,
3016
3074
  signal,
3017
3075
  occupancy: metrics,
3018
- meshId
3076
+ meshId,
3077
+ requestSeq
3019
3078
  });
3020
3079
  }
3021
3080
  async _runCompaction(params) {
3022
- const { chatId, transcript, trigger, signal, meshId } = params;
3081
+ const { chatId, transcript, trigger, signal, meshId, requestSeq } = params;
3023
3082
  let statusId = null;
3024
3083
  const started = Date.now();
3025
3084
  try {
@@ -3032,7 +3091,8 @@ export class QvacAssistant extends ReadyResource {
3032
3091
  statusId = status.id;
3033
3092
  },
3034
3093
  meshId,
3035
- signal
3094
+ signal,
3095
+ requestSeq
3036
3096
  });
3037
3097
  this._log?.info('runs', 'compaction', {
3038
3098
  chatId,
@@ -3117,12 +3177,13 @@ export class QvacAssistant extends ReadyResource {
3117
3177
  .catch(() => ({ chunks: [] }));
3118
3178
  const lastThroughId = lastCompactedThroughId(chunks);
3119
3179
  const lastThroughSeq = chunks.find((chunk) => chunk.id === lastThroughId)?.seq ?? -1;
3120
- const userTurns = chunks.filter((chunk) => chunk.userMessage).length;
3180
+ const beforeSeq = options.requestSeq ?? Infinity;
3181
+ const userTurns = chunks.filter((chunk) => chunk.userMessage && chunk.seq < beforeSeq).length;
3121
3182
  let announced = false;
3122
3183
  let keepTurns = COMPACTION_KEEP_TURNS;
3123
3184
  const senders = await this._senderNames(engine, chunks);
3124
3185
  for (;;) {
3125
- const boundary = compactionBoundary(chunks, keepTurns);
3186
+ const boundary = compactionBoundary(chunks, keepTurns, beforeSeq);
3126
3187
  if (!boundary) {
3127
3188
  return { compacted: false, reason: 'nothing-to-fold' };
3128
3189
  }
@@ -3133,7 +3194,8 @@ export class QvacAssistant extends ReadyResource {
3133
3194
  if (through.seq <= lastThroughSeq) {
3134
3195
  return { compacted: false, reason: 'already-compacted' };
3135
3196
  }
3136
- const inRange = chunks.filter((chunk) => chunk.seq <= through.seq);
3197
+ const folds = foldsThrough(chunks, through);
3198
+ const inRange = chunks.filter((chunk) => chunk.compaction || folds(chunk));
3137
3199
  const folded = historyFromChunks(inRange, staleToolRunIds(inRange, priorAsks ?? [], ''), {
3138
3200
  senderNames: senders
3139
3201
  });
@@ -3219,9 +3281,10 @@ export class QvacAssistant extends ReadyResource {
3219
3281
  }
3220
3282
  return { summary: parseCompactionSummary(text), overflowed: false };
3221
3283
  }
3222
- async _autoTitle(engine, { chatId, messages, model, agent, probe }) {
3284
+ async _autoTitle(engine, { chatId, history, messages, model, agent, probe }) {
3223
3285
  try {
3224
- const first = messages.find((turn) => turn.role === 'user' && turn.content.trim());
3286
+ const first = history.find((chunk) => chunk.userMessage?.text.trim())?.userMessage?.text ??
3287
+ messages.find((turn) => turn.role === 'user' && turn.content.trim())?.content;
3225
3288
  if (!first) {
3226
3289
  return;
3227
3290
  }
@@ -3229,7 +3292,7 @@ export class QvacAssistant extends ReadyResource {
3229
3292
  if (!isDefaultChatTitle(chat?.title)) {
3230
3293
  return;
3231
3294
  }
3232
- await this._generateChatTitle({ engine, chatId, input: first.content, model, agent, probe });
3295
+ await this._generateChatTitle({ engine, chatId, input: first, model, agent, probe });
3233
3296
  }
3234
3297
  catch (err) {
3235
3298
  this._log?.warn('runs', 'auto-title skipped', { chatId, err });
@@ -3663,7 +3726,7 @@ export class QvacAssistant extends ReadyResource {
3663
3726
  };
3664
3727
  }
3665
3728
  }
3666
- async function injectRetrievedKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords, injected }) {
3729
+ async function injectRetrievedKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords, pictured, injected }) {
3667
3730
  const records = sourcesFromHits(knowledge.hits, RAG_SOURCE_BOUNDS);
3668
3731
  const notice = chooseRagNotice({
3669
3732
  errorMessage: knowledge.search.errorMessage,
@@ -3686,7 +3749,9 @@ async function injectRetrievedKnowledge(knowledge, { transcript, sourceTagFilter
3686
3749
  });
3687
3750
  injected.sourcesWritten = true;
3688
3751
  if (records.length > 0) {
3689
- knowledge.lastUser.content = `${formatRagContext(knowledge.hits, knowledge.search.availableFiles ?? [])}\n\nThe user's message:\n${knowledge.lastUser.content}\n\n${RAG_CITATION_LINE}`;
3752
+ const framing = formatRagContext(knowledge.hits, knowledge.search.availableFiles ?? []);
3753
+ const pages = pictured ? `\n${PDF_PAGES_SOURCE_LINE}` : '';
3754
+ knowledge.lastUser.content = `${framing}${pages}\n\nThe user's message:\n${knowledge.lastUser.content}\n\n${RAG_CITATION_LINE}`;
3690
3755
  injected.ragInjected = true;
3691
3756
  injected.documentsInjected = true;
3692
3757
  }
@@ -3707,7 +3772,7 @@ function toolCallChunk(event) {
3707
3772
  function toolResultChunk(event) {
3708
3773
  return {
3709
3774
  type: 'tool-result',
3710
- toolResult: { name: event.name, result: withoutDataChannel(event.result) }
3775
+ toolResult: { name: event.name, result: withoutDataChannel(event.result) ?? null }
3711
3776
  };
3712
3777
  }
3713
3778
  function runMetricsSummary(metrics) {
@@ -14,7 +14,8 @@ export interface CompactionBoundary {
14
14
  throughId: string;
15
15
  foldedTurns: number;
16
16
  }
17
- export declare function compactionBoundary(chunks: ReadonlyArray<AssistantChunk>, keepTurns?: number): CompactionBoundary | null;
17
+ export declare function compactionBoundary(chunks: ReadonlyArray<AssistantChunk>, keepTurns?: number, beforeSeq?: number): CompactionBoundary | null;
18
+ export declare function foldsThrough(chunks: ReadonlyArray<AssistantChunk>, through: AssistantChunk): (chunk: AssistantChunk) => boolean;
18
19
  export declare function lastCompactedThroughId(chunks: ReadonlyArray<AssistantChunk>): string | null;
19
20
  export interface FoldedChunks {
20
21
  chunks: ReadonlyArray<AssistantChunk>;