@qvac/core 0.10.3 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CHANGELOG.md +71 -1
  2. package/dist-lib/lib/assistant/history.d.ts +2 -1
  3. package/dist-lib/lib/assistant/history.js +32 -5
  4. package/dist-lib/lib/assistant/message.d.ts +20 -0
  5. package/dist-lib/lib/assistant/message.js +36 -0
  6. package/dist-lib/lib/assistant/pdf-budget.d.ts +3 -0
  7. package/dist-lib/lib/assistant/pdf-budget.js +13 -0
  8. package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
  9. package/dist-lib/lib/assistant/video-budget.js +29 -0
  10. package/dist-lib/lib/assistant-tree.d.ts +1 -0
  11. package/dist-lib/lib/assistant-tree.js +9 -2
  12. package/dist-lib/lib/assistant.d.ts +26 -3
  13. package/dist-lib/lib/assistant.js +471 -47
  14. package/dist-lib/lib/core/chat-fork.js +1 -0
  15. package/dist-lib/lib/core/chat-search.js +1 -1
  16. package/dist-lib/lib/core/chunk.d.ts +2 -2
  17. package/dist-lib/lib/core/chunk.js +5 -1
  18. package/dist-lib/lib/core/departures.js +6 -0
  19. package/dist-lib/lib/core/document-attachment.d.ts +7 -0
  20. package/dist-lib/lib/core/document-attachment.js +9 -0
  21. package/dist-lib/lib/core/files/index.js +4 -4
  22. package/dist-lib/lib/core/files/records/chunks.js +21 -0
  23. package/dist-lib/lib/core/files/records/purge.js +4 -3
  24. package/dist-lib/lib/core/files/records/source.js +3 -5
  25. package/dist-lib/lib/core/files/upload.js +3 -2
  26. package/dist-lib/lib/core/identity.d.ts +2 -0
  27. package/dist-lib/lib/core/identity.js +19 -0
  28. package/dist-lib/lib/core/pdf/rasterize.d.ts +0 -1
  29. package/dist-lib/lib/core/pdf/rasterize.js +0 -1
  30. package/dist-lib/lib/core/video/attachment.d.ts +1 -0
  31. package/dist-lib/lib/core/video/attachment.js +7 -0
  32. package/dist-lib/lib/core/video/digest.d.ts +16 -0
  33. package/dist-lib/lib/core/video/digest.js +57 -0
  34. package/dist-lib/lib/core/video/frames.d.ts +32 -0
  35. package/dist-lib/lib/core/video/frames.js +224 -0
  36. package/dist-lib/lib/core.d.ts +1 -0
  37. package/dist-lib/lib/core.js +44 -7
  38. package/dist-lib/lib/credential-pusher.d.ts +6 -0
  39. package/dist-lib/lib/credential-pusher.js +76 -59
  40. package/dist-lib/lib/errors.d.ts +8 -0
  41. package/dist-lib/lib/errors.js +8 -0
  42. package/dist-lib/lib/oauth/flow-core.js +2 -0
  43. package/dist-lib/lib/oauth/loopback.d.ts +1 -0
  44. package/dist-lib/lib/oauth/loopback.js +31 -1
  45. package/dist-lib/lib/oauth/register.js +33 -18
  46. package/dist-lib/lib/oauth/request.d.ts +10 -0
  47. package/dist-lib/lib/oauth/request.js +22 -0
  48. package/dist-lib/lib/oauth/token-exchange-core.js +29 -32
  49. package/dist-lib/lib/oauth/types.d.ts +3 -0
  50. package/dist-lib/lib/oauth.d.ts +2 -1
  51. package/dist-lib/lib/oauth.js +1 -0
  52. package/dist-lib/lib/run-notices.d.ts +1 -0
  53. package/dist-lib/lib/run-notices.js +2 -1
  54. package/dist-lib/lib/run-replay.d.ts +8 -2
  55. package/dist-lib/lib/run-replay.js +29 -11
  56. package/dist-lib/lib/task.js +1 -1
  57. package/dist-lib/schema/engine.js +4 -2
  58. package/dist-lib/schema/indexes.js +2 -2
  59. package/dist-lib/schema/qvac.js +4 -2
  60. package/dist-lib/spec/hrpc/messages.js +16 -6
  61. package/dist-lib/spec/hyperschema/index.js +16 -6
  62. package/dist-lib/spec/hyperschema/schema.json +10 -0
  63. package/dist-lib/spec/hyperschema/types.d.ts +2 -0
  64. package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
  65. package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
  66. package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
  67. package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
  68. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  69. package/package.json +7 -4
  70. package/tui/app.mjs +34 -1
  71. package/tui/lib/render.mjs +5 -0
  72. package/tui/main.mjs +8 -0
@@ -22,10 +22,10 @@ import { buildTitlePrompt, deriveChatTitle, isDefaultChatTitle, parseTitleRespon
22
22
  import { matchApprovalRule } from './core/approval-rules.js';
23
23
  import { isAudioAttachment } from './core/audio-attachment.js';
24
24
  import { attachTurnDocumentIds, formatRagContext, chooseRagNotice, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, createRagSourceTagFilter, citedByPhraseOverlap, formatFullDocumentContext, isFullTextHit, isTruncatedFullTextHit, sourcesFromHits, WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from './core/files/index.js';
25
- import { annotateDocumentAttachments, documentUnreadNotice, documentUnreadReason } from './core/document-attachment.js';
25
+ import { annotateDocumentAttachments, documentUnreadNotice, documentUnreadReason, pdfPagesCutNotice, pdfPagesSkippedNotice } from './core/document-attachment.js';
26
26
  import { imageCacheKey, imageCacheName, sweepImageCache } from './core/image/cache-sweep.js';
27
27
  import { IMAGE_MIMES, normalizeImageForModel } from './core/image/normalize.js';
28
- import { MAX_PDF_PAGE_IMAGES, createPdfRasterizer } from './core/pdf/rasterize.js';
28
+ import { createPdfRasterizer } from './core/pdf/rasterize.js';
29
29
  import { recordBackends, summarizeRuns } from './diagnostics.js';
30
30
  import { resolveFsGrants } from './fs-drive-bridge.js';
31
31
  import { InferenceCallStore } from './inference-calls.js';
@@ -44,14 +44,19 @@ import { isLifecycleTeardown, knownAbortOutcome, RUN_ABORT_REASONS, RUN_FAILURE_
44
44
  import { unref } from './unref.js';
45
45
  import { encodeAgentSkillReadiness } from './skill-readiness.js';
46
46
  import { harnessClearKvCache, harnessDeathReason, harnessDeleteModel, harnessDied, harnessListModels, harnessPreviewModelSource, harnessSkillCatalog, harnessSkillManagement, harnessDetectLanguage, harnessSpeak, harnessSpeakStream, harnessTools, harnessTranscribe, heldLoadModel, droppedHeldLoadModel, imageDownloadProgress, isHarnessEvent, modelCatalogChanged, toolReportEntry } from './assistant/harness.js';
47
- import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders } from './assistant/message.js';
47
+ import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders, restoreSentContent, hiddenVideoNotice, lentVideoNotice, unreadVideoNotice, unsyncedVideoNotice, VIDEO_NO_AUDIO, VIDEO_SILENT, videoAnnotation, videoTranscriptBlock } from './assistant/message.js';
48
+ import { frameTimes, probeVideo, sampleFrames } from './core/video/frames.js';
49
+ import { packVideoDigest, parseVideoDigestHeader, unpackVideoDigest, VIDEO_DIGEST_MIME, videoDigestFileName } from './core/video/digest.js';
50
+ import { VIDEO_FRAME_EDGE, videoFrameCount } from './assistant/video-budget.js';
48
51
  import { drainBody, raceInactivity } from './assistant/fetch-body.js';
52
+ import { pdfPageBudget } from './assistant/pdf-budget.js';
53
+ import { effectiveCtxSize } from './api/effective-ctx-size.js';
49
54
  import { GENERIC_ACKNOWLEDGEMENTS, HISTORY_TYPES, SKILL_TOOL, TERMINAL_STATES, couldNameSenders, historyFromChunks, inputAvailableToRun, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
50
55
  import { errorMessage, isRecord } from './guards.js';
51
56
  import { CoreError, isCoreError } from './errors.js';
52
57
  export { harnessClearKvCache, harnessDeleteModel, harnessListModels, harnessPreviewModelSource, harnessSkillCatalog, harnessSkillManagement, harnessDetectLanguage, harnessSpeak, harnessSpeakStream, harnessTools, harnessTranscribe } from './assistant/harness.js';
53
58
  export {} from './assistant/skill-encode.js';
54
- export { annotateUnreadableImages, unheardAudioNotice, unreadableImagesNotice, DOCUMENT_ONLY_USER_PROMPT, IMAGE_AND_DOCUMENT_USER_PROMPT, IMAGE_ONLY_USER_PROMPT, nameSenders } from './assistant/message.js';
59
+ export { annotateUnreadableImages, unheardAudioNotice, unreadableImagesNotice, DOCUMENT_ONLY_USER_PROMPT, IMAGE_AND_DOCUMENT_USER_PROMPT, IMAGE_ONLY_USER_PROMPT, nameSenders, restoreSentContent } from './assistant/message.js';
55
60
  export { couldNameSenders, historyFromChunks, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
56
61
  const NO_SENDERS = new Map();
57
62
  const DEFAULT_PROFILE = 'mid';
@@ -59,6 +64,7 @@ const QvacState = local.getEnum('@local/qvac-state');
59
64
  const FINAL_FLUSH_MS = 10_000;
60
65
  const UPLOAD_FRAME_BYTES = 64 * 1024;
61
66
  const RUN_IDLE_TIMEOUT_MS = 15 * 60_000;
67
+ const MODEL_RELOAD_DELAY_MS = 2_500;
62
68
  function idleWatch(controller, ms) {
63
69
  let last = 0;
64
70
  let parked = 0;
@@ -106,6 +112,20 @@ const RunOccupancySchema = z.object({
106
112
  export const CAPTURE_INFERENCE_CALLS_KEY = 'inspector.captureInferenceCalls';
107
113
  export const CAPTURE_ON = 'on';
108
114
  export const CAPTURE_OFF = 'off';
115
+ function pdfCut(doc, from) {
116
+ const pageCount = doc.pageCount();
117
+ for (let page = from; page < pageCount; page++) {
118
+ try {
119
+ const flags = doc.pageFlags(page);
120
+ if (flags.hasImage || !flags.hasText) {
121
+ return { lastPage: page, pageCount };
122
+ }
123
+ }
124
+ catch {
125
+ }
126
+ }
127
+ return undefined;
128
+ }
109
129
  function attachPages(turn, files) {
110
130
  if (!files.length)
111
131
  return;
@@ -114,6 +134,24 @@ function attachPages(turn, files) {
114
134
  function noBytes() {
115
135
  return Promise.resolve(null);
116
136
  }
137
+ function videoFrameCacheName(video, count, frame) {
138
+ return imageCacheName({ blobId: video.blobId, fileName: `frames.${count}.f${frame}.jpg` });
139
+ }
140
+ function videoHeaderCacheName(digestBlobId) {
141
+ return imageCacheName({ blobId: digestBlobId, fileName: 'frames.json' });
142
+ }
143
+ function videoAudioBlock(fileName, { audio, transcript }) {
144
+ if (audio === 'none') {
145
+ return VIDEO_NO_AUDIO;
146
+ }
147
+ if (audio === 'silent') {
148
+ return VIDEO_SILENT;
149
+ }
150
+ if (audio === 'speech' && transcript) {
151
+ return videoTranscriptBlock(transcript);
152
+ }
153
+ return unheardAudioNotice(`The audio of the attached video "${fileName}"`, 'transcription failed');
154
+ }
117
155
  function pdfPageCacheName(pdf, page) {
118
156
  return imageCacheName({ blobId: pdf.blobId, fileName: `${pdf.fileName}.p${page}.jpg` });
119
157
  }
@@ -146,6 +184,11 @@ function isSetUpHere(status) {
146
184
  function sameNameList(left, right) {
147
185
  return left.length === right.length && left.every((name, index) => name === right[index]);
148
186
  }
187
+ function sameLoadSettings(a, b) {
188
+ return ((a.ctxSize ?? 0) === (b.ctxSize ?? 0) &&
189
+ (a.kvCacheK ?? '') === (b.kvCacheK ?? '') &&
190
+ (a.kvCacheV ?? '') === (b.kvCacheV ?? ''));
191
+ }
149
192
  function withConfigDefaults(agent, config) {
150
193
  if (agent.ctxSize || !config?.ctxSize)
151
194
  return agent;
@@ -178,6 +221,8 @@ export class QvacAssistant extends ReadyResource {
178
221
  _drives;
179
222
  _blobReadTimeoutMs;
180
223
  _runIdleTimeoutMs;
224
+ _modelReloadDelayMs;
225
+ _reloads = new Map();
181
226
  _pdfRasterizer;
182
227
  _warmAbort = new AbortController();
183
228
  _unloading = new Map();
@@ -218,6 +263,7 @@ export class QvacAssistant extends ReadyResource {
218
263
  _runReplay;
219
264
  _lifecycle;
220
265
  _audioTranscripts;
266
+ _videoProbes;
221
267
  _pdfPages;
222
268
  _chats;
223
269
  _runImageCacheNames;
@@ -226,7 +272,7 @@ export class QvacAssistant extends ReadyResource {
226
272
  _modelCatalog = new ModelCatalogRevisionHub();
227
273
  _onHarnessStatus;
228
274
  _onModelCatalogEvent;
229
- constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
275
+ constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, modelReloadDelayMs = MODEL_RELOAD_DELAY_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
230
276
  super();
231
277
  this._engine = engine;
232
278
  this._harness = harness;
@@ -242,6 +288,7 @@ export class QvacAssistant extends ReadyResource {
242
288
  this._preloadCurrentAgentModel = preloadCurrentAgentModel;
243
289
  this._blobReadTimeoutMs = blobReadTimeoutMs;
244
290
  this._runIdleTimeoutMs = runIdleTimeoutMs;
291
+ this._modelReloadDelayMs = modelReloadDelayMs;
245
292
  this._pdfRasterizer = pdfRasterizerOption ?? createPdfRasterizer(logger);
246
293
  this._deviceId = null;
247
294
  this._info = { toolNames: [], skills: [], supportsImages: false };
@@ -268,6 +315,7 @@ export class QvacAssistant extends ReadyResource {
268
315
  this._runReplay = new RunReplayStore(runReplayDir ?? persistentDir('run-replay'));
269
316
  this._lifecycle = new AbortController();
270
317
  this._audioTranscripts = new Map();
318
+ this._videoProbes = new Map();
271
319
  this._pdfPages = new Map();
272
320
  this._chats = new Map();
273
321
  this._runImageCacheNames = new Map();
@@ -731,6 +779,7 @@ export class QvacAssistant extends ReadyResource {
731
779
  this._harness.off?.('status', this._onHarnessStatus);
732
780
  this._harness.off?.('model', this._onModelCatalogEvent);
733
781
  this._warmAbort.abort(RUNNER_CLOSED);
782
+ this._cancelModelReloads();
734
783
  this._onlineBoot.destroy(new CoreError('ASSISTANT_CLOSING'));
735
784
  for (const run of this._runs.values()) {
736
785
  run.controller.abort(RUNNER_CLOSED);
@@ -769,6 +818,7 @@ export class QvacAssistant extends ReadyResource {
769
818
  this._log?.info('assistant', 'suspending', { runs: this._runs.size });
770
819
  this._parked.clear();
771
820
  this._stopMeshIntake();
821
+ this._cancelModelReloads();
772
822
  await this._downloads?.suspend();
773
823
  this._meshList?.destroy();
774
824
  this._accountFrames?.destroy();
@@ -851,6 +901,44 @@ export class QvacAssistant extends ReadyResource {
851
901
  const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
852
902
  return rows?.some((row) => row.name === model && row.isCached) === true;
853
903
  }
904
+ _scheduleModelReload(agent) {
905
+ const deviceId = this._deviceId;
906
+ if (!this._harness.loadModel ||
907
+ !deviceId ||
908
+ !agent.modelName ||
909
+ !agent.providerDeviceId ||
910
+ !b4a.equals(agent.providerDeviceId, deviceId)) {
911
+ return;
912
+ }
913
+ const name = agent.modelName;
914
+ clearTimeout(this._reloads.get(name));
915
+ const timer = setTimeout(() => {
916
+ this._reloads.delete(name);
917
+ this._reloadLoadedModel(agent).catch((err) => this._log?.warn('models', 'model reload failed', { model: name, err }));
918
+ }, this._modelReloadDelayMs);
919
+ unref(timer);
920
+ this._reloads.set(name, timer);
921
+ }
922
+ async _reloadLoadedModel(agent) {
923
+ if (this.closing || this.suspended || !this._harness.loadModel)
924
+ return;
925
+ const model = runnableModelName(agent.modelName, await this._runnableHost());
926
+ const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
927
+ const busy = [...this._runs.values()].some((run) => run.executing && !run.borrowed && run.model === model);
928
+ if (busy || !rows?.some((row) => row.name === model && row.isLoaded)) {
929
+ return;
930
+ }
931
+ const config = await this._engine.getConfig({}).catch(noop);
932
+ const knobs = withConfigDefaults(agent, config);
933
+ const { loaded, reason } = await this._harness.loadModel({ model, agent: knobs });
934
+ this._log?.info('models', 'model reload requested', { model, loaded, reason });
935
+ }
936
+ _cancelModelReloads() {
937
+ for (const timer of this._reloads.values()) {
938
+ clearTimeout(timer);
939
+ }
940
+ this._reloads.clear();
941
+ }
854
942
  async _agentRunning(model, activeAgentId) {
855
943
  const deviceId = this._requireDeviceId();
856
944
  const meshId = this._accountMeshId;
@@ -1073,21 +1161,26 @@ export class QvacAssistant extends ReadyResource {
1073
1161
  .forget(chatId)
1074
1162
  .catch((err) => this._log?.warn('assistant', 'inspector records not dropped', { err }));
1075
1163
  }
1076
- async _reclaimRekeyedAgents(meshId, frames) {
1164
+ async _followAgentEdits(meshId, frames) {
1077
1165
  let known = null;
1078
1166
  for await (const frame of frames) {
1079
1167
  if (this.closing || this.suspended)
1080
1168
  return;
1081
1169
  const agents = frame.agents ?? [];
1082
1170
  this._followHosted(meshId, agents);
1083
- const next = new Map(agents.map((agent) => [agent.id, agentRekeyFingerprint(agent)]));
1084
1171
  for (const agent of agents) {
1085
1172
  const previous = known?.get(agent.id);
1086
- if (previous !== undefined && previous !== next.get(agent.id)) {
1173
+ if (!previous) {
1174
+ continue;
1175
+ }
1176
+ if (agentRekeyFingerprint(previous) !== agentRekeyFingerprint(agent)) {
1087
1177
  void this._reclaimAgentChatCaches(meshId, agent);
1088
1178
  }
1179
+ if (!sameLoadSettings(previous, agent)) {
1180
+ this._scheduleModelReload(agent);
1181
+ }
1089
1182
  }
1090
- known = next;
1183
+ known = new Map(agents.map((agent) => [agent.id, agent]));
1091
1184
  }
1092
1185
  }
1093
1186
  async _reclaimAgentChatCaches(meshId, agent) {
@@ -1195,18 +1288,21 @@ export class QvacAssistant extends ReadyResource {
1195
1288
  }
1196
1289
  async _sentRuns(chatId, holder) {
1197
1290
  try {
1198
- return await this._runReplay.load({ holder, chatId });
1291
+ return {
1292
+ sentRuns: await this._runReplay.load({ holder, chatId }),
1293
+ askedRuns: await this._runReplay.loadAsked({ holder, chatId })
1294
+ };
1199
1295
  }
1200
1296
  catch (err) {
1201
1297
  this._log?.warn('assistant', 'run replay unreadable', { chatId, err });
1202
1298
  await this._deleteChatKvCache(chatId, 'run replay unreadable', holder);
1203
- return new Map();
1299
+ return { sentRuns: new Map(), askedRuns: new Map() };
1204
1300
  }
1205
1301
  }
1206
- async _recordSentRun(chatId, { runId, turns, holder }) {
1302
+ async _recordSentRun(chatId, { runId, turns, asked, holder }) {
1207
1303
  if (turns) {
1208
1304
  try {
1209
- await this._runReplay.save({ holder, chatId }, runId, turns);
1305
+ await this._runReplay.save({ holder, chatId }, runId, { messages: turns, asked });
1210
1306
  return;
1211
1307
  }
1212
1308
  catch (err) {
@@ -1300,7 +1396,7 @@ export class QvacAssistant extends ReadyResource {
1300
1396
  const watches = [
1301
1397
  this._streamWatch(`run-requests:${meshId}`, () => this._engine.runRequestsWatch({ meshId }), (frames) => this._dispatchRunRequests(meshId, frames)),
1302
1398
  this._streamWatch(`chat-list:${meshId}`, () => this._engine.chatsWatch({ meshId }), (frames) => this._reclaimDeletedChats(frames)),
1303
- this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._reclaimRekeyedAgents(meshId, frames))
1399
+ this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._followAgentEdits(meshId, frames))
1304
1400
  ];
1305
1401
  return {
1306
1402
  start: () => watches.forEach((watch) => watch.start()),
@@ -1612,7 +1708,7 @@ export class QvacAssistant extends ReadyResource {
1612
1708
  model,
1613
1709
  log: this._log
1614
1710
  });
1615
- await this._compactIfNearFull({
1711
+ const folded = await this._compactIfNearFull({
1616
1712
  chatId,
1617
1713
  transcript,
1618
1714
  trigger: 'before-run',
@@ -1653,8 +1749,12 @@ export class QvacAssistant extends ReadyResource {
1653
1749
  await this._reconcileAttribution(chatId, senders, holder);
1654
1750
  }
1655
1751
  const stale = staleToolRunIds(history, priorAsks ?? [], runId);
1656
- const sentRuns = await this._sentRuns(chatId, holder);
1657
- const messages = historyFromChunks(history, stale, { senderNames: senders, sentRuns });
1752
+ const { sentRuns, askedRuns } = await this._sentRuns(chatId, holder);
1753
+ const messages = historyFromChunks(history, stale, {
1754
+ senderNames: senders,
1755
+ sentRuns,
1756
+ askedRuns
1757
+ });
1658
1758
  const typedText = currentRepair
1659
1759
  ? undefined
1660
1760
  : history.findLast((chunk) => chunk.userMessage)?.userMessage?.text;
@@ -1672,12 +1772,11 @@ export class QvacAssistant extends ReadyResource {
1672
1772
  }
1673
1773
  : configuredAgent;
1674
1774
  const sendsFiles = lender === null;
1675
- const seesImages = sendsFiles &&
1676
- (this._harness.supportsImagesFor
1677
- ? this._harness.supportsImagesFor(model)
1678
- : (this._harness.supportsImages ?? this._info.supportsImages) !== false);
1775
+ const carriesMedia = messages.some((turn) => turn.images?.length || turn.pdfs?.length || turn.videos?.length);
1776
+ const seesImages = sendsFiles && carriesMedia && (await this._seesImages(model));
1679
1777
  let unreadableImages = new Map();
1680
- let renderedPdfIds = new Set();
1778
+ const sampled = [];
1779
+ let pdfPages = { rendered: new Set(), cuts: [] };
1681
1780
  if (!seesImages) {
1682
1781
  const dropped = messages.reduce((count, turn) => count + (turn.images?.length ?? 0), 0);
1683
1782
  if (dropped && (this._harness.supportsImages ?? this._info.supportsImages)) {
@@ -1781,6 +1880,7 @@ export class QvacAssistant extends ReadyResource {
1781
1880
  let runVerdict = 'completed';
1782
1881
  let sentRun = null;
1783
1882
  let inputLength = 0;
1883
+ let asked;
1784
1884
  const writeText = async (type, raw) => {
1785
1885
  const text = type === 'content' ? sourceTagFilter.push(raw) : raw;
1786
1886
  if (!text) {
@@ -1868,20 +1968,37 @@ export class QvacAssistant extends ReadyResource {
1868
1968
  const injected = knowledge
1869
1969
  ? await this._injectKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords })
1870
1970
  : { documentsInjected: false, sourcesWritten: false, ragInjected: false };
1871
- const hasAudio = messages.some((turn) => turn.audioAttachments?.length);
1971
+ const hasAudio = messages.some((turn) => turn.audioAttachments?.length || turn.videos?.length);
1872
1972
  if (hasAudio) {
1873
1973
  modelWatch.transcribe(true);
1874
1974
  }
1875
1975
  await this._hydrateAudio(messages, controller.signal);
1976
+ if (messages.some((turn) => turn.videos?.length)) {
1977
+ await this._hydrateVideos(messages, {
1978
+ signal: controller.signal,
1979
+ runId,
1980
+ seesImages,
1981
+ lent: !sendsFiles,
1982
+ reading: await this._contextReading({ chatId, meshId, folded }, configuredAgent.ctxSize),
1983
+ sampled
1984
+ });
1985
+ }
1876
1986
  if (hasAudio) {
1877
1987
  modelWatch.transcribe(false);
1878
1988
  }
1879
1989
  backfillImageOnlyTurns(messages);
1880
1990
  if (seesImages) {
1881
- renderedPdfIds = await this._hydratePdfPages(messages, controller.signal, runId);
1991
+ pdfPages = await this._hydratePdfPages(messages, {
1992
+ signal: controller.signal,
1993
+ runId,
1994
+ window: { ctxSize: configuredAgent.ctxSize, modelName: model }
1995
+ });
1882
1996
  }
1883
1997
  annotateDocumentAttachments(messages);
1884
- const unreadIds = attachTurnIds.filter((id) => !renderedPdfIds.has(id));
1998
+ for (const { turn, notice } of pdfPages.cuts) {
1999
+ turn.content = `${turn.content}\n${notice}`;
2000
+ }
2001
+ const unreadIds = attachTurnIds.filter((id) => !pdfPages.rendered.has(id));
1885
2002
  if (!diagramRepair && unreadIds.length > 0 && !injected.documentsInjected) {
1886
2003
  const reason = documentUnreadReason({
1887
2004
  knowledgeOn: toolsetEnabled('knowledge', runAgent.disabledToolsets),
@@ -1920,6 +2037,8 @@ export class QvacAssistant extends ReadyResource {
1920
2037
  cacheKey,
1921
2038
  turns: messages.length
1922
2039
  });
2040
+ restoreSentContent(messages);
2041
+ asked = messages.findLast((turn) => turn.role === 'user')?.content;
1923
2042
  const input = nameSenders(messages);
1924
2043
  inputLength = input.length;
1925
2044
  const inspecting = await this._capturingInferenceCalls();
@@ -2070,6 +2189,7 @@ export class QvacAssistant extends ReadyResource {
2070
2189
  await this._recordSentRun(chatId, {
2071
2190
  runId,
2072
2191
  turns: sentRun.turns(inputLength),
2192
+ asked,
2073
2193
  holder
2074
2194
  });
2075
2195
  sentRun = null;
@@ -2079,6 +2199,7 @@ export class QvacAssistant extends ReadyResource {
2079
2199
  metrics: citedTitles ? { ...metrics, citedTitles } : metrics
2080
2200
  });
2081
2201
  titleOutcome = 'completed';
2202
+ await this._storeVideoDigests(sampled, { meshId, chatId, runId, deviceId });
2082
2203
  }
2083
2204
  catch (err) {
2084
2205
  const outcome = controller.signal.aborted
@@ -2336,9 +2457,18 @@ export class QvacAssistant extends ReadyResource {
2336
2457
  return null;
2337
2458
  }
2338
2459
  }
2339
- async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId }) {
2460
+ async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId, sourceId }) {
2340
2461
  const duplex = this._engine.uploadAttachment();
2341
- duplex.write({ chatId, runId, originDeviceId: deviceId, fileName, mimeType, raw, meshId });
2462
+ duplex.write({
2463
+ chatId,
2464
+ runId,
2465
+ originDeviceId: deviceId,
2466
+ fileName,
2467
+ mimeType,
2468
+ raw,
2469
+ meshId,
2470
+ sourceId
2471
+ });
2342
2472
  const buf = b4a.toBuffer(bytes);
2343
2473
  for (let i = 0; i < buf.length; i += UPLOAD_FRAME_BYTES) {
2344
2474
  duplex.write({ data: buf.subarray(i, i + UPLOAD_FRAME_BYTES) });
@@ -2404,6 +2534,259 @@ export class QvacAssistant extends ReadyResource {
2404
2534
  return unheardAudioNotice(label, 'transcription failed on this device');
2405
2535
  }
2406
2536
  }
2537
+ async _seesImages(model) {
2538
+ if (this._harness.supportsImagesFor) {
2539
+ return this._harness.supportsImagesFor(model);
2540
+ }
2541
+ const anyModel = (this._harness.supportsImages ?? this._info.supportsImages) !== false;
2542
+ if (!anyModel || !this._harness.list) {
2543
+ return anyModel;
2544
+ }
2545
+ const rows = await this._harness.list({ type: 'llm' }).catch(noop);
2546
+ if (!rows) {
2547
+ return anyModel;
2548
+ }
2549
+ const row = rows.find((candidate) => candidate.name === model);
2550
+ return row ? (row.supportsImages ?? anyModel) : false;
2551
+ }
2552
+ async _hydrateVideos(messages, run) {
2553
+ const { signal, runId, seesImages, lent, reading, sampled } = run;
2554
+ const turns = messages.filter((turn) => turn.videos?.length);
2555
+ if (!turns.length) {
2556
+ return;
2557
+ }
2558
+ const asked = messages.findLast((turn) => turn.role === 'user')?.videos ?? [];
2559
+ const dir = this._imageCacheDir;
2560
+ await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
2561
+ const pinned = this._runImageCacheNames.get(runId) ?? new Set();
2562
+ this._runImageCacheNames.set(runId, pinned);
2563
+ const unsampled = turns.flatMap((turn) => turn.videos ?? []).filter((ref) => !ref.digest);
2564
+ const budget = { ...reading, videos: unsampled.length };
2565
+ for (const turn of turns) {
2566
+ for (const ref of turn.videos ?? []) {
2567
+ if (signal.aborted)
2568
+ return;
2569
+ const hydration = { dir, ref, pinned, seesImages, lent, signal, budget, sampled };
2570
+ const blocks = await this._hydrateVideo(turn, hydration);
2571
+ turn.content = [turn.content.trim(), ...blocks].filter(Boolean).join('\n\n');
2572
+ }
2573
+ delete turn.videos;
2574
+ }
2575
+ const pictured = asked.some((ref) => ref.digest || this._videoProbes.get(ref.blobId)?.hasVideo);
2576
+ if (!seesImages && !lent && pictured) {
2577
+ throw new Error(RUN_FAILURE_REASONS.videoNeedsVision);
2578
+ }
2579
+ }
2580
+ async _hydrateVideo(turn, hydration) {
2581
+ const { ref, seesImages, lent, signal } = hydration;
2582
+ const source = { opened: null };
2583
+ const open = () => (source.opened ??= this._openVideo(hydration));
2584
+ try {
2585
+ const video = await this._readVideo(hydration, open);
2586
+ if (video === 'unsynced') {
2587
+ return [unsyncedVideoNotice(ref.fileName)];
2588
+ }
2589
+ if (video === 'audio') {
2590
+ return [await this._transcribeAttachment(ref, signal)];
2591
+ }
2592
+ if (!seesImages || !('files' in video)) {
2593
+ const notice = lent ? lentVideoNotice(ref.fileName) : hiddenVideoNotice(ref.fileName);
2594
+ return [notice, videoAudioBlock(ref.fileName, video)];
2595
+ }
2596
+ attachPages(turn, video.files);
2597
+ const annotation = videoAnnotation({
2598
+ fileName: ref.fileName,
2599
+ duration: video.duration,
2600
+ times: video.times
2601
+ });
2602
+ return [annotation, videoAudioBlock(ref.fileName, video)];
2603
+ }
2604
+ catch (err) {
2605
+ if (signal.aborted) {
2606
+ throw err;
2607
+ }
2608
+ this._log?.warn('assistant', `video hydration failed — ${ref.fileName}`, err);
2609
+ return [unreadVideoNotice(ref.fileName)];
2610
+ }
2611
+ finally {
2612
+ const video = await source.opened?.catch(noop);
2613
+ if (video) {
2614
+ await fs.promises.unlink(video.file).catch(noop);
2615
+ }
2616
+ }
2617
+ }
2618
+ async _readVideo(hydration, open) {
2619
+ const { ref, seesImages, signal } = hydration;
2620
+ const stored = ref.digest ? await this._storedVideo(ref.digest, hydration) : null;
2621
+ if (stored) {
2622
+ return stored;
2623
+ }
2624
+ const probe = await this._probeVideo(ref, open);
2625
+ if (!probe) {
2626
+ return 'unsynced';
2627
+ }
2628
+ if (!probe.hasVideo) {
2629
+ return 'audio';
2630
+ }
2631
+ if (!seesImages) {
2632
+ return probe.hasAudio ? this._hearVideo(ref, open, signal) : { audio: 'none' };
2633
+ }
2634
+ const video = await open();
2635
+ if (!video) {
2636
+ return 'unsynced';
2637
+ }
2638
+ return this._sampleVideo(video, probe, hydration);
2639
+ }
2640
+ async _probeVideo(ref, open) {
2641
+ const known = this._videoProbes.get(ref.blobId);
2642
+ if (known) {
2643
+ return known;
2644
+ }
2645
+ const video = await open();
2646
+ if (!video) {
2647
+ return null;
2648
+ }
2649
+ const probe = await probeVideo(video.file);
2650
+ this._videoProbes.set(ref.blobId, probe);
2651
+ return probe;
2652
+ }
2653
+ async _storedVideo(digest, { dir, ref, pinned, signal }) {
2654
+ const headerName = videoHeaderCacheName(digest.blobId);
2655
+ pinned.add(headerName);
2656
+ const header = await fs.promises
2657
+ .readFile(path.join(dir, headerName), 'utf8')
2658
+ .then(parseVideoDigestHeader, noop);
2659
+ if (header) {
2660
+ const files = header.times.map((_, i) => {
2661
+ const name = videoFrameCacheName(ref, header.times.length, i);
2662
+ pinned.add(name);
2663
+ return path.join(dir, name);
2664
+ });
2665
+ const cached = await Promise.all(files.map((file) => cacheOrTouch(file, noBytes)));
2666
+ if (cached.every(Boolean)) {
2667
+ return { ...header, files };
2668
+ }
2669
+ }
2670
+ const bytes = await this._readBlob(digest.url, signal);
2671
+ if (!bytes) {
2672
+ return null;
2673
+ }
2674
+ const target = { dir, pinned, digestBlobId: digest.blobId };
2675
+ try {
2676
+ return await this._cacheVideo(ref, unpackVideoDigest(bytes), target);
2677
+ }
2678
+ catch (err) {
2679
+ this._log?.warn('assistant', `video digest unreadable — ${ref.fileName}`, err);
2680
+ return null;
2681
+ }
2682
+ }
2683
+ async _cacheVideo(ref, digest, { dir, pinned, digestBlobId }) {
2684
+ const { frames, ...header } = digest;
2685
+ const files = frames.map((_, i) => {
2686
+ const name = videoFrameCacheName(ref, frames.length, i);
2687
+ pinned.add(name);
2688
+ return path.join(dir, name);
2689
+ });
2690
+ await Promise.all(frames.map((frame, i) => cacheOrTouch(files[i], () => Promise.resolve(frame))));
2691
+ if (digestBlobId) {
2692
+ const headerName = videoHeaderCacheName(digestBlobId);
2693
+ pinned.add(headerName);
2694
+ await cacheOrTouch(path.join(dir, headerName), () => Promise.resolve(b4a.from(JSON.stringify(header))));
2695
+ }
2696
+ return { ...header, files };
2697
+ }
2698
+ async _sampleVideo(video, probe, hydration) {
2699
+ const { ref, budget, signal } = hydration;
2700
+ const started = Date.now();
2701
+ const count = videoFrameCount(probe, budget);
2702
+ const times = frameTimes(probe.duration, count);
2703
+ const frames = await sampleFrames(video.file, { times, maxEdge: VIDEO_FRAME_EDGE, probe });
2704
+ const sampleMs = Date.now() - started;
2705
+ const heard = probe.hasAudio
2706
+ ? await this._hearVideo(ref, () => Promise.resolve(video), signal)
2707
+ : { audio: 'none' };
2708
+ const digest = {
2709
+ duration: probe.duration,
2710
+ times: frames.map((frame) => frame.time),
2711
+ ...heard,
2712
+ frames: frames.map((frame) => frame.jpeg)
2713
+ };
2714
+ if (!hydration.ref.digest) {
2715
+ hydration.sampled.push({ ref, digest });
2716
+ }
2717
+ this._log?.info('assistant', 'video frames', {
2718
+ fileName: ref.fileName,
2719
+ duration: probe.duration,
2720
+ frames: frames.length,
2721
+ budget: count,
2722
+ ctxSize: budget.ctxSize ?? null,
2723
+ cacheTokens: budget.cacheTokens ?? null,
2724
+ audio: heard.audio,
2725
+ sampleMs
2726
+ });
2727
+ return this._cacheVideo(ref, digest, hydration);
2728
+ }
2729
+ async _storeVideoDigests(sampled, target) {
2730
+ const dir = this._imageCacheDir;
2731
+ const pinned = this._runImageCacheNames.get(target.runId) ?? new Set();
2732
+ for (const { ref, digest } of sampled) {
2733
+ try {
2734
+ const blobId = await this._uploadVideoDigest(ref, digest, target);
2735
+ await this._cacheVideo(ref, digest, { dir, pinned, digestBlobId: blobId });
2736
+ }
2737
+ catch (err) {
2738
+ this._log?.warn('assistant', `video digest not stored — ${ref.fileName}`, err);
2739
+ }
2740
+ }
2741
+ }
2742
+ async _uploadVideoDigest(ref, digest, target) {
2743
+ const attachment = await this._uploadGenerated({
2744
+ fileName: videoDigestFileName(ref.fileName),
2745
+ mimeType: VIDEO_DIGEST_MIME,
2746
+ bytes: packVideoDigest(digest),
2747
+ raw: true
2748
+ }, { ...target, sourceId: ref.id });
2749
+ return attachment.blobId;
2750
+ }
2751
+ async _openVideo({ dir, ref, pinned, signal }) {
2752
+ const bytes = await this._readBlob(ref.url, signal);
2753
+ if (!bytes) {
2754
+ return null;
2755
+ }
2756
+ const name = `${imageCacheName(ref)}.source`;
2757
+ pinned.add(name);
2758
+ const file = path.join(dir, name);
2759
+ await fs.promises.writeFile(file, bytes, { mode: 0o600 });
2760
+ return { bytes, file };
2761
+ }
2762
+ async _hearVideo(ref, open, signal) {
2763
+ const cached = this._audioTranscripts.get(ref.blobId);
2764
+ if (cached !== undefined) {
2765
+ return cached ? { audio: 'speech', transcript: cached } : { audio: 'silent' };
2766
+ }
2767
+ if (!this._harness.transcribe) {
2768
+ return { audio: 'unheard' };
2769
+ }
2770
+ try {
2771
+ const video = await open();
2772
+ if (!video) {
2773
+ return { audio: 'unheard' };
2774
+ }
2775
+ const model = mediaModelNameFor('transcription', await this._persistedProfile());
2776
+ await this.ensureMediaModelUnpaused('transcription', model);
2777
+ await this._downloads?.waitUntilCached(model, signal);
2778
+ const transcript = (await this._harness.transcribe({ model: '', audio: b4a.toBuffer(video.bytes), signal })).trim();
2779
+ this._audioTranscripts.set(ref.blobId, transcript);
2780
+ return transcript ? { audio: 'speech', transcript } : { audio: 'silent' };
2781
+ }
2782
+ catch (err) {
2783
+ if (signal.aborted) {
2784
+ throw err;
2785
+ }
2786
+ this._log?.warn('assistant', `video transcription failed — ${ref.fileName}`, err);
2787
+ return { audio: 'unheard' };
2788
+ }
2789
+ }
2407
2790
  async _hydrateImages(messages, signal, runId) {
2408
2791
  const unreadable = new Map();
2409
2792
  const turns = messages.filter((turn) => turn.images?.length);
@@ -2454,36 +2837,66 @@ export class QvacAssistant extends ReadyResource {
2454
2837
  });
2455
2838
  return file;
2456
2839
  }
2457
- async _hydratePdfPages(messages, signal, runId) {
2840
+ async _hydratePdfPages(messages, { signal, runId, window }) {
2458
2841
  const rendered = new Set();
2842
+ const cuts = [];
2459
2843
  const turns = messages.filter((turn) => turn.pdfs?.length);
2460
2844
  if (!turns.length) {
2461
- return rendered;
2845
+ return { rendered, cuts };
2462
2846
  }
2463
2847
  const dir = this._imageCacheDir;
2464
2848
  await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
2465
2849
  const pinned = this._runImageCacheNames.get(runId) ?? new Set();
2466
2850
  this._runImageCacheNames.set(runId, pinned);
2467
2851
  const latestUser = messages.findLast((turn) => turn.role === 'user');
2852
+ const maxPages = await this._pdfPageBudget(window);
2468
2853
  for (const turn of turns) {
2469
- let budget = MAX_PDF_PAGE_IMAGES;
2470
- for (const pdf of turn.pdfs ?? []) {
2471
- if (budget <= 0)
2472
- break;
2473
- const pages = await this._tryHydratePdf(dir, turn, {
2474
- pdf,
2475
- pinned,
2476
- maxPages: budget,
2477
- signal
2478
- });
2479
- budget -= pages.tried;
2480
- if (turn === latestUser && pages.names.length) {
2481
- rendered.add(pdf.id);
2854
+ const { shown, notices } = await this._hydrateTurnPdfs(turn, {
2855
+ dir,
2856
+ pinned,
2857
+ signal,
2858
+ maxPages
2859
+ });
2860
+ if (turn === latestUser) {
2861
+ for (const id of shown) {
2862
+ rendered.add(id);
2482
2863
  }
2483
2864
  }
2484
- delete turn.pdfs;
2865
+ cuts.push(...notices.map((notice) => ({ turn, notice })));
2866
+ }
2867
+ return { rendered, cuts };
2868
+ }
2869
+ async _pdfPageBudget({ ctxSize, modelName }) {
2870
+ const custom = await this._customModel(modelName);
2871
+ const window = effectiveCtxSize({ ctxSize, modelName }, { appCtxSize: 0, platform: this._platform, custom });
2872
+ return pdfPageBudget(window);
2873
+ }
2874
+ async _customModel(name) {
2875
+ for await (const list of this._engine.customModelsWatch({})) {
2876
+ return list.models?.find((row) => row.name === name) ?? null;
2877
+ }
2878
+ return null;
2879
+ }
2880
+ async _hydrateTurnPdfs(turn, { dir, pinned, signal, maxPages }) {
2881
+ const shown = [];
2882
+ const notices = [];
2883
+ let budget = maxPages;
2884
+ for (const pdf of turn.pdfs ?? []) {
2885
+ if (budget <= 0) {
2886
+ notices.push(pdfPagesSkippedNotice(pdf.fileName));
2887
+ continue;
2888
+ }
2889
+ const pages = await this._tryHydratePdf(dir, turn, { pdf, pinned, maxPages: budget, signal });
2890
+ budget -= pages.tried;
2891
+ if (pages.names.length) {
2892
+ shown.push(pdf.id);
2893
+ }
2894
+ if (pages.cut) {
2895
+ notices.push(pdfPagesCutNotice({ fileName: pdf.fileName, ...pages.cut }));
2896
+ }
2485
2897
  }
2486
- return rendered;
2898
+ delete turn.pdfs;
2899
+ return { shown, notices };
2487
2900
  }
2488
2901
  async _tryHydratePdf(dir, turn, opts) {
2489
2902
  try {
@@ -2533,8 +2946,9 @@ export class QvacAssistant extends ReadyResource {
2533
2946
  try {
2534
2947
  const names = [];
2535
2948
  let tried = 0;
2536
- const count = doc.pageCount();
2537
- for (let page = 0; page < count && tried < maxPages; page++) {
2949
+ let page = 0;
2950
+ const pageCount = doc.pageCount();
2951
+ for (; page < pageCount && tried < maxPages; page++) {
2538
2952
  if (signal.aborted)
2539
2953
  throw new Error('hydration aborted');
2540
2954
  const outcome = await this._renderPdfPage(doc, { dir, pdf, page, pinned, signal });
@@ -2545,7 +2959,7 @@ export class QvacAssistant extends ReadyResource {
2545
2959
  names.push(outcome.name);
2546
2960
  }
2547
2961
  }
2548
- return { names, tried };
2962
+ return { names, tried, cut: pdfCut(doc, page) };
2549
2963
  }
2550
2964
  finally {
2551
2965
  doc.close();
@@ -2633,6 +3047,16 @@ export class QvacAssistant extends ReadyResource {
2633
3047
  }
2634
3048
  }
2635
3049
  }
3050
+ async _contextReading({ chatId, meshId, folded }, ctxSize) {
3051
+ const last = folded ? {} : await this._lastRunMetrics(chatId, meshId);
3052
+ const window = ctxSize || last.ctxSize;
3053
+ if (!window) {
3054
+ return {};
3055
+ }
3056
+ return last.ctxSize === window
3057
+ ? { ctxSize: window, cacheTokens: last.cacheTokens }
3058
+ : { ctxSize: window };
3059
+ }
2636
3060
  async _lastRunMetrics(chatId, meshId) {
2637
3061
  const deviceId = this._requireDeviceId();
2638
3062
  const { chunks } = await meshScoped(this._engine, meshId)