@qvac/core 0.10.2 → 0.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/CHANGELOG.md +66 -1
  2. package/dist-lib/lib/api/agent-folders.d.ts +22 -0
  3. package/dist-lib/lib/api/agent-folders.js +115 -0
  4. package/dist-lib/lib/api/agents.d.ts +3 -1
  5. package/dist-lib/lib/api/agents.js +4 -2
  6. package/dist-lib/lib/api/index.d.ts +1 -0
  7. package/dist-lib/lib/api/index.js +4 -0
  8. package/dist-lib/lib/assistant/history.js +18 -2
  9. package/dist-lib/lib/assistant/message.d.ts +17 -0
  10. package/dist-lib/lib/assistant/message.js +28 -0
  11. package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
  12. package/dist-lib/lib/assistant/video-budget.js +29 -0
  13. package/dist-lib/lib/assistant-tree.js +2 -1
  14. package/dist-lib/lib/assistant.d.ts +22 -2
  15. package/dist-lib/lib/assistant.js +377 -15
  16. package/dist-lib/lib/core/chat-fork.js +1 -0
  17. package/dist-lib/lib/core/chat-search.js +1 -1
  18. package/dist-lib/lib/core/chat.d.ts +1 -0
  19. package/dist-lib/lib/core/chat.js +2 -1
  20. package/dist-lib/lib/core/chunk.d.ts +2 -2
  21. package/dist-lib/lib/core/chunk.js +5 -1
  22. package/dist-lib/lib/core/document-attachment.js +3 -0
  23. package/dist-lib/lib/core/files/index.js +4 -4
  24. package/dist-lib/lib/core/files/records/chunks.js +21 -0
  25. package/dist-lib/lib/core/files/records/purge.js +4 -3
  26. package/dist-lib/lib/core/files/records/source.js +3 -5
  27. package/dist-lib/lib/core/files/upload.js +3 -2
  28. package/dist-lib/lib/core/video/attachment.d.ts +1 -0
  29. package/dist-lib/lib/core/video/attachment.js +7 -0
  30. package/dist-lib/lib/core/video/digest.d.ts +16 -0
  31. package/dist-lib/lib/core/video/digest.js +57 -0
  32. package/dist-lib/lib/core/video/frames.d.ts +32 -0
  33. package/dist-lib/lib/core/video/frames.js +224 -0
  34. package/dist-lib/lib/core.js +25 -5
  35. package/dist-lib/lib/credential-pusher.d.ts +5 -0
  36. package/dist-lib/lib/credential-pusher.js +42 -11
  37. package/dist-lib/lib/errors.d.ts +20 -0
  38. package/dist-lib/lib/errors.js +20 -0
  39. package/dist-lib/lib/lent-harness.d.ts +2 -0
  40. package/dist-lib/lib/lent-harness.js +51 -0
  41. package/dist-lib/lib/mesh-scoped-engine.js +3 -0
  42. package/dist-lib/lib/run-notices.d.ts +1 -0
  43. package/dist-lib/lib/run-notices.js +2 -1
  44. package/dist-lib/lib/task.js +1 -1
  45. package/dist-lib/schema/engine.js +53 -2
  46. package/dist-lib/schema/indexes.js +2 -2
  47. package/dist-lib/schema/qvac.js +4 -2
  48. package/dist-lib/spec/bind.js +9 -0
  49. package/dist-lib/spec/calls.js +3 -0
  50. package/dist-lib/spec/capabilities.d.ts +6 -0
  51. package/dist-lib/spec/hrpc/hrpc.json +39 -0
  52. package/dist-lib/spec/hrpc/index.js +39 -3
  53. package/dist-lib/spec/hrpc/messages.js +177 -6
  54. package/dist-lib/spec/hyperschema/index.js +177 -6
  55. package/dist-lib/spec/hyperschema/schema.json +126 -0
  56. package/dist-lib/spec/hyperschema/types.d.ts +35 -0
  57. package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
  58. package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
  59. package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
  60. package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
  61. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  62. package/package.json +4 -1
  63. package/tui/README.md +10 -0
  64. package/tui/app.mjs +63 -2
  65. package/tui/lib/folders-command.mjs +92 -0
  66. package/tui/lib/knowledge-import.mjs +3 -21
  67. package/tui/lib/local-path.mjs +21 -0
  68. package/tui/lib/render.mjs +5 -0
  69. package/tui/main.mjs +8 -0
@@ -44,7 +44,10 @@ import { isLifecycleTeardown, knownAbortOutcome, RUN_ABORT_REASONS, RUN_FAILURE_
44
44
  import { unref } from './unref.js';
45
45
  import { encodeAgentSkillReadiness } from './skill-readiness.js';
46
46
  import { harnessClearKvCache, harnessDeathReason, harnessDeleteModel, harnessDied, harnessListModels, harnessPreviewModelSource, harnessSkillCatalog, harnessSkillManagement, harnessDetectLanguage, harnessSpeak, harnessSpeakStream, harnessTools, harnessTranscribe, heldLoadModel, droppedHeldLoadModel, imageDownloadProgress, isHarnessEvent, modelCatalogChanged, toolReportEntry } from './assistant/harness.js';
47
- import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders } from './assistant/message.js';
47
+ import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders, hiddenVideoNotice, lentVideoNotice, unreadVideoNotice, unsyncedVideoNotice, VIDEO_NO_AUDIO, VIDEO_SILENT, videoAnnotation, videoTranscriptBlock } from './assistant/message.js';
48
+ import { frameTimes, probeVideo, sampleFrames } from './core/video/frames.js';
49
+ import { packVideoDigest, parseVideoDigestHeader, unpackVideoDigest, VIDEO_DIGEST_MIME, videoDigestFileName } from './core/video/digest.js';
50
+ import { VIDEO_FRAME_EDGE, videoFrameCount } from './assistant/video-budget.js';
48
51
  import { drainBody, raceInactivity } from './assistant/fetch-body.js';
49
52
  import { GENERIC_ACKNOWLEDGEMENTS, HISTORY_TYPES, SKILL_TOOL, TERMINAL_STATES, couldNameSenders, historyFromChunks, inputAvailableToRun, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
50
53
  import { errorMessage, isRecord } from './guards.js';
@@ -59,6 +62,7 @@ const QvacState = local.getEnum('@local/qvac-state');
59
62
  const FINAL_FLUSH_MS = 10_000;
60
63
  const UPLOAD_FRAME_BYTES = 64 * 1024;
61
64
  const RUN_IDLE_TIMEOUT_MS = 15 * 60_000;
65
+ const MODEL_RELOAD_DELAY_MS = 2_500;
62
66
  function idleWatch(controller, ms) {
63
67
  let last = 0;
64
68
  let parked = 0;
@@ -114,6 +118,24 @@ function attachPages(turn, files) {
114
118
  function noBytes() {
115
119
  return Promise.resolve(null);
116
120
  }
121
+ function videoFrameCacheName(video, count, frame) {
122
+ return imageCacheName({ blobId: video.blobId, fileName: `frames.${count}.f${frame}.jpg` });
123
+ }
124
+ function videoHeaderCacheName(digestBlobId) {
125
+ return imageCacheName({ blobId: digestBlobId, fileName: 'frames.json' });
126
+ }
127
+ function videoAudioBlock(fileName, { audio, transcript }) {
128
+ if (audio === 'none') {
129
+ return VIDEO_NO_AUDIO;
130
+ }
131
+ if (audio === 'silent') {
132
+ return VIDEO_SILENT;
133
+ }
134
+ if (audio === 'speech' && transcript) {
135
+ return videoTranscriptBlock(transcript);
136
+ }
137
+ return unheardAudioNotice(`The audio of the attached video "${fileName}"`, 'transcription failed');
138
+ }
117
139
  function pdfPageCacheName(pdf, page) {
118
140
  return imageCacheName({ blobId: pdf.blobId, fileName: `${pdf.fileName}.p${page}.jpg` });
119
141
  }
@@ -146,6 +168,11 @@ function isSetUpHere(status) {
146
168
  function sameNameList(left, right) {
147
169
  return left.length === right.length && left.every((name, index) => name === right[index]);
148
170
  }
171
+ function sameLoadSettings(a, b) {
172
+ return ((a.ctxSize ?? 0) === (b.ctxSize ?? 0) &&
173
+ (a.kvCacheK ?? '') === (b.kvCacheK ?? '') &&
174
+ (a.kvCacheV ?? '') === (b.kvCacheV ?? ''));
175
+ }
149
176
  function withConfigDefaults(agent, config) {
150
177
  if (agent.ctxSize || !config?.ctxSize)
151
178
  return agent;
@@ -161,6 +188,7 @@ const RAG_SOURCE_BOUNDS = {
161
188
  excerptChars: RAG_SOURCE_EXCERPT_CHARS
162
189
  };
163
190
  const WEB_RAG_SOURCE_BOUNDS = { ...RAG_SOURCE_BOUNDS, maxUrlChars: MAX_RAG_SOURCE_URL_CHARS };
191
+ const NO_FS_GRANTS = { roots: [], write: [] };
164
192
  export class QvacAssistant extends ReadyResource {
165
193
  _engine;
166
194
  _harness;
@@ -177,6 +205,8 @@ export class QvacAssistant extends ReadyResource {
177
205
  _drives;
178
206
  _blobReadTimeoutMs;
179
207
  _runIdleTimeoutMs;
208
+ _modelReloadDelayMs;
209
+ _reloads = new Map();
180
210
  _pdfRasterizer;
181
211
  _warmAbort = new AbortController();
182
212
  _unloading = new Map();
@@ -217,6 +247,7 @@ export class QvacAssistant extends ReadyResource {
217
247
  _runReplay;
218
248
  _lifecycle;
219
249
  _audioTranscripts;
250
+ _videoProbes;
220
251
  _pdfPages;
221
252
  _chats;
222
253
  _runImageCacheNames;
@@ -225,7 +256,7 @@ export class QvacAssistant extends ReadyResource {
225
256
  _modelCatalog = new ModelCatalogRevisionHub();
226
257
  _onHarnessStatus;
227
258
  _onModelCatalogEvent;
228
- constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
259
+ constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, modelReloadDelayMs = MODEL_RELOAD_DELAY_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
229
260
  super();
230
261
  this._engine = engine;
231
262
  this._harness = harness;
@@ -241,6 +272,7 @@ export class QvacAssistant extends ReadyResource {
241
272
  this._preloadCurrentAgentModel = preloadCurrentAgentModel;
242
273
  this._blobReadTimeoutMs = blobReadTimeoutMs;
243
274
  this._runIdleTimeoutMs = runIdleTimeoutMs;
275
+ this._modelReloadDelayMs = modelReloadDelayMs;
244
276
  this._pdfRasterizer = pdfRasterizerOption ?? createPdfRasterizer(logger);
245
277
  this._deviceId = null;
246
278
  this._info = { toolNames: [], skills: [], supportsImages: false };
@@ -267,6 +299,7 @@ export class QvacAssistant extends ReadyResource {
267
299
  this._runReplay = new RunReplayStore(runReplayDir ?? persistentDir('run-replay'));
268
300
  this._lifecycle = new AbortController();
269
301
  this._audioTranscripts = new Map();
302
+ this._videoProbes = new Map();
270
303
  this._pdfPages = new Map();
271
304
  this._chats = new Map();
272
305
  this._runImageCacheNames = new Map();
@@ -730,6 +763,7 @@ export class QvacAssistant extends ReadyResource {
730
763
  this._harness.off?.('status', this._onHarnessStatus);
731
764
  this._harness.off?.('model', this._onModelCatalogEvent);
732
765
  this._warmAbort.abort(RUNNER_CLOSED);
766
+ this._cancelModelReloads();
733
767
  this._onlineBoot.destroy(new CoreError('ASSISTANT_CLOSING'));
734
768
  for (const run of this._runs.values()) {
735
769
  run.controller.abort(RUNNER_CLOSED);
@@ -768,6 +802,7 @@ export class QvacAssistant extends ReadyResource {
768
802
  this._log?.info('assistant', 'suspending', { runs: this._runs.size });
769
803
  this._parked.clear();
770
804
  this._stopMeshIntake();
805
+ this._cancelModelReloads();
771
806
  await this._downloads?.suspend();
772
807
  this._meshList?.destroy();
773
808
  this._accountFrames?.destroy();
@@ -850,6 +885,44 @@ export class QvacAssistant extends ReadyResource {
850
885
  const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
851
886
  return rows?.some((row) => row.name === model && row.isCached) === true;
852
887
  }
888
+ _scheduleModelReload(agent) {
889
+ const deviceId = this._deviceId;
890
+ if (!this._harness.loadModel ||
891
+ !deviceId ||
892
+ !agent.modelName ||
893
+ !agent.providerDeviceId ||
894
+ !b4a.equals(agent.providerDeviceId, deviceId)) {
895
+ return;
896
+ }
897
+ const name = agent.modelName;
898
+ clearTimeout(this._reloads.get(name));
899
+ const timer = setTimeout(() => {
900
+ this._reloads.delete(name);
901
+ this._reloadLoadedModel(agent).catch((err) => this._log?.warn('models', 'model reload failed', { model: name, err }));
902
+ }, this._modelReloadDelayMs);
903
+ unref(timer);
904
+ this._reloads.set(name, timer);
905
+ }
906
+ async _reloadLoadedModel(agent) {
907
+ if (this.closing || this.suspended || !this._harness.loadModel)
908
+ return;
909
+ const model = runnableModelName(agent.modelName, await this._runnableHost());
910
+ const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
911
+ const busy = [...this._runs.values()].some((run) => run.executing && !run.borrowed && run.model === model);
912
+ if (busy || !rows?.some((row) => row.name === model && row.isLoaded)) {
913
+ return;
914
+ }
915
+ const config = await this._engine.getConfig({}).catch(noop);
916
+ const knobs = withConfigDefaults(agent, config);
917
+ const { loaded, reason } = await this._harness.loadModel({ model, agent: knobs });
918
+ this._log?.info('models', 'model reload requested', { model, loaded, reason });
919
+ }
920
+ _cancelModelReloads() {
921
+ for (const timer of this._reloads.values()) {
922
+ clearTimeout(timer);
923
+ }
924
+ this._reloads.clear();
925
+ }
853
926
  async _agentRunning(model, activeAgentId) {
854
927
  const deviceId = this._requireDeviceId();
855
928
  const meshId = this._accountMeshId;
@@ -1072,21 +1145,26 @@ export class QvacAssistant extends ReadyResource {
1072
1145
  .forget(chatId)
1073
1146
  .catch((err) => this._log?.warn('assistant', 'inspector records not dropped', { err }));
1074
1147
  }
1075
- async _reclaimRekeyedAgents(meshId, frames) {
1148
+ async _followAgentEdits(meshId, frames) {
1076
1149
  let known = null;
1077
1150
  for await (const frame of frames) {
1078
1151
  if (this.closing || this.suspended)
1079
1152
  return;
1080
1153
  const agents = frame.agents ?? [];
1081
1154
  this._followHosted(meshId, agents);
1082
- const next = new Map(agents.map((agent) => [agent.id, agentRekeyFingerprint(agent)]));
1083
1155
  for (const agent of agents) {
1084
1156
  const previous = known?.get(agent.id);
1085
- if (previous !== undefined && previous !== next.get(agent.id)) {
1157
+ if (!previous) {
1158
+ continue;
1159
+ }
1160
+ if (agentRekeyFingerprint(previous) !== agentRekeyFingerprint(agent)) {
1086
1161
  void this._reclaimAgentChatCaches(meshId, agent);
1087
1162
  }
1163
+ if (!sameLoadSettings(previous, agent)) {
1164
+ this._scheduleModelReload(agent);
1165
+ }
1088
1166
  }
1089
- known = next;
1167
+ known = new Map(agents.map((agent) => [agent.id, agent]));
1090
1168
  }
1091
1169
  }
1092
1170
  async _reclaimAgentChatCaches(meshId, agent) {
@@ -1299,7 +1377,7 @@ export class QvacAssistant extends ReadyResource {
1299
1377
  const watches = [
1300
1378
  this._streamWatch(`run-requests:${meshId}`, () => this._engine.runRequestsWatch({ meshId }), (frames) => this._dispatchRunRequests(meshId, frames)),
1301
1379
  this._streamWatch(`chat-list:${meshId}`, () => this._engine.chatsWatch({ meshId }), (frames) => this._reclaimDeletedChats(frames)),
1302
- this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._reclaimRekeyedAgents(meshId, frames))
1380
+ this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._followAgentEdits(meshId, frames))
1303
1381
  ];
1304
1382
  return {
1305
1383
  start: () => watches.forEach((watch) => watch.start()),
@@ -1611,7 +1689,7 @@ export class QvacAssistant extends ReadyResource {
1611
1689
  model,
1612
1690
  log: this._log
1613
1691
  });
1614
- await this._compactIfNearFull({
1692
+ const folded = await this._compactIfNearFull({
1615
1693
  chatId,
1616
1694
  transcript,
1617
1695
  trigger: 'before-run',
@@ -1619,7 +1697,7 @@ export class QvacAssistant extends ReadyResource {
1619
1697
  meshId
1620
1698
  });
1621
1699
  const toolConfigsPromise = this._loadToolConfigs(engine, agent.id);
1622
- const fsGrantsPromise = this._loadFsGrants(agent.id);
1700
+ const fsGrantsPromise = lender ? Promise.resolve(NO_FS_GRANTS) : this._loadFsGrants(agent.id);
1623
1701
  const all = await listHistoryChunks(engine, chatId);
1624
1702
  if (agentSwitched(all, request, deviceId)) {
1625
1703
  await this._deleteChatKvCache(chatId, 'agent switched');
@@ -1670,10 +1748,11 @@ export class QvacAssistant extends ReadyResource {
1670
1748
  enabledSkills: []
1671
1749
  }
1672
1750
  : configuredAgent;
1673
- const seesImages = this._harness.supportsImagesFor
1674
- ? this._harness.supportsImagesFor(model)
1675
- : (this._harness.supportsImages ?? this._info.supportsImages) !== false;
1751
+ const sendsFiles = lender === null;
1752
+ const carriesMedia = messages.some((turn) => turn.images?.length || turn.pdfs?.length || turn.videos?.length);
1753
+ const seesImages = sendsFiles && carriesMedia && (await this._seesImages(model));
1676
1754
  let unreadableImages = new Map();
1755
+ const sampled = [];
1677
1756
  let renderedPdfIds = new Set();
1678
1757
  if (!seesImages) {
1679
1758
  const dropped = messages.reduce((count, turn) => count + (turn.images?.length ?? 0), 0);
@@ -1865,11 +1944,21 @@ export class QvacAssistant extends ReadyResource {
1865
1944
  const injected = knowledge
1866
1945
  ? await this._injectKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords })
1867
1946
  : { documentsInjected: false, sourcesWritten: false, ragInjected: false };
1868
- const hasAudio = messages.some((turn) => turn.audioAttachments?.length);
1947
+ const hasAudio = messages.some((turn) => turn.audioAttachments?.length || turn.videos?.length);
1869
1948
  if (hasAudio) {
1870
1949
  modelWatch.transcribe(true);
1871
1950
  }
1872
1951
  await this._hydrateAudio(messages, controller.signal);
1952
+ if (messages.some((turn) => turn.videos?.length)) {
1953
+ await this._hydrateVideos(messages, {
1954
+ signal: controller.signal,
1955
+ runId,
1956
+ seesImages,
1957
+ lent: !sendsFiles,
1958
+ reading: await this._contextReading({ chatId, meshId, folded }, configuredAgent.ctxSize),
1959
+ sampled
1960
+ });
1961
+ }
1873
1962
  if (hasAudio) {
1874
1963
  modelWatch.transcribe(false);
1875
1964
  }
@@ -2076,6 +2165,7 @@ export class QvacAssistant extends ReadyResource {
2076
2165
  metrics: citedTitles ? { ...metrics, citedTitles } : metrics
2077
2166
  });
2078
2167
  titleOutcome = 'completed';
2168
+ await this._storeVideoDigests(sampled, { meshId, chatId, runId, deviceId });
2079
2169
  }
2080
2170
  catch (err) {
2081
2171
  const outcome = controller.signal.aborted
@@ -2333,9 +2423,18 @@ export class QvacAssistant extends ReadyResource {
2333
2423
  return null;
2334
2424
  }
2335
2425
  }
2336
- async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId }) {
2426
+ async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId, sourceId }) {
2337
2427
  const duplex = this._engine.uploadAttachment();
2338
- duplex.write({ chatId, runId, originDeviceId: deviceId, fileName, mimeType, raw, meshId });
2428
+ duplex.write({
2429
+ chatId,
2430
+ runId,
2431
+ originDeviceId: deviceId,
2432
+ fileName,
2433
+ mimeType,
2434
+ raw,
2435
+ meshId,
2436
+ sourceId
2437
+ });
2339
2438
  const buf = b4a.toBuffer(bytes);
2340
2439
  for (let i = 0; i < buf.length; i += UPLOAD_FRAME_BYTES) {
2341
2440
  duplex.write({ data: buf.subarray(i, i + UPLOAD_FRAME_BYTES) });
@@ -2401,6 +2500,259 @@ export class QvacAssistant extends ReadyResource {
2401
2500
  return unheardAudioNotice(label, 'transcription failed on this device');
2402
2501
  }
2403
2502
  }
2503
+ async _seesImages(model) {
2504
+ if (this._harness.supportsImagesFor) {
2505
+ return this._harness.supportsImagesFor(model);
2506
+ }
2507
+ const anyModel = (this._harness.supportsImages ?? this._info.supportsImages) !== false;
2508
+ if (!anyModel || !this._harness.list) {
2509
+ return anyModel;
2510
+ }
2511
+ const rows = await this._harness.list({ type: 'llm' }).catch(noop);
2512
+ if (!rows) {
2513
+ return anyModel;
2514
+ }
2515
+ const row = rows.find((candidate) => candidate.name === model);
2516
+ return row ? (row.supportsImages ?? anyModel) : false;
2517
+ }
2518
+ async _hydrateVideos(messages, run) {
2519
+ const { signal, runId, seesImages, lent, reading, sampled } = run;
2520
+ const turns = messages.filter((turn) => turn.videos?.length);
2521
+ if (!turns.length) {
2522
+ return;
2523
+ }
2524
+ const asked = messages.findLast((turn) => turn.role === 'user')?.videos ?? [];
2525
+ const dir = this._imageCacheDir;
2526
+ await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
2527
+ const pinned = this._runImageCacheNames.get(runId) ?? new Set();
2528
+ this._runImageCacheNames.set(runId, pinned);
2529
+ const unsampled = turns.flatMap((turn) => turn.videos ?? []).filter((ref) => !ref.digest);
2530
+ const budget = { ...reading, videos: unsampled.length };
2531
+ for (const turn of turns) {
2532
+ for (const ref of turn.videos ?? []) {
2533
+ if (signal.aborted)
2534
+ return;
2535
+ const hydration = { dir, ref, pinned, seesImages, lent, signal, budget, sampled };
2536
+ const blocks = await this._hydrateVideo(turn, hydration);
2537
+ turn.content = [turn.content.trim(), ...blocks].filter(Boolean).join('\n\n');
2538
+ }
2539
+ delete turn.videos;
2540
+ }
2541
+ const pictured = asked.some((ref) => ref.digest || this._videoProbes.get(ref.blobId)?.hasVideo);
2542
+ if (!seesImages && !lent && pictured) {
2543
+ throw new Error(RUN_FAILURE_REASONS.videoNeedsVision);
2544
+ }
2545
+ }
2546
+ async _hydrateVideo(turn, hydration) {
2547
+ const { ref, seesImages, lent, signal } = hydration;
2548
+ const source = { opened: null };
2549
+ const open = () => (source.opened ??= this._openVideo(hydration));
2550
+ try {
2551
+ const video = await this._readVideo(hydration, open);
2552
+ if (video === 'unsynced') {
2553
+ return [unsyncedVideoNotice(ref.fileName)];
2554
+ }
2555
+ if (video === 'audio') {
2556
+ return [await this._transcribeAttachment(ref, signal)];
2557
+ }
2558
+ if (!seesImages || !('files' in video)) {
2559
+ const notice = lent ? lentVideoNotice(ref.fileName) : hiddenVideoNotice(ref.fileName);
2560
+ return [notice, videoAudioBlock(ref.fileName, video)];
2561
+ }
2562
+ attachPages(turn, video.files);
2563
+ const annotation = videoAnnotation({
2564
+ fileName: ref.fileName,
2565
+ duration: video.duration,
2566
+ times: video.times
2567
+ });
2568
+ return [annotation, videoAudioBlock(ref.fileName, video)];
2569
+ }
2570
+ catch (err) {
2571
+ if (signal.aborted) {
2572
+ throw err;
2573
+ }
2574
+ this._log?.warn('assistant', `video hydration failed — ${ref.fileName}`, err);
2575
+ return [unreadVideoNotice(ref.fileName)];
2576
+ }
2577
+ finally {
2578
+ const video = await source.opened?.catch(noop);
2579
+ if (video) {
2580
+ await fs.promises.unlink(video.file).catch(noop);
2581
+ }
2582
+ }
2583
+ }
2584
+ async _readVideo(hydration, open) {
2585
+ const { ref, seesImages, signal } = hydration;
2586
+ const stored = ref.digest ? await this._storedVideo(ref.digest, hydration) : null;
2587
+ if (stored) {
2588
+ return stored;
2589
+ }
2590
+ const probe = await this._probeVideo(ref, open);
2591
+ if (!probe) {
2592
+ return 'unsynced';
2593
+ }
2594
+ if (!probe.hasVideo) {
2595
+ return 'audio';
2596
+ }
2597
+ if (!seesImages) {
2598
+ return probe.hasAudio ? this._hearVideo(ref, open, signal) : { audio: 'none' };
2599
+ }
2600
+ const video = await open();
2601
+ if (!video) {
2602
+ return 'unsynced';
2603
+ }
2604
+ return this._sampleVideo(video, probe, hydration);
2605
+ }
2606
+ async _probeVideo(ref, open) {
2607
+ const known = this._videoProbes.get(ref.blobId);
2608
+ if (known) {
2609
+ return known;
2610
+ }
2611
+ const video = await open();
2612
+ if (!video) {
2613
+ return null;
2614
+ }
2615
+ const probe = await probeVideo(video.file);
2616
+ this._videoProbes.set(ref.blobId, probe);
2617
+ return probe;
2618
+ }
2619
+ async _storedVideo(digest, { dir, ref, pinned, signal }) {
2620
+ const headerName = videoHeaderCacheName(digest.blobId);
2621
+ pinned.add(headerName);
2622
+ const header = await fs.promises
2623
+ .readFile(path.join(dir, headerName), 'utf8')
2624
+ .then(parseVideoDigestHeader, noop);
2625
+ if (header) {
2626
+ const files = header.times.map((_, i) => {
2627
+ const name = videoFrameCacheName(ref, header.times.length, i);
2628
+ pinned.add(name);
2629
+ return path.join(dir, name);
2630
+ });
2631
+ const cached = await Promise.all(files.map((file) => cacheOrTouch(file, noBytes)));
2632
+ if (cached.every(Boolean)) {
2633
+ return { ...header, files };
2634
+ }
2635
+ }
2636
+ const bytes = await this._readBlob(digest.url, signal);
2637
+ if (!bytes) {
2638
+ return null;
2639
+ }
2640
+ const target = { dir, pinned, digestBlobId: digest.blobId };
2641
+ try {
2642
+ return await this._cacheVideo(ref, unpackVideoDigest(bytes), target);
2643
+ }
2644
+ catch (err) {
2645
+ this._log?.warn('assistant', `video digest unreadable — ${ref.fileName}`, err);
2646
+ return null;
2647
+ }
2648
+ }
2649
+ async _cacheVideo(ref, digest, { dir, pinned, digestBlobId }) {
2650
+ const { frames, ...header } = digest;
2651
+ const files = frames.map((_, i) => {
2652
+ const name = videoFrameCacheName(ref, frames.length, i);
2653
+ pinned.add(name);
2654
+ return path.join(dir, name);
2655
+ });
2656
+ await Promise.all(frames.map((frame, i) => cacheOrTouch(files[i], () => Promise.resolve(frame))));
2657
+ if (digestBlobId) {
2658
+ const headerName = videoHeaderCacheName(digestBlobId);
2659
+ pinned.add(headerName);
2660
+ await cacheOrTouch(path.join(dir, headerName), () => Promise.resolve(b4a.from(JSON.stringify(header))));
2661
+ }
2662
+ return { ...header, files };
2663
+ }
2664
+ async _sampleVideo(video, probe, hydration) {
2665
+ const { ref, budget, signal } = hydration;
2666
+ const started = Date.now();
2667
+ const count = videoFrameCount(probe, budget);
2668
+ const times = frameTimes(probe.duration, count);
2669
+ const frames = await sampleFrames(video.file, { times, maxEdge: VIDEO_FRAME_EDGE, probe });
2670
+ const sampleMs = Date.now() - started;
2671
+ const heard = probe.hasAudio
2672
+ ? await this._hearVideo(ref, () => Promise.resolve(video), signal)
2673
+ : { audio: 'none' };
2674
+ const digest = {
2675
+ duration: probe.duration,
2676
+ times: frames.map((frame) => frame.time),
2677
+ ...heard,
2678
+ frames: frames.map((frame) => frame.jpeg)
2679
+ };
2680
+ if (!hydration.ref.digest) {
2681
+ hydration.sampled.push({ ref, digest });
2682
+ }
2683
+ this._log?.info('assistant', 'video frames', {
2684
+ fileName: ref.fileName,
2685
+ duration: probe.duration,
2686
+ frames: frames.length,
2687
+ budget: count,
2688
+ ctxSize: budget.ctxSize ?? null,
2689
+ cacheTokens: budget.cacheTokens ?? null,
2690
+ audio: heard.audio,
2691
+ sampleMs
2692
+ });
2693
+ return this._cacheVideo(ref, digest, hydration);
2694
+ }
2695
+ async _storeVideoDigests(sampled, target) {
2696
+ const dir = this._imageCacheDir;
2697
+ const pinned = this._runImageCacheNames.get(target.runId) ?? new Set();
2698
+ for (const { ref, digest } of sampled) {
2699
+ try {
2700
+ const blobId = await this._uploadVideoDigest(ref, digest, target);
2701
+ await this._cacheVideo(ref, digest, { dir, pinned, digestBlobId: blobId });
2702
+ }
2703
+ catch (err) {
2704
+ this._log?.warn('assistant', `video digest not stored — ${ref.fileName}`, err);
2705
+ }
2706
+ }
2707
+ }
2708
+ async _uploadVideoDigest(ref, digest, target) {
2709
+ const attachment = await this._uploadGenerated({
2710
+ fileName: videoDigestFileName(ref.fileName),
2711
+ mimeType: VIDEO_DIGEST_MIME,
2712
+ bytes: packVideoDigest(digest),
2713
+ raw: true
2714
+ }, { ...target, sourceId: ref.id });
2715
+ return attachment.blobId;
2716
+ }
2717
+ async _openVideo({ dir, ref, pinned, signal }) {
2718
+ const bytes = await this._readBlob(ref.url, signal);
2719
+ if (!bytes) {
2720
+ return null;
2721
+ }
2722
+ const name = `${imageCacheName(ref)}.source`;
2723
+ pinned.add(name);
2724
+ const file = path.join(dir, name);
2725
+ await fs.promises.writeFile(file, bytes, { mode: 0o600 });
2726
+ return { bytes, file };
2727
+ }
2728
+ async _hearVideo(ref, open, signal) {
2729
+ const cached = this._audioTranscripts.get(ref.blobId);
2730
+ if (cached !== undefined) {
2731
+ return cached ? { audio: 'speech', transcript: cached } : { audio: 'silent' };
2732
+ }
2733
+ if (!this._harness.transcribe) {
2734
+ return { audio: 'unheard' };
2735
+ }
2736
+ try {
2737
+ const video = await open();
2738
+ if (!video) {
2739
+ return { audio: 'unheard' };
2740
+ }
2741
+ const model = mediaModelNameFor('transcription', await this._persistedProfile());
2742
+ await this.ensureMediaModelUnpaused('transcription', model);
2743
+ await this._downloads?.waitUntilCached(model, signal);
2744
+ const transcript = (await this._harness.transcribe({ model: '', audio: b4a.toBuffer(video.bytes), signal })).trim();
2745
+ this._audioTranscripts.set(ref.blobId, transcript);
2746
+ return transcript ? { audio: 'speech', transcript } : { audio: 'silent' };
2747
+ }
2748
+ catch (err) {
2749
+ if (signal.aborted) {
2750
+ throw err;
2751
+ }
2752
+ this._log?.warn('assistant', `video transcription failed — ${ref.fileName}`, err);
2753
+ return { audio: 'unheard' };
2754
+ }
2755
+ }
2404
2756
  async _hydrateImages(messages, signal, runId) {
2405
2757
  const unreadable = new Map();
2406
2758
  const turns = messages.filter((turn) => turn.images?.length);
@@ -2630,6 +2982,16 @@ export class QvacAssistant extends ReadyResource {
2630
2982
  }
2631
2983
  }
2632
2984
  }
2985
+ async _contextReading({ chatId, meshId, folded }, ctxSize) {
2986
+ const last = folded ? {} : await this._lastRunMetrics(chatId, meshId);
2987
+ const window = ctxSize || last.ctxSize;
2988
+ if (!window) {
2989
+ return {};
2990
+ }
2991
+ return last.ctxSize === window
2992
+ ? { ctxSize: window, cacheTokens: last.cacheTokens }
2993
+ : { ctxSize: window };
2994
+ }
2633
2995
  async _lastRunMetrics(chatId, meshId) {
2634
2996
  const deviceId = this._requireDeviceId();
2635
2997
  const { chunks } = await meshScoped(this._engine, meshId)
@@ -166,6 +166,7 @@ async function forkedChunk(ctx, chunk) {
166
166
  chatId: ctx.chatId,
167
167
  runId: chunk.runId && forkedId(ctx, chunk.runId),
168
168
  parentId: chunk.parentId && ctx.kept.has(chunk.parentId) ? forkedId(ctx, chunk.parentId) : null,
169
+ sourceId: chunk.sourceId && forkedId(ctx, chunk.sourceId),
169
170
  compaction: chunk.compaction && {
170
171
  ...chunk.compaction,
171
172
  throughId: forkedId(ctx, chunk.compaction.throughId)
@@ -177,7 +177,7 @@ function matchAttachments(chunks, needle, cap) {
177
177
  if (matches.length >= cap)
178
178
  break;
179
179
  const attachment = chunk.attachment;
180
- if (!attachment || (attachment.deletedAt ?? 0) > 0) {
180
+ if (!attachment || (attachment.deletedAt ?? 0) > 0 || chunk.sourceId) {
181
181
  continue;
182
182
  }
183
183
  const found = locate(attachment.fileName, needle);
@@ -55,4 +55,5 @@ export interface RemoveChatAttachmentsOptions {
55
55
  export declare function removeChatAttachments(store: Mesh, chatId: string, { deviceId, at }: RemoveChatAttachmentsOptions): Promise<void>;
56
56
  export declare function deleteChat(store: Mesh, deviceId: Buffer | null, id: string): Promise<ChatListing>;
57
57
  export declare function softDeleteChatsForAgent(store: Mesh, deviceId: Buffer | null, agentId: string): Promise<void>;
58
+ export declare const AGENT_DELETED = "agent-deleted";
58
59
  export declare function cancelRunsForAgent(store: Mesh, deviceId: Buffer, agentId: string): Promise<void>;
@@ -291,8 +291,9 @@ export async function deleteChat(store, deviceId, id) {
291
291
  export function softDeleteChatsForAgent(store, deviceId, agentId) {
292
292
  return eachChatOfAgent(store, agentId, (chatId) => deleteChat(store, deviceId, chatId));
293
293
  }
294
+ export const AGENT_DELETED = 'agent-deleted';
294
295
  export function cancelRunsForAgent(store, deviceId, agentId) {
295
- return eachChatOfAgent(store, agentId, (chatId) => cancelActiveRuns(store, chatId, { deviceId, agentId }));
296
+ return eachChatOfAgent(store, agentId, (chatId) => cancelActiveRuns(store, chatId, { deviceId, agentId, reason: AGENT_DELETED }));
296
297
  }
297
298
  async function eachChatOfAgent(store, agentId, fn) {
298
299
  const snap = store.view.snapshot();
@@ -3,7 +3,7 @@ import type { QvacChunk, QvacUpdateChunkOperation } from '../../spec/mesh/hypers
3
3
  import type { Mesh, MeshDispatcher } from './mesh.ts';
4
4
  import { type Bounds } from './range.ts';
5
5
  type ChunkPayload = Pick<QvacChunk, 'userMessage' | 'content' | 'thinking' | 'toolCall' | 'toolResult' | 'attachment' | 'approvalRequest' | 'runStatus' | 'runRequest' | 'sources' | 'compaction'>;
6
- type AppendChunkInput = Pick<QvacChunk, 'chatId' | 'runId' | 'type' | 'originDeviceId' | 'parentId' | 'effect' | 'intent' | 'display' | 'rawCall' | 'capability' | 'capabilityArgs' | 'enhance'> & ChunkPayload & {
6
+ type AppendChunkInput = Pick<QvacChunk, 'chatId' | 'runId' | 'type' | 'originDeviceId' | 'parentId' | 'effect' | 'intent' | 'display' | 'rawCall' | 'capability' | 'capabilityArgs' | 'enhance' | 'sourceId'> & ChunkPayload & {
7
7
  id?: string;
8
8
  };
9
9
  type UpdateChunkInput = Pick<QvacChunk, 'id' | 'runStatus' | 'attachment'>;
@@ -40,7 +40,7 @@ export declare function getChunk(store: Mesh, id: string): Promise<QvacChunk | n
40
40
  export interface AppendedChunk {
41
41
  readonly id: string;
42
42
  }
43
- export declare function appendChunk(store: Mesh, { id, chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, ...payload }: AppendChunkInput, into?: MeshDispatcher): Promise<AppendedChunk>;
43
+ export declare function appendChunk(store: Mesh, { id, chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, sourceId, ...payload }: AppendChunkInput, into?: MeshDispatcher): Promise<AppendedChunk>;
44
44
  export declare function updateChunk(store: Mesh, { id, runStatus, attachment }: UpdateChunkInput): Promise<QvacChunk>;
45
45
  export declare function updateOp(chunk: QvacChunk, { id, runStatus, attachment }: UpdateChunkInput): QvacUpdateChunkOperation;
46
46
  export interface ResolveApprovalInput {
@@ -118,7 +118,7 @@ function assertPayloadMatchesType(type, payload) {
118
118
  throw new CoreError('CHUNK_PAYLOAD_INVALID', [`${type} carries ${field} and nothing else`]);
119
119
  }
120
120
  }
121
- export async function appendChunk(store, { id = randomId(), chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, ...payload }, into = store) {
121
+ export async function appendChunk(store, { id = randomId(), chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, sourceId, ...payload }, into = store) {
122
122
  assertPayloadMatchesType(type, payload);
123
123
  if (type === 'sources' && !runId) {
124
124
  throw new CoreError('CHUNK_PAYLOAD_INVALID', ['a sources chunk requires runId']);
@@ -129,6 +129,9 @@ export async function appendChunk(store, { id = randomId(), chatId, runId, type,
129
129
  if (capability && type !== 'run-request') {
130
130
  throw new CoreError('CHUNK_PAYLOAD_INVALID', ['only a run-request carries a capability']);
131
131
  }
132
+ if (sourceId && type !== 'attachment') {
133
+ throw new CoreError('CHUNK_PAYLOAD_INVALID', ['only an attachment is derived from another']);
134
+ }
132
135
  if (type === 'user-message' && !store.ourMemberId) {
133
136
  await store.refreshOurMember();
134
137
  }
@@ -154,6 +157,7 @@ export async function appendChunk(store, { id = randomId(), chatId, runId, type,
154
157
  capability,
155
158
  capabilityArgs,
156
159
  enhance,
160
+ sourceId,
157
161
  createdAt: Date.now(),
158
162
  senderMemberId: type === 'user-message' && store.ourDeviceId && b4a.equals(store.ourDeviceId, originDeviceId)
159
163
  ? store.ourMemberId || undefined
@@ -1,9 +1,12 @@
1
1
  import { isAudioAttachment } from './audio-attachment.js';
2
2
  import { classifyRagDocument, RAG_DOCUMENT_KIND } from './files/rag/index.js';
3
3
  import { IMAGE_MIMES } from './image/normalize.js';
4
+ import { isVideoAttachment } from './video/attachment.js';
4
5
  export function isDocumentAttachment(ref) {
5
6
  if (IMAGE_MIMES.has(ref.mimeType.toLowerCase()))
6
7
  return false;
8
+ if (isVideoAttachment(ref.fileName, ref.mimeType))
9
+ return false;
7
10
  return !isAudioAttachment(ref.fileName, ref.mimeType);
8
11
  }
9
12
  export function isPdfAttachment(ref) {