@qvac/core 0.10.3 → 0.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +37 -1
  2. package/dist-lib/lib/assistant/history.js +18 -2
  3. package/dist-lib/lib/assistant/message.d.ts +17 -0
  4. package/dist-lib/lib/assistant/message.js +28 -0
  5. package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
  6. package/dist-lib/lib/assistant/video-budget.js +29 -0
  7. package/dist-lib/lib/assistant.d.ts +22 -2
  8. package/dist-lib/lib/assistant.js +374 -15
  9. package/dist-lib/lib/core/chat-fork.js +1 -0
  10. package/dist-lib/lib/core/chat-search.js +1 -1
  11. package/dist-lib/lib/core/chunk.d.ts +2 -2
  12. package/dist-lib/lib/core/chunk.js +5 -1
  13. package/dist-lib/lib/core/document-attachment.js +3 -0
  14. package/dist-lib/lib/core/files/index.js +4 -4
  15. package/dist-lib/lib/core/files/records/chunks.js +21 -0
  16. package/dist-lib/lib/core/files/records/purge.js +4 -3
  17. package/dist-lib/lib/core/files/records/source.js +3 -5
  18. package/dist-lib/lib/core/files/upload.js +3 -2
  19. package/dist-lib/lib/core/video/attachment.d.ts +1 -0
  20. package/dist-lib/lib/core/video/attachment.js +7 -0
  21. package/dist-lib/lib/core/video/digest.d.ts +16 -0
  22. package/dist-lib/lib/core/video/digest.js +57 -0
  23. package/dist-lib/lib/core/video/frames.d.ts +32 -0
  24. package/dist-lib/lib/core/video/frames.js +224 -0
  25. package/dist-lib/lib/core.js +19 -1
  26. package/dist-lib/lib/credential-pusher.d.ts +5 -0
  27. package/dist-lib/lib/credential-pusher.js +42 -11
  28. package/dist-lib/lib/run-notices.d.ts +1 -0
  29. package/dist-lib/lib/run-notices.js +2 -1
  30. package/dist-lib/lib/task.js +1 -1
  31. package/dist-lib/schema/engine.js +4 -2
  32. package/dist-lib/schema/indexes.js +2 -2
  33. package/dist-lib/schema/qvac.js +4 -2
  34. package/dist-lib/spec/hrpc/messages.js +16 -6
  35. package/dist-lib/spec/hyperschema/index.js +16 -6
  36. package/dist-lib/spec/hyperschema/schema.json +10 -0
  37. package/dist-lib/spec/hyperschema/types.d.ts +2 -0
  38. package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
  39. package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
  40. package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
  41. package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
  42. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  43. package/package.json +3 -1
  44. package/tui/app.mjs +34 -1
  45. package/tui/lib/render.mjs +5 -0
  46. package/tui/main.mjs +8 -0
@@ -44,7 +44,10 @@ import { isLifecycleTeardown, knownAbortOutcome, RUN_ABORT_REASONS, RUN_FAILURE_
44
44
  import { unref } from './unref.js';
45
45
  import { encodeAgentSkillReadiness } from './skill-readiness.js';
46
46
  import { harnessClearKvCache, harnessDeathReason, harnessDeleteModel, harnessDied, harnessListModels, harnessPreviewModelSource, harnessSkillCatalog, harnessSkillManagement, harnessDetectLanguage, harnessSpeak, harnessSpeakStream, harnessTools, harnessTranscribe, heldLoadModel, droppedHeldLoadModel, imageDownloadProgress, isHarnessEvent, modelCatalogChanged, toolReportEntry } from './assistant/harness.js';
47
- import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders } from './assistant/message.js';
47
+ import { annotateUnreadableImages, audioTranscriptBlock, backfillDocumentOnlyTurn, backfillImageOnlyTurns, silentAudioNotice, unheardAudioNotice, nameSenders, hiddenVideoNotice, lentVideoNotice, unreadVideoNotice, unsyncedVideoNotice, VIDEO_NO_AUDIO, VIDEO_SILENT, videoAnnotation, videoTranscriptBlock } from './assistant/message.js';
48
+ import { frameTimes, probeVideo, sampleFrames } from './core/video/frames.js';
49
+ import { packVideoDigest, parseVideoDigestHeader, unpackVideoDigest, VIDEO_DIGEST_MIME, videoDigestFileName } from './core/video/digest.js';
50
+ import { VIDEO_FRAME_EDGE, videoFrameCount } from './assistant/video-budget.js';
48
51
  import { drainBody, raceInactivity } from './assistant/fetch-body.js';
49
52
  import { GENERIC_ACKNOWLEDGEMENTS, HISTORY_TYPES, SKILL_TOOL, TERMINAL_STATES, couldNameSenders, historyFromChunks, inputAvailableToRun, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds } from './assistant/history.js';
50
53
  import { errorMessage, isRecord } from './guards.js';
@@ -59,6 +62,7 @@ const QvacState = local.getEnum('@local/qvac-state');
59
62
  const FINAL_FLUSH_MS = 10_000;
60
63
  const UPLOAD_FRAME_BYTES = 64 * 1024;
61
64
  const RUN_IDLE_TIMEOUT_MS = 15 * 60_000;
65
+ const MODEL_RELOAD_DELAY_MS = 2_500;
62
66
  function idleWatch(controller, ms) {
63
67
  let last = 0;
64
68
  let parked = 0;
@@ -114,6 +118,24 @@ function attachPages(turn, files) {
114
118
  function noBytes() {
115
119
  return Promise.resolve(null);
116
120
  }
121
+ function videoFrameCacheName(video, count, frame) {
122
+ return imageCacheName({ blobId: video.blobId, fileName: `frames.${count}.f${frame}.jpg` });
123
+ }
124
+ function videoHeaderCacheName(digestBlobId) {
125
+ return imageCacheName({ blobId: digestBlobId, fileName: 'frames.json' });
126
+ }
127
+ function videoAudioBlock(fileName, { audio, transcript }) {
128
+ if (audio === 'none') {
129
+ return VIDEO_NO_AUDIO;
130
+ }
131
+ if (audio === 'silent') {
132
+ return VIDEO_SILENT;
133
+ }
134
+ if (audio === 'speech' && transcript) {
135
+ return videoTranscriptBlock(transcript);
136
+ }
137
+ return unheardAudioNotice(`The audio of the attached video "${fileName}"`, 'transcription failed');
138
+ }
117
139
  function pdfPageCacheName(pdf, page) {
118
140
  return imageCacheName({ blobId: pdf.blobId, fileName: `${pdf.fileName}.p${page}.jpg` });
119
141
  }
@@ -146,6 +168,11 @@ function isSetUpHere(status) {
146
168
  function sameNameList(left, right) {
147
169
  return left.length === right.length && left.every((name, index) => name === right[index]);
148
170
  }
171
+ function sameLoadSettings(a, b) {
172
+ return ((a.ctxSize ?? 0) === (b.ctxSize ?? 0) &&
173
+ (a.kvCacheK ?? '') === (b.kvCacheK ?? '') &&
174
+ (a.kvCacheV ?? '') === (b.kvCacheV ?? ''));
175
+ }
149
176
  function withConfigDefaults(agent, config) {
150
177
  if (agent.ctxSize || !config?.ctxSize)
151
178
  return agent;
@@ -178,6 +205,8 @@ export class QvacAssistant extends ReadyResource {
178
205
  _drives;
179
206
  _blobReadTimeoutMs;
180
207
  _runIdleTimeoutMs;
208
+ _modelReloadDelayMs;
209
+ _reloads = new Map();
181
210
  _pdfRasterizer;
182
211
  _warmAbort = new AbortController();
183
212
  _unloading = new Map();
@@ -218,6 +247,7 @@ export class QvacAssistant extends ReadyResource {
218
247
  _runReplay;
219
248
  _lifecycle;
220
249
  _audioTranscripts;
250
+ _videoProbes;
221
251
  _pdfPages;
222
252
  _chats;
223
253
  _runImageCacheNames;
@@ -226,7 +256,7 @@ export class QvacAssistant extends ReadyResource {
226
256
  _modelCatalog = new ModelCatalogRevisionHub();
227
257
  _onHarnessStatus;
228
258
  _onModelCatalogEvent;
229
- constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
259
+ constructor(engine, harness, { logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform = os.platform(), preloadCurrentAgentModel = true, drives, borrowHarness, blobReadTimeoutMs = BLOB_READ_TIMEOUT_MS, runIdleTimeoutMs = RUN_IDLE_TIMEOUT_MS, modelReloadDelayMs = MODEL_RELOAD_DELAY_MS, pdfRasterizer: pdfRasterizerOption } = {}) {
230
260
  super();
231
261
  this._engine = engine;
232
262
  this._harness = harness;
@@ -242,6 +272,7 @@ export class QvacAssistant extends ReadyResource {
242
272
  this._preloadCurrentAgentModel = preloadCurrentAgentModel;
243
273
  this._blobReadTimeoutMs = blobReadTimeoutMs;
244
274
  this._runIdleTimeoutMs = runIdleTimeoutMs;
275
+ this._modelReloadDelayMs = modelReloadDelayMs;
245
276
  this._pdfRasterizer = pdfRasterizerOption ?? createPdfRasterizer(logger);
246
277
  this._deviceId = null;
247
278
  this._info = { toolNames: [], skills: [], supportsImages: false };
@@ -268,6 +299,7 @@ export class QvacAssistant extends ReadyResource {
268
299
  this._runReplay = new RunReplayStore(runReplayDir ?? persistentDir('run-replay'));
269
300
  this._lifecycle = new AbortController();
270
301
  this._audioTranscripts = new Map();
302
+ this._videoProbes = new Map();
271
303
  this._pdfPages = new Map();
272
304
  this._chats = new Map();
273
305
  this._runImageCacheNames = new Map();
@@ -731,6 +763,7 @@ export class QvacAssistant extends ReadyResource {
731
763
  this._harness.off?.('status', this._onHarnessStatus);
732
764
  this._harness.off?.('model', this._onModelCatalogEvent);
733
765
  this._warmAbort.abort(RUNNER_CLOSED);
766
+ this._cancelModelReloads();
734
767
  this._onlineBoot.destroy(new CoreError('ASSISTANT_CLOSING'));
735
768
  for (const run of this._runs.values()) {
736
769
  run.controller.abort(RUNNER_CLOSED);
@@ -769,6 +802,7 @@ export class QvacAssistant extends ReadyResource {
769
802
  this._log?.info('assistant', 'suspending', { runs: this._runs.size });
770
803
  this._parked.clear();
771
804
  this._stopMeshIntake();
805
+ this._cancelModelReloads();
772
806
  await this._downloads?.suspend();
773
807
  this._meshList?.destroy();
774
808
  this._accountFrames?.destroy();
@@ -851,6 +885,44 @@ export class QvacAssistant extends ReadyResource {
851
885
  const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
852
886
  return rows?.some((row) => row.name === model && row.isCached) === true;
853
887
  }
888
+ _scheduleModelReload(agent) {
889
+ const deviceId = this._deviceId;
890
+ if (!this._harness.loadModel ||
891
+ !deviceId ||
892
+ !agent.modelName ||
893
+ !agent.providerDeviceId ||
894
+ !b4a.equals(agent.providerDeviceId, deviceId)) {
895
+ return;
896
+ }
897
+ const name = agent.modelName;
898
+ clearTimeout(this._reloads.get(name));
899
+ const timer = setTimeout(() => {
900
+ this._reloads.delete(name);
901
+ this._reloadLoadedModel(agent).catch((err) => this._log?.warn('models', 'model reload failed', { model: name, err }));
902
+ }, this._modelReloadDelayMs);
903
+ unref(timer);
904
+ this._reloads.set(name, timer);
905
+ }
906
+ async _reloadLoadedModel(agent) {
907
+ if (this.closing || this.suspended || !this._harness.loadModel)
908
+ return;
909
+ const model = runnableModelName(agent.modelName, await this._runnableHost());
910
+ const rows = await this._harness.list?.({ cachedOnly: true }).catch(noop);
911
+ const busy = [...this._runs.values()].some((run) => run.executing && !run.borrowed && run.model === model);
912
+ if (busy || !rows?.some((row) => row.name === model && row.isLoaded)) {
913
+ return;
914
+ }
915
+ const config = await this._engine.getConfig({}).catch(noop);
916
+ const knobs = withConfigDefaults(agent, config);
917
+ const { loaded, reason } = await this._harness.loadModel({ model, agent: knobs });
918
+ this._log?.info('models', 'model reload requested', { model, loaded, reason });
919
+ }
920
+ _cancelModelReloads() {
921
+ for (const timer of this._reloads.values()) {
922
+ clearTimeout(timer);
923
+ }
924
+ this._reloads.clear();
925
+ }
854
926
  async _agentRunning(model, activeAgentId) {
855
927
  const deviceId = this._requireDeviceId();
856
928
  const meshId = this._accountMeshId;
@@ -1073,21 +1145,26 @@ export class QvacAssistant extends ReadyResource {
1073
1145
  .forget(chatId)
1074
1146
  .catch((err) => this._log?.warn('assistant', 'inspector records not dropped', { err }));
1075
1147
  }
1076
- async _reclaimRekeyedAgents(meshId, frames) {
1148
+ async _followAgentEdits(meshId, frames) {
1077
1149
  let known = null;
1078
1150
  for await (const frame of frames) {
1079
1151
  if (this.closing || this.suspended)
1080
1152
  return;
1081
1153
  const agents = frame.agents ?? [];
1082
1154
  this._followHosted(meshId, agents);
1083
- const next = new Map(agents.map((agent) => [agent.id, agentRekeyFingerprint(agent)]));
1084
1155
  for (const agent of agents) {
1085
1156
  const previous = known?.get(agent.id);
1086
- if (previous !== undefined && previous !== next.get(agent.id)) {
1157
+ if (!previous) {
1158
+ continue;
1159
+ }
1160
+ if (agentRekeyFingerprint(previous) !== agentRekeyFingerprint(agent)) {
1087
1161
  void this._reclaimAgentChatCaches(meshId, agent);
1088
1162
  }
1163
+ if (!sameLoadSettings(previous, agent)) {
1164
+ this._scheduleModelReload(agent);
1165
+ }
1089
1166
  }
1090
- known = next;
1167
+ known = new Map(agents.map((agent) => [agent.id, agent]));
1091
1168
  }
1092
1169
  }
1093
1170
  async _reclaimAgentChatCaches(meshId, agent) {
@@ -1300,7 +1377,7 @@ export class QvacAssistant extends ReadyResource {
1300
1377
  const watches = [
1301
1378
  this._streamWatch(`run-requests:${meshId}`, () => this._engine.runRequestsWatch({ meshId }), (frames) => this._dispatchRunRequests(meshId, frames)),
1302
1379
  this._streamWatch(`chat-list:${meshId}`, () => this._engine.chatsWatch({ meshId }), (frames) => this._reclaimDeletedChats(frames)),
1303
- this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._reclaimRekeyedAgents(meshId, frames))
1380
+ this._streamWatch(`agent:${meshId}`, () => this._engine.agentsWatch({ meshId }), (frames) => this._followAgentEdits(meshId, frames))
1304
1381
  ];
1305
1382
  return {
1306
1383
  start: () => watches.forEach((watch) => watch.start()),
@@ -1612,7 +1689,7 @@ export class QvacAssistant extends ReadyResource {
1612
1689
  model,
1613
1690
  log: this._log
1614
1691
  });
1615
- await this._compactIfNearFull({
1692
+ const folded = await this._compactIfNearFull({
1616
1693
  chatId,
1617
1694
  transcript,
1618
1695
  trigger: 'before-run',
@@ -1672,11 +1749,10 @@ export class QvacAssistant extends ReadyResource {
1672
1749
  }
1673
1750
  : configuredAgent;
1674
1751
  const sendsFiles = lender === null;
1675
- const seesImages = sendsFiles &&
1676
- (this._harness.supportsImagesFor
1677
- ? this._harness.supportsImagesFor(model)
1678
- : (this._harness.supportsImages ?? this._info.supportsImages) !== false);
1752
+ const carriesMedia = messages.some((turn) => turn.images?.length || turn.pdfs?.length || turn.videos?.length);
1753
+ const seesImages = sendsFiles && carriesMedia && (await this._seesImages(model));
1679
1754
  let unreadableImages = new Map();
1755
+ const sampled = [];
1680
1756
  let renderedPdfIds = new Set();
1681
1757
  if (!seesImages) {
1682
1758
  const dropped = messages.reduce((count, turn) => count + (turn.images?.length ?? 0), 0);
@@ -1868,11 +1944,21 @@ export class QvacAssistant extends ReadyResource {
1868
1944
  const injected = knowledge
1869
1945
  ? await this._injectKnowledge(knowledge, { transcript, sourceTagFilter, retrievedRecords })
1870
1946
  : { documentsInjected: false, sourcesWritten: false, ragInjected: false };
1871
- const hasAudio = messages.some((turn) => turn.audioAttachments?.length);
1947
+ const hasAudio = messages.some((turn) => turn.audioAttachments?.length || turn.videos?.length);
1872
1948
  if (hasAudio) {
1873
1949
  modelWatch.transcribe(true);
1874
1950
  }
1875
1951
  await this._hydrateAudio(messages, controller.signal);
1952
+ if (messages.some((turn) => turn.videos?.length)) {
1953
+ await this._hydrateVideos(messages, {
1954
+ signal: controller.signal,
1955
+ runId,
1956
+ seesImages,
1957
+ lent: !sendsFiles,
1958
+ reading: await this._contextReading({ chatId, meshId, folded }, configuredAgent.ctxSize),
1959
+ sampled
1960
+ });
1961
+ }
1876
1962
  if (hasAudio) {
1877
1963
  modelWatch.transcribe(false);
1878
1964
  }
@@ -2079,6 +2165,7 @@ export class QvacAssistant extends ReadyResource {
2079
2165
  metrics: citedTitles ? { ...metrics, citedTitles } : metrics
2080
2166
  });
2081
2167
  titleOutcome = 'completed';
2168
+ await this._storeVideoDigests(sampled, { meshId, chatId, runId, deviceId });
2082
2169
  }
2083
2170
  catch (err) {
2084
2171
  const outcome = controller.signal.aborted
@@ -2336,9 +2423,18 @@ export class QvacAssistant extends ReadyResource {
2336
2423
  return null;
2337
2424
  }
2338
2425
  }
2339
- async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId }) {
2426
+ async _uploadGenerated({ fileName, mimeType, bytes, raw }, { meshId, chatId, runId, deviceId, sourceId }) {
2340
2427
  const duplex = this._engine.uploadAttachment();
2341
- duplex.write({ chatId, runId, originDeviceId: deviceId, fileName, mimeType, raw, meshId });
2428
+ duplex.write({
2429
+ chatId,
2430
+ runId,
2431
+ originDeviceId: deviceId,
2432
+ fileName,
2433
+ mimeType,
2434
+ raw,
2435
+ meshId,
2436
+ sourceId
2437
+ });
2342
2438
  const buf = b4a.toBuffer(bytes);
2343
2439
  for (let i = 0; i < buf.length; i += UPLOAD_FRAME_BYTES) {
2344
2440
  duplex.write({ data: buf.subarray(i, i + UPLOAD_FRAME_BYTES) });
@@ -2404,6 +2500,259 @@ export class QvacAssistant extends ReadyResource {
2404
2500
  return unheardAudioNotice(label, 'transcription failed on this device');
2405
2501
  }
2406
2502
  }
2503
+ async _seesImages(model) {
2504
+ if (this._harness.supportsImagesFor) {
2505
+ return this._harness.supportsImagesFor(model);
2506
+ }
2507
+ const anyModel = (this._harness.supportsImages ?? this._info.supportsImages) !== false;
2508
+ if (!anyModel || !this._harness.list) {
2509
+ return anyModel;
2510
+ }
2511
+ const rows = await this._harness.list({ type: 'llm' }).catch(noop);
2512
+ if (!rows) {
2513
+ return anyModel;
2514
+ }
2515
+ const row = rows.find((candidate) => candidate.name === model);
2516
+ return row ? (row.supportsImages ?? anyModel) : false;
2517
+ }
2518
+ async _hydrateVideos(messages, run) {
2519
+ const { signal, runId, seesImages, lent, reading, sampled } = run;
2520
+ const turns = messages.filter((turn) => turn.videos?.length);
2521
+ if (!turns.length) {
2522
+ return;
2523
+ }
2524
+ const asked = messages.findLast((turn) => turn.role === 'user')?.videos ?? [];
2525
+ const dir = this._imageCacheDir;
2526
+ await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
2527
+ const pinned = this._runImageCacheNames.get(runId) ?? new Set();
2528
+ this._runImageCacheNames.set(runId, pinned);
2529
+ const unsampled = turns.flatMap((turn) => turn.videos ?? []).filter((ref) => !ref.digest);
2530
+ const budget = { ...reading, videos: unsampled.length };
2531
+ for (const turn of turns) {
2532
+ for (const ref of turn.videos ?? []) {
2533
+ if (signal.aborted)
2534
+ return;
2535
+ const hydration = { dir, ref, pinned, seesImages, lent, signal, budget, sampled };
2536
+ const blocks = await this._hydrateVideo(turn, hydration);
2537
+ turn.content = [turn.content.trim(), ...blocks].filter(Boolean).join('\n\n');
2538
+ }
2539
+ delete turn.videos;
2540
+ }
2541
+ const pictured = asked.some((ref) => ref.digest || this._videoProbes.get(ref.blobId)?.hasVideo);
2542
+ if (!seesImages && !lent && pictured) {
2543
+ throw new Error(RUN_FAILURE_REASONS.videoNeedsVision);
2544
+ }
2545
+ }
2546
+ async _hydrateVideo(turn, hydration) {
2547
+ const { ref, seesImages, lent, signal } = hydration;
2548
+ const source = { opened: null };
2549
+ const open = () => (source.opened ??= this._openVideo(hydration));
2550
+ try {
2551
+ const video = await this._readVideo(hydration, open);
2552
+ if (video === 'unsynced') {
2553
+ return [unsyncedVideoNotice(ref.fileName)];
2554
+ }
2555
+ if (video === 'audio') {
2556
+ return [await this._transcribeAttachment(ref, signal)];
2557
+ }
2558
+ if (!seesImages || !('files' in video)) {
2559
+ const notice = lent ? lentVideoNotice(ref.fileName) : hiddenVideoNotice(ref.fileName);
2560
+ return [notice, videoAudioBlock(ref.fileName, video)];
2561
+ }
2562
+ attachPages(turn, video.files);
2563
+ const annotation = videoAnnotation({
2564
+ fileName: ref.fileName,
2565
+ duration: video.duration,
2566
+ times: video.times
2567
+ });
2568
+ return [annotation, videoAudioBlock(ref.fileName, video)];
2569
+ }
2570
+ catch (err) {
2571
+ if (signal.aborted) {
2572
+ throw err;
2573
+ }
2574
+ this._log?.warn('assistant', `video hydration failed — ${ref.fileName}`, err);
2575
+ return [unreadVideoNotice(ref.fileName)];
2576
+ }
2577
+ finally {
2578
+ const video = await source.opened?.catch(noop);
2579
+ if (video) {
2580
+ await fs.promises.unlink(video.file).catch(noop);
2581
+ }
2582
+ }
2583
+ }
2584
+ async _readVideo(hydration, open) {
2585
+ const { ref, seesImages, signal } = hydration;
2586
+ const stored = ref.digest ? await this._storedVideo(ref.digest, hydration) : null;
2587
+ if (stored) {
2588
+ return stored;
2589
+ }
2590
+ const probe = await this._probeVideo(ref, open);
2591
+ if (!probe) {
2592
+ return 'unsynced';
2593
+ }
2594
+ if (!probe.hasVideo) {
2595
+ return 'audio';
2596
+ }
2597
+ if (!seesImages) {
2598
+ return probe.hasAudio ? this._hearVideo(ref, open, signal) : { audio: 'none' };
2599
+ }
2600
+ const video = await open();
2601
+ if (!video) {
2602
+ return 'unsynced';
2603
+ }
2604
+ return this._sampleVideo(video, probe, hydration);
2605
+ }
2606
+ async _probeVideo(ref, open) {
2607
+ const known = this._videoProbes.get(ref.blobId);
2608
+ if (known) {
2609
+ return known;
2610
+ }
2611
+ const video = await open();
2612
+ if (!video) {
2613
+ return null;
2614
+ }
2615
+ const probe = await probeVideo(video.file);
2616
+ this._videoProbes.set(ref.blobId, probe);
2617
+ return probe;
2618
+ }
2619
+ async _storedVideo(digest, { dir, ref, pinned, signal }) {
2620
+ const headerName = videoHeaderCacheName(digest.blobId);
2621
+ pinned.add(headerName);
2622
+ const header = await fs.promises
2623
+ .readFile(path.join(dir, headerName), 'utf8')
2624
+ .then(parseVideoDigestHeader, noop);
2625
+ if (header) {
2626
+ const files = header.times.map((_, i) => {
2627
+ const name = videoFrameCacheName(ref, header.times.length, i);
2628
+ pinned.add(name);
2629
+ return path.join(dir, name);
2630
+ });
2631
+ const cached = await Promise.all(files.map((file) => cacheOrTouch(file, noBytes)));
2632
+ if (cached.every(Boolean)) {
2633
+ return { ...header, files };
2634
+ }
2635
+ }
2636
+ const bytes = await this._readBlob(digest.url, signal);
2637
+ if (!bytes) {
2638
+ return null;
2639
+ }
2640
+ const target = { dir, pinned, digestBlobId: digest.blobId };
2641
+ try {
2642
+ return await this._cacheVideo(ref, unpackVideoDigest(bytes), target);
2643
+ }
2644
+ catch (err) {
2645
+ this._log?.warn('assistant', `video digest unreadable — ${ref.fileName}`, err);
2646
+ return null;
2647
+ }
2648
+ }
2649
+ async _cacheVideo(ref, digest, { dir, pinned, digestBlobId }) {
2650
+ const { frames, ...header } = digest;
2651
+ const files = frames.map((_, i) => {
2652
+ const name = videoFrameCacheName(ref, frames.length, i);
2653
+ pinned.add(name);
2654
+ return path.join(dir, name);
2655
+ });
2656
+ await Promise.all(frames.map((frame, i) => cacheOrTouch(files[i], () => Promise.resolve(frame))));
2657
+ if (digestBlobId) {
2658
+ const headerName = videoHeaderCacheName(digestBlobId);
2659
+ pinned.add(headerName);
2660
+ await cacheOrTouch(path.join(dir, headerName), () => Promise.resolve(b4a.from(JSON.stringify(header))));
2661
+ }
2662
+ return { ...header, files };
2663
+ }
2664
+ async _sampleVideo(video, probe, hydration) {
2665
+ const { ref, budget, signal } = hydration;
2666
+ const started = Date.now();
2667
+ const count = videoFrameCount(probe, budget);
2668
+ const times = frameTimes(probe.duration, count);
2669
+ const frames = await sampleFrames(video.file, { times, maxEdge: VIDEO_FRAME_EDGE, probe });
2670
+ const sampleMs = Date.now() - started;
2671
+ const heard = probe.hasAudio
2672
+ ? await this._hearVideo(ref, () => Promise.resolve(video), signal)
2673
+ : { audio: 'none' };
2674
+ const digest = {
2675
+ duration: probe.duration,
2676
+ times: frames.map((frame) => frame.time),
2677
+ ...heard,
2678
+ frames: frames.map((frame) => frame.jpeg)
2679
+ };
2680
+ if (!hydration.ref.digest) {
2681
+ hydration.sampled.push({ ref, digest });
2682
+ }
2683
+ this._log?.info('assistant', 'video frames', {
2684
+ fileName: ref.fileName,
2685
+ duration: probe.duration,
2686
+ frames: frames.length,
2687
+ budget: count,
2688
+ ctxSize: budget.ctxSize ?? null,
2689
+ cacheTokens: budget.cacheTokens ?? null,
2690
+ audio: heard.audio,
2691
+ sampleMs
2692
+ });
2693
+ return this._cacheVideo(ref, digest, hydration);
2694
+ }
2695
+ async _storeVideoDigests(sampled, target) {
2696
+ const dir = this._imageCacheDir;
2697
+ const pinned = this._runImageCacheNames.get(target.runId) ?? new Set();
2698
+ for (const { ref, digest } of sampled) {
2699
+ try {
2700
+ const blobId = await this._uploadVideoDigest(ref, digest, target);
2701
+ await this._cacheVideo(ref, digest, { dir, pinned, digestBlobId: blobId });
2702
+ }
2703
+ catch (err) {
2704
+ this._log?.warn('assistant', `video digest not stored — ${ref.fileName}`, err);
2705
+ }
2706
+ }
2707
+ }
2708
+ async _uploadVideoDigest(ref, digest, target) {
2709
+ const attachment = await this._uploadGenerated({
2710
+ fileName: videoDigestFileName(ref.fileName),
2711
+ mimeType: VIDEO_DIGEST_MIME,
2712
+ bytes: packVideoDigest(digest),
2713
+ raw: true
2714
+ }, { ...target, sourceId: ref.id });
2715
+ return attachment.blobId;
2716
+ }
2717
+ async _openVideo({ dir, ref, pinned, signal }) {
2718
+ const bytes = await this._readBlob(ref.url, signal);
2719
+ if (!bytes) {
2720
+ return null;
2721
+ }
2722
+ const name = `${imageCacheName(ref)}.source`;
2723
+ pinned.add(name);
2724
+ const file = path.join(dir, name);
2725
+ await fs.promises.writeFile(file, bytes, { mode: 0o600 });
2726
+ return { bytes, file };
2727
+ }
2728
+ async _hearVideo(ref, open, signal) {
2729
+ const cached = this._audioTranscripts.get(ref.blobId);
2730
+ if (cached !== undefined) {
2731
+ return cached ? { audio: 'speech', transcript: cached } : { audio: 'silent' };
2732
+ }
2733
+ if (!this._harness.transcribe) {
2734
+ return { audio: 'unheard' };
2735
+ }
2736
+ try {
2737
+ const video = await open();
2738
+ if (!video) {
2739
+ return { audio: 'unheard' };
2740
+ }
2741
+ const model = mediaModelNameFor('transcription', await this._persistedProfile());
2742
+ await this.ensureMediaModelUnpaused('transcription', model);
2743
+ await this._downloads?.waitUntilCached(model, signal);
2744
+ const transcript = (await this._harness.transcribe({ model: '', audio: b4a.toBuffer(video.bytes), signal })).trim();
2745
+ this._audioTranscripts.set(ref.blobId, transcript);
2746
+ return transcript ? { audio: 'speech', transcript } : { audio: 'silent' };
2747
+ }
2748
+ catch (err) {
2749
+ if (signal.aborted) {
2750
+ throw err;
2751
+ }
2752
+ this._log?.warn('assistant', `video transcription failed — ${ref.fileName}`, err);
2753
+ return { audio: 'unheard' };
2754
+ }
2755
+ }
2407
2756
  async _hydrateImages(messages, signal, runId) {
2408
2757
  const unreadable = new Map();
2409
2758
  const turns = messages.filter((turn) => turn.images?.length);
@@ -2633,6 +2982,16 @@ export class QvacAssistant extends ReadyResource {
2633
2982
  }
2634
2983
  }
2635
2984
  }
2985
+ async _contextReading({ chatId, meshId, folded }, ctxSize) {
2986
+ const last = folded ? {} : await this._lastRunMetrics(chatId, meshId);
2987
+ const window = ctxSize || last.ctxSize;
2988
+ if (!window) {
2989
+ return {};
2990
+ }
2991
+ return last.ctxSize === window
2992
+ ? { ctxSize: window, cacheTokens: last.cacheTokens }
2993
+ : { ctxSize: window };
2994
+ }
2636
2995
  async _lastRunMetrics(chatId, meshId) {
2637
2996
  const deviceId = this._requireDeviceId();
2638
2997
  const { chunks } = await meshScoped(this._engine, meshId)
@@ -166,6 +166,7 @@ async function forkedChunk(ctx, chunk) {
166
166
  chatId: ctx.chatId,
167
167
  runId: chunk.runId && forkedId(ctx, chunk.runId),
168
168
  parentId: chunk.parentId && ctx.kept.has(chunk.parentId) ? forkedId(ctx, chunk.parentId) : null,
169
+ sourceId: chunk.sourceId && forkedId(ctx, chunk.sourceId),
169
170
  compaction: chunk.compaction && {
170
171
  ...chunk.compaction,
171
172
  throughId: forkedId(ctx, chunk.compaction.throughId)
@@ -177,7 +177,7 @@ function matchAttachments(chunks, needle, cap) {
177
177
  if (matches.length >= cap)
178
178
  break;
179
179
  const attachment = chunk.attachment;
180
- if (!attachment || (attachment.deletedAt ?? 0) > 0) {
180
+ if (!attachment || (attachment.deletedAt ?? 0) > 0 || chunk.sourceId) {
181
181
  continue;
182
182
  }
183
183
  const found = locate(attachment.fileName, needle);
@@ -3,7 +3,7 @@ import type { QvacChunk, QvacUpdateChunkOperation } from '../../spec/mesh/hypers
3
3
  import type { Mesh, MeshDispatcher } from './mesh.ts';
4
4
  import { type Bounds } from './range.ts';
5
5
  type ChunkPayload = Pick<QvacChunk, 'userMessage' | 'content' | 'thinking' | 'toolCall' | 'toolResult' | 'attachment' | 'approvalRequest' | 'runStatus' | 'runRequest' | 'sources' | 'compaction'>;
6
- type AppendChunkInput = Pick<QvacChunk, 'chatId' | 'runId' | 'type' | 'originDeviceId' | 'parentId' | 'effect' | 'intent' | 'display' | 'rawCall' | 'capability' | 'capabilityArgs' | 'enhance'> & ChunkPayload & {
6
+ type AppendChunkInput = Pick<QvacChunk, 'chatId' | 'runId' | 'type' | 'originDeviceId' | 'parentId' | 'effect' | 'intent' | 'display' | 'rawCall' | 'capability' | 'capabilityArgs' | 'enhance' | 'sourceId'> & ChunkPayload & {
7
7
  id?: string;
8
8
  };
9
9
  type UpdateChunkInput = Pick<QvacChunk, 'id' | 'runStatus' | 'attachment'>;
@@ -40,7 +40,7 @@ export declare function getChunk(store: Mesh, id: string): Promise<QvacChunk | n
40
40
  export interface AppendedChunk {
41
41
  readonly id: string;
42
42
  }
43
- export declare function appendChunk(store: Mesh, { id, chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, ...payload }: AppendChunkInput, into?: MeshDispatcher): Promise<AppendedChunk>;
43
+ export declare function appendChunk(store: Mesh, { id, chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, sourceId, ...payload }: AppendChunkInput, into?: MeshDispatcher): Promise<AppendedChunk>;
44
44
  export declare function updateChunk(store: Mesh, { id, runStatus, attachment }: UpdateChunkInput): Promise<QvacChunk>;
45
45
  export declare function updateOp(chunk: QvacChunk, { id, runStatus, attachment }: UpdateChunkInput): QvacUpdateChunkOperation;
46
46
  export interface ResolveApprovalInput {
@@ -118,7 +118,7 @@ function assertPayloadMatchesType(type, payload) {
118
118
  throw new CoreError('CHUNK_PAYLOAD_INVALID', [`${type} carries ${field} and nothing else`]);
119
119
  }
120
120
  }
121
- export async function appendChunk(store, { id = randomId(), chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, ...payload }, into = store) {
121
+ export async function appendChunk(store, { id = randomId(), chatId, runId, type, originDeviceId, parentId, effect, intent, display, rawCall, capability, capabilityArgs, enhance, sourceId, ...payload }, into = store) {
122
122
  assertPayloadMatchesType(type, payload);
123
123
  if (type === 'sources' && !runId) {
124
124
  throw new CoreError('CHUNK_PAYLOAD_INVALID', ['a sources chunk requires runId']);
@@ -129,6 +129,9 @@ export async function appendChunk(store, { id = randomId(), chatId, runId, type,
129
129
  if (capability && type !== 'run-request') {
130
130
  throw new CoreError('CHUNK_PAYLOAD_INVALID', ['only a run-request carries a capability']);
131
131
  }
132
+ if (sourceId && type !== 'attachment') {
133
+ throw new CoreError('CHUNK_PAYLOAD_INVALID', ['only an attachment is derived from another']);
134
+ }
132
135
  if (type === 'user-message' && !store.ourMemberId) {
133
136
  await store.refreshOurMember();
134
137
  }
@@ -154,6 +157,7 @@ export async function appendChunk(store, { id = randomId(), chatId, runId, type,
154
157
  capability,
155
158
  capabilityArgs,
156
159
  enhance,
160
+ sourceId,
157
161
  createdAt: Date.now(),
158
162
  senderMemberId: type === 'user-message' && store.ourDeviceId && b4a.equals(store.ourDeviceId, originDeviceId)
159
163
  ? store.ourMemberId || undefined
@@ -1,9 +1,12 @@
1
1
  import { isAudioAttachment } from './audio-attachment.js';
2
2
  import { classifyRagDocument, RAG_DOCUMENT_KIND } from './files/rag/index.js';
3
3
  import { IMAGE_MIMES } from './image/normalize.js';
4
+ import { isVideoAttachment } from './video/attachment.js';
4
5
  export function isDocumentAttachment(ref) {
5
6
  if (IMAGE_MIMES.has(ref.mimeType.toLowerCase()))
6
7
  return false;
8
+ if (isVideoAttachment(ref.fileName, ref.mimeType))
9
+ return false;
7
10
  return !isAudioAttachment(ref.fileName, ref.mimeType);
8
11
  }
9
12
  export function isPdfAttachment(ref) {
@@ -4,7 +4,7 @@ import { attachmentFiles } from './attachments.js';
4
4
  import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, reclaimPurgedFiles, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
5
5
  import { knowledgeFiles } from './knowledge.js';
6
6
  import { randomId } from '../id.js';
7
- import { getChunk, listChunks } from '../chunk.js';
7
+ import { findChunksOfType, getChunk, listChunks } from '../chunk.js';
8
8
  import { settleFiles, isClaimLive, listIndexingByChat, FileState, FileProcessing } from './processing/index.js';
9
9
  import { DeadlineScheduler } from '../../deadline.js';
10
10
  import { liveDeviceForWriter } from '../contract/activity.js';
@@ -14,7 +14,7 @@ import { RagProcessor } from './rag/index.js';
14
14
  import { PreviewProcessor, PREVIEW_VERSION, previewId } from './preview/index.js';
15
15
  import { CoreError } from '../../errors.js';
16
16
  import { errorMessage } from '../../guards.js';
17
- import { ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS, CHUNKS_BY_CHAT, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE } from '../collections.js';
17
+ import { ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE } from '../collections.js';
18
18
  export { fullTextBudgetChars, fullTextHits, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
19
19
  export { isKnowledgeRoutedAttachment } from './records/index.js';
20
20
  export { assertRemovableAt } from './records/index.js';
@@ -457,13 +457,13 @@ export class Files extends EventEmitter {
457
457
  }
458
458
  const agent = await snap.get(AGENTS, { id: chat.agentId });
459
459
  const knowledge = await snap.find(ACTIVE_KNOWLEDGE_BY_CHAT, range).toArray();
460
- const chunks = await snap.find(CHUNKS_BY_CHAT, range).toArray();
460
+ const chunks = await findChunksOfType(snap, { chatId, type: 'attachment' });
461
461
  const statuses = await snap.find(INDEXING_STATUS_BY_CHAT, range).toArray();
462
462
  const sources = new Map(knowledge.map((row) => [row.id, row]));
463
463
  const ragIds = new Set(sources.keys());
464
464
  for (const chunk of chunks) {
465
465
  const file = chunk.attachment;
466
- if (!file || file.deletedAt || sources.has(file.id)) {
466
+ if (!file || file.deletedAt || chunk.sourceId || sources.has(file.id)) {
467
467
  continue;
468
468
  }
469
469
  sources.set(file.id, { ...file, chatId, updatedAt: file.createdAt });