@qvac/core 0.10.3 → 0.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +37 -1
  2. package/dist-lib/lib/assistant/history.js +18 -2
  3. package/dist-lib/lib/assistant/message.d.ts +17 -0
  4. package/dist-lib/lib/assistant/message.js +28 -0
  5. package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
  6. package/dist-lib/lib/assistant/video-budget.js +29 -0
  7. package/dist-lib/lib/assistant.d.ts +22 -2
  8. package/dist-lib/lib/assistant.js +374 -15
  9. package/dist-lib/lib/core/chat-fork.js +1 -0
  10. package/dist-lib/lib/core/chat-search.js +1 -1
  11. package/dist-lib/lib/core/chunk.d.ts +2 -2
  12. package/dist-lib/lib/core/chunk.js +5 -1
  13. package/dist-lib/lib/core/document-attachment.js +3 -0
  14. package/dist-lib/lib/core/files/index.js +4 -4
  15. package/dist-lib/lib/core/files/records/chunks.js +21 -0
  16. package/dist-lib/lib/core/files/records/purge.js +4 -3
  17. package/dist-lib/lib/core/files/records/source.js +3 -5
  18. package/dist-lib/lib/core/files/upload.js +3 -2
  19. package/dist-lib/lib/core/video/attachment.d.ts +1 -0
  20. package/dist-lib/lib/core/video/attachment.js +7 -0
  21. package/dist-lib/lib/core/video/digest.d.ts +16 -0
  22. package/dist-lib/lib/core/video/digest.js +57 -0
  23. package/dist-lib/lib/core/video/frames.d.ts +32 -0
  24. package/dist-lib/lib/core/video/frames.js +224 -0
  25. package/dist-lib/lib/core.js +19 -1
  26. package/dist-lib/lib/credential-pusher.d.ts +5 -0
  27. package/dist-lib/lib/credential-pusher.js +42 -11
  28. package/dist-lib/lib/run-notices.d.ts +1 -0
  29. package/dist-lib/lib/run-notices.js +2 -1
  30. package/dist-lib/lib/task.js +1 -1
  31. package/dist-lib/schema/engine.js +4 -2
  32. package/dist-lib/schema/indexes.js +2 -2
  33. package/dist-lib/schema/qvac.js +4 -2
  34. package/dist-lib/spec/hrpc/messages.js +16 -6
  35. package/dist-lib/spec/hyperschema/index.js +16 -6
  36. package/dist-lib/spec/hyperschema/schema.json +10 -0
  37. package/dist-lib/spec/hyperschema/types.d.ts +2 -0
  38. package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
  39. package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
  40. package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
  41. package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
  42. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  43. package/package.json +3 -1
  44. package/tui/app.mjs +34 -1
  45. package/tui/lib/render.mjs +5 -0
  46. package/tui/main.mjs +8 -0
package/CHANGELOG.md CHANGED
@@ -13,6 +13,41 @@ release.
13
13
 
14
14
  ## [Unreleased]
15
15
 
16
+ ## [0.10.4] - 2026-10-22
17
+
18
+ ### Added
19
+
20
+ - A video attachment reaches the chat's model as up to 32 frames, sized to the
21
+ context window, with their times and a transcript of its audio track. The
22
+ first run stores them as an attachment chunk carrying `sourceId`, the video's
23
+ attachment id, which later turns and other devices read instead of the video;
24
+ an app should not render a chunk with `sourceId`. `uploadAttachment` takes
25
+ `sourceId` on its first frame. Core now depends on `bare-ffmpeg`. See
26
+ [attachments](docs/api/attachments.md) and [chunks](docs/api/chunks.md).
27
+ (#496)
28
+ - A `ctxSize`, `kvCacheK` or `kvCacheV` change reloads the agent's model on its
29
+ host 2.5 seconds after the last edit, if it is loaded, instead of on the
30
+ agent's next message. (#495)
31
+
32
+ ### Changed
33
+
34
+ - `fileStatusWatch` reads only the chat's attachments, not every message in the
35
+ chat, on each refresh. (#518)
36
+
37
+ ### Fixed
38
+
39
+ - A device that unpairs itself hands the agents it hosted to its member's
40
+ earliest-paired live device, in the account and every shared mesh, instead of
41
+ leaving them orphaned. A leave from an earlier release is handed over on the
42
+ next boot. (#514)
43
+ - On a harness in its own process, an image sent to a model without a projector
44
+ is no longer dropped silently: the run asks the harness about the chat's
45
+ model, and the turn says the image is not visible to it. (#494)
46
+ - A token saved for an agent reaches `skills` and `skillCatalog` readiness as
47
+ soon as it is written, instead of after the ten-minute credential refresh.
48
+ (#523)
49
+ - `duck` reopens the meshes it had saved when it starts. (#519)
50
+
16
51
  ## [0.10.3] - 2026-10-21
17
52
 
18
53
  ### Added
@@ -1857,7 +1892,8 @@ Published under the `release-0.1` dist-tag.
1857
1892
  await assistant.ready()
1858
1893
  ```
1859
1894
 
1860
- [Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.10.3...HEAD
1895
+ [Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.10.4...HEAD
1896
+ [0.10.4]: https://github.com/tetherto/qvac-core/compare/v0.10.3...v0.10.4
1861
1897
  [0.10.3]: https://github.com/tetherto/qvac-core/compare/v0.10.2...v0.10.3
1862
1898
  [0.10.2]: https://github.com/tetherto/qvac-core/compare/v0.10.1...v0.10.2
1863
1899
  [0.10.1]: https://github.com/tetherto/qvac-core/compare/v0.10.0...v0.10.1
@@ -4,6 +4,7 @@ import { isAudioAttachment } from '../core/audio-attachment.js';
4
4
  import { isDocumentAttachment, isPdfAttachment } from '../core/document-attachment.js';
5
5
  import { groupAttachmentsByMessage } from '../core/files/index.js';
6
6
  import { IMAGE_MIMES } from '../core/image/normalize.js';
7
+ import { isVideoAttachment } from '../core/video/attachment.js';
7
8
  import { compactionTurn, foldCompactedChunks } from '../compaction.js';
8
9
  import b4a from 'b4a';
9
10
  import { isRecord } from '../guards.js';
@@ -76,6 +77,16 @@ export function senderAttribution(names) {
76
77
  const distinct = [...new Set(names.values())].sort();
77
78
  return distinct.length > 1 ? distinct.join('\u0000') : '';
78
79
  }
80
+ function derivedAttachments(chunks) {
81
+ const derived = new Map();
82
+ for (const chunk of chunks) {
83
+ const { sourceId, attachment } = chunk;
84
+ if (sourceId && attachment && !attachment.deletedAt && !derived.has(sourceId)) {
85
+ derived.set(sourceId, attachment);
86
+ }
87
+ }
88
+ return derived;
89
+ }
79
90
  export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(), sentRuns = new Map() } = {}) {
80
91
  const attemptStarts = latestAttemptStarts(chunks);
81
92
  const shared = new Set(senderNames.values()).size > 1;
@@ -86,7 +97,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
86
97
  const attachments = [];
87
98
  const userMessages = [];
88
99
  for (const chunk of source) {
89
- if (chunk.type === 'attachment') {
100
+ if (chunk.type === 'attachment' && !chunk.sourceId) {
90
101
  attachments.push(chunk);
91
102
  }
92
103
  else if (chunk.userMessage) {
@@ -94,6 +105,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
94
105
  }
95
106
  }
96
107
  const byMessage = groupAttachmentsByMessage(attachments, userMessages);
108
+ const digests = derivedAttachments(chunks);
97
109
  const history = [];
98
110
  const replayedSkills = new Set();
99
111
  if (folded.summary) {
@@ -132,7 +144,8 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
132
144
  .flatMap((attachmentChunk) => attachmentChunk.attachment ?? [])
133
145
  .filter((ref) => !ref.deletedAt);
134
146
  const images = refs.filter((ref) => IMAGE_MIMES.has(ref.mimeType.toLowerCase()));
135
- const audio = refs.filter((ref) => isAudioAttachment(ref.fileName, ref.mimeType));
147
+ const videos = refs.filter((ref) => isVideoAttachment(ref.fileName, ref.mimeType));
148
+ const audio = refs.filter((ref) => isAudioAttachment(ref.fileName, ref.mimeType) && !videos.includes(ref));
136
149
  const documents = refs.filter((ref) => isDocumentAttachment(ref));
137
150
  const pdfs = documents.filter(isPdfAttachment);
138
151
  if (images.length) {
@@ -141,6 +154,9 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
141
154
  if (audio.length) {
142
155
  turn.audioAttachments = audio;
143
156
  }
157
+ if (videos.length) {
158
+ turn.videos = videos.map((ref) => ({ ...ref, digest: digests.get(ref.id) }));
159
+ }
144
160
  if (documents.length) {
145
161
  turn.documents = documents;
146
162
  }
@@ -8,6 +8,7 @@ export interface ChatMessage {
8
8
  }[];
9
9
  images?: AttachmentRef[];
10
10
  audioAttachments?: AttachmentRef[];
11
+ videos?: VideoRef[];
11
12
  documents?: DocumentAttachment[];
12
13
  pdfs?: AttachmentRef[];
13
14
  }
@@ -24,6 +25,19 @@ export interface UnheardAudioNoticeOptions {
24
25
  export declare function unheardAudioNotice(label: string, reason: string, opts?: UnheardAudioNoticeOptions): string;
25
26
  export declare function silentAudioNotice(fileName: string): string;
26
27
  export declare function audioTranscriptBlock(fileName: string, transcript: string): string;
28
+ export interface VideoFrames {
29
+ readonly fileName: string;
30
+ readonly duration: number;
31
+ readonly times: ReadonlyArray<number>;
32
+ }
33
+ export declare function videoAnnotation({ fileName, duration, times }: VideoFrames): string;
34
+ export declare function videoTranscriptBlock(transcript: string): string;
35
+ export declare function unsyncedVideoNotice(fileName: string): string;
36
+ export declare function unreadVideoNotice(fileName: string): string;
37
+ export declare function lentVideoNotice(fileName: string): string;
38
+ export declare function hiddenVideoNotice(fileName: string): string;
39
+ export declare const VIDEO_NO_AUDIO = "[The video has no audio track.]";
40
+ export declare const VIDEO_SILENT = "[The video's audio contained no detectable speech.]";
27
41
  export interface AttachmentRef {
28
42
  id: string;
29
43
  fileName: string;
@@ -31,6 +45,9 @@ export interface AttachmentRef {
31
45
  blobId: string;
32
46
  url?: string | null;
33
47
  }
48
+ export interface VideoRef extends AttachmentRef {
49
+ digest?: AttachmentRef;
50
+ }
34
51
  export interface AttachInput {
35
52
  fileName: string;
36
53
  mimeType: string;
@@ -47,6 +47,34 @@ export function audioTranscriptBlock(fileName, transcript) {
47
47
  transcript
48
48
  ].join('\n');
49
49
  }
50
+ function clock(seconds) {
51
+ const s = Math.round(seconds);
52
+ return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`;
53
+ }
54
+ export function videoAnnotation({ fileName, duration, times }) {
55
+ return (`[Video "${fileName}", ${clock(duration)} long. You are seeing ${times.length} frames taken ` +
56
+ `evenly across the whole video, in order, at ${times.map(clock).join(', ')}. You cannot see ` +
57
+ 'what happens between frames.]');
58
+ }
59
+ export function videoTranscriptBlock(transcript) {
60
+ return `The video's audio was transcribed on-device. Transcript:\n${transcript}`;
61
+ }
62
+ export function unsyncedVideoNotice(fileName) {
63
+ return `[Attached video "${fileName}" has not reached this device yet — the upload may still be syncing.]`;
64
+ }
65
+ export function unreadVideoNotice(fileName) {
66
+ return (`[Attached video "${fileName}" could not be read on this device: the file may be damaged, ` +
67
+ 'or in a format it cannot decode. Tell the user so; do not try to open it another way.]');
68
+ }
69
+ export function lentVideoNotice(fileName) {
70
+ return (`[Attached video "${fileName}" — its frames cannot be sent to a model lent by another ` +
71
+ 'device; only its audio is below. Tell the user if they ask what it shows.]');
72
+ }
73
+ export function hiddenVideoNotice(fileName) {
74
+ return `[Attached video "${fileName}" — its frames are not visible to this model]`;
75
+ }
76
+ export const VIDEO_NO_AUDIO = '[The video has no audio track.]';
77
+ export const VIDEO_SILENT = "[The video's audio contained no detectable speech.]";
50
78
  const MAX_SENDER_NAME_CHARS = 64;
51
79
  function displayableSender(name) {
52
80
  const flat = name?.replace(/\s+/g, ' ').trim();
@@ -0,0 +1,13 @@
1
+ import { type Size, type VideoProbe } from '../core/video/frames.ts';
2
+ export declare const VIDEO_FRAME_EDGE = 512;
3
+ export declare const MAX_VIDEO_FRAMES = 32;
4
+ export declare const MIN_VIDEO_FRAMES = 8;
5
+ export interface ContextReading {
6
+ readonly ctxSize?: number;
7
+ readonly cacheTokens?: number;
8
+ }
9
+ export interface FrameBudget extends ContextReading {
10
+ readonly videos: number;
11
+ }
12
+ export declare function frameTokens({ width, height }: Size): number;
13
+ export declare function videoFrameCount(probe: VideoProbe, budget: FrameBudget): number;
@@ -0,0 +1,29 @@
1
+ import { frameSize } from '../core/video/frames.js';
2
+ export const VIDEO_FRAME_EDGE = 512;
3
+ export const MAX_VIDEO_FRAMES = 32;
4
+ export const MIN_VIDEO_FRAMES = 8;
5
+ const ANSWER_SHARE = 0.25;
6
+ const UNREAD_SHARE = 0.5;
7
+ const TILE = 32;
8
+ const FRAME_MARKER_TOKENS = 6;
9
+ export function frameTokens({ width, height }) {
10
+ return Math.ceil(width / TILE) * Math.ceil(height / TILE) + FRAME_MARKER_TOKENS;
11
+ }
12
+ function room({ ctxSize, cacheTokens }) {
13
+ if (!ctxSize) {
14
+ return null;
15
+ }
16
+ if (cacheTokens === undefined) {
17
+ return ctxSize * UNREAD_SHARE;
18
+ }
19
+ return ctxSize * (1 - ANSWER_SHARE) - cacheTokens;
20
+ }
21
+ export function videoFrameCount(probe, budget) {
22
+ const perSecond = Math.max(1, Math.floor(probe.duration));
23
+ const free = room(budget);
24
+ const fits = free === null
25
+ ? MAX_VIDEO_FRAMES
26
+ : Math.floor(free / Math.max(1, budget.videos) / frameTokens(frameSize(probe, VIDEO_FRAME_EDGE)));
27
+ const frames = Math.min(MAX_VIDEO_FRAMES, Math.max(MIN_VIDEO_FRAMES, fits));
28
+ return Math.min(perSecond, frames);
29
+ }
@@ -32,6 +32,7 @@ export interface AssistantOptions {
32
32
  borrowHarness?: (deviceId: string) => Duplex;
33
33
  blobReadTimeoutMs?: number;
34
34
  runIdleTimeoutMs?: number;
35
+ modelReloadDelayMs?: number;
35
36
  pdfRasterizer?: PdfRasterizer;
36
37
  }
37
38
  export interface SkillListOptions extends SkillOptions {
@@ -53,6 +54,8 @@ export declare class QvacAssistant extends ReadyResource {
53
54
  private readonly _drives?;
54
55
  private readonly _blobReadTimeoutMs;
55
56
  private readonly _runIdleTimeoutMs;
57
+ private readonly _modelReloadDelayMs;
58
+ private readonly _reloads;
56
59
  private readonly _pdfRasterizer;
57
60
  private readonly _warmAbort;
58
61
  private readonly _unloading;
@@ -93,6 +96,7 @@ export declare class QvacAssistant extends ReadyResource {
93
96
  private readonly _runReplay;
94
97
  private readonly _lifecycle;
95
98
  private readonly _audioTranscripts;
99
+ private readonly _videoProbes;
96
100
  private readonly _pdfPages;
97
101
  private readonly _chats;
98
102
  private readonly _runImageCacheNames;
@@ -101,7 +105,7 @@ export declare class QvacAssistant extends ReadyResource {
101
105
  private readonly _modelCatalog;
102
106
  private readonly _onHarnessStatus;
103
107
  private readonly _onModelCatalogEvent;
104
- constructor(engine: Capabilities, harness: Harness, { logger, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform, preloadCurrentAgentModel, drives, borrowHarness, blobReadTimeoutMs, runIdleTimeoutMs, pdfRasterizer: pdfRasterizerOption }?: AssistantOptions);
108
+ constructor(engine: Capabilities, harness: Harness, { logger, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform, preloadCurrentAgentModel, drives, borrowHarness, blobReadTimeoutMs, runIdleTimeoutMs, modelReloadDelayMs, pdfRasterizer: pdfRasterizerOption }?: AssistantOptions);
105
109
  private _requireDeviceId;
106
110
  _open(): Promise<void>;
107
111
  private _triggerModelWarm;
@@ -141,6 +145,9 @@ export declare class QvacAssistant extends ReadyResource {
141
145
  modelCatalogWatch(): AsyncIterable<AssistantModelCatalogRevision>;
142
146
  loadModel(name: string): Promise<AssistantLoadModelResponse>;
143
147
  private _isCached;
148
+ private _scheduleModelReload;
149
+ private _reloadLoadedModel;
150
+ private _cancelModelReloads;
144
151
  private _agentRunning;
145
152
  unloadModel(name: string): Promise<AssistantUnloadModelResponse>;
146
153
  private _unloadModel;
@@ -157,7 +164,7 @@ export declare class QvacAssistant extends ReadyResource {
157
164
  private _supervisedWatch;
158
165
  private _reclaimDeletedChats;
159
166
  private _dropDeletedChat;
160
- private _reclaimRekeyedAgents;
167
+ private _followAgentEdits;
161
168
  private _reclaimAgentChatCaches;
162
169
  private _reconcileCompaction;
163
170
  private _senderNames;
@@ -201,6 +208,18 @@ export declare class QvacAssistant extends ReadyResource {
201
208
  private _uploadGenerated;
202
209
  private _hydrateAudio;
203
210
  private _transcribeAttachment;
211
+ private _seesImages;
212
+ private _hydrateVideos;
213
+ private _hydrateVideo;
214
+ private _readVideo;
215
+ private _probeVideo;
216
+ private _storedVideo;
217
+ private _cacheVideo;
218
+ private _sampleVideo;
219
+ private _storeVideoDigests;
220
+ private _uploadVideoDigest;
221
+ private _openVideo;
222
+ private _hearVideo;
204
223
  private _hydrateImages;
205
224
  private _hydrateTurnImages;
206
225
  private _hydrateImage;
@@ -211,6 +230,7 @@ export declare class QvacAssistant extends ReadyResource {
211
230
  private _renderPdfPage;
212
231
  private _compactIfNearFull;
213
232
  private _runCompaction;
233
+ private _contextReading;
214
234
  private _lastRunMetrics;
215
235
  compactChat(chatId: string, options: CompactOptions): Promise<CompactResult>;
216
236
  private _summarize;