@qvac/core 0.10.3 → 0.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -1
- package/dist-lib/lib/assistant/history.js +18 -2
- package/dist-lib/lib/assistant/message.d.ts +17 -0
- package/dist-lib/lib/assistant/message.js +28 -0
- package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
- package/dist-lib/lib/assistant/video-budget.js +29 -0
- package/dist-lib/lib/assistant.d.ts +22 -2
- package/dist-lib/lib/assistant.js +374 -15
- package/dist-lib/lib/core/chat-fork.js +1 -0
- package/dist-lib/lib/core/chat-search.js +1 -1
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +5 -1
- package/dist-lib/lib/core/document-attachment.js +3 -0
- package/dist-lib/lib/core/files/index.js +4 -4
- package/dist-lib/lib/core/files/records/chunks.js +21 -0
- package/dist-lib/lib/core/files/records/purge.js +4 -3
- package/dist-lib/lib/core/files/records/source.js +3 -5
- package/dist-lib/lib/core/files/upload.js +3 -2
- package/dist-lib/lib/core/video/attachment.d.ts +1 -0
- package/dist-lib/lib/core/video/attachment.js +7 -0
- package/dist-lib/lib/core/video/digest.d.ts +16 -0
- package/dist-lib/lib/core/video/digest.js +57 -0
- package/dist-lib/lib/core/video/frames.d.ts +32 -0
- package/dist-lib/lib/core/video/frames.js +224 -0
- package/dist-lib/lib/core.js +19 -1
- package/dist-lib/lib/credential-pusher.d.ts +5 -0
- package/dist-lib/lib/credential-pusher.js +42 -11
- package/dist-lib/lib/run-notices.d.ts +1 -0
- package/dist-lib/lib/run-notices.js +2 -1
- package/dist-lib/lib/task.js +1 -1
- package/dist-lib/schema/engine.js +4 -2
- package/dist-lib/schema/indexes.js +2 -2
- package/dist-lib/schema/qvac.js +4 -2
- package/dist-lib/spec/hrpc/messages.js +16 -6
- package/dist-lib/spec/hyperschema/index.js +16 -6
- package/dist-lib/spec/hyperschema/schema.json +10 -0
- package/dist-lib/spec/hyperschema/types.d.ts +2 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
- package/package.json +3 -1
- package/tui/app.mjs +34 -1
- package/tui/lib/render.mjs +5 -0
- package/tui/main.mjs +8 -0
package/CHANGELOG.md
CHANGED
|
@@ -13,6 +13,41 @@ release.
|
|
|
13
13
|
|
|
14
14
|
## [Unreleased]
|
|
15
15
|
|
|
16
|
+
## [0.10.4] - 2026-10-22
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- A video attachment reaches the chat's model as up to 32 frames, sized to the
|
|
21
|
+
context window, with their times and a transcript of its audio track. The
|
|
22
|
+
first run stores them as an attachment chunk carrying `sourceId`, the video's
|
|
23
|
+
attachment id, which later turns and other devices read instead of the video;
|
|
24
|
+
an app should not render a chunk with `sourceId`. `uploadAttachment` takes
|
|
25
|
+
`sourceId` on its first frame. Core now depends on `bare-ffmpeg`. See
|
|
26
|
+
[attachments](docs/api/attachments.md) and [chunks](docs/api/chunks.md).
|
|
27
|
+
(#496)
|
|
28
|
+
- A `ctxSize`, `kvCacheK` or `kvCacheV` change reloads the agent's model on its
|
|
29
|
+
host 2.5 seconds after the last edit, if it is loaded, instead of on the
|
|
30
|
+
agent's next message. (#495)
|
|
31
|
+
|
|
32
|
+
### Changed
|
|
33
|
+
|
|
34
|
+
- `fileStatusWatch` reads only the chat's attachments, not every message in the
|
|
35
|
+
chat, on each refresh. (#518)
|
|
36
|
+
|
|
37
|
+
### Fixed
|
|
38
|
+
|
|
39
|
+
- A device that unpairs itself hands the agents it hosted to its member's
|
|
40
|
+
earliest-paired live device, in the account and every shared mesh, instead of
|
|
41
|
+
leaving them orphaned. A leave from an earlier release is handed over on the
|
|
42
|
+
next boot. (#514)
|
|
43
|
+
- On a harness in its own process, an image sent to a model without a projector
|
|
44
|
+
is no longer dropped silently: the run asks the harness about the chat's
|
|
45
|
+
model, and the turn says the image is not visible to it. (#494)
|
|
46
|
+
- A token saved for an agent reaches `skills` and `skillCatalog` readiness as
|
|
47
|
+
soon as it is written, instead of after the ten-minute credential refresh.
|
|
48
|
+
(#523)
|
|
49
|
+
- `duck` reopens the meshes it had saved when it starts. (#519)
|
|
50
|
+
|
|
16
51
|
## [0.10.3] - 2026-10-21
|
|
17
52
|
|
|
18
53
|
### Added
|
|
@@ -1857,7 +1892,8 @@ Published under the `release-0.1` dist-tag.
|
|
|
1857
1892
|
await assistant.ready()
|
|
1858
1893
|
```
|
|
1859
1894
|
|
|
1860
|
-
[Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.10.
|
|
1895
|
+
[Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.10.4...HEAD
|
|
1896
|
+
[0.10.4]: https://github.com/tetherto/qvac-core/compare/v0.10.3...v0.10.4
|
|
1861
1897
|
[0.10.3]: https://github.com/tetherto/qvac-core/compare/v0.10.2...v0.10.3
|
|
1862
1898
|
[0.10.2]: https://github.com/tetherto/qvac-core/compare/v0.10.1...v0.10.2
|
|
1863
1899
|
[0.10.1]: https://github.com/tetherto/qvac-core/compare/v0.10.0...v0.10.1
|
|
@@ -4,6 +4,7 @@ import { isAudioAttachment } from '../core/audio-attachment.js';
|
|
|
4
4
|
import { isDocumentAttachment, isPdfAttachment } from '../core/document-attachment.js';
|
|
5
5
|
import { groupAttachmentsByMessage } from '../core/files/index.js';
|
|
6
6
|
import { IMAGE_MIMES } from '../core/image/normalize.js';
|
|
7
|
+
import { isVideoAttachment } from '../core/video/attachment.js';
|
|
7
8
|
import { compactionTurn, foldCompactedChunks } from '../compaction.js';
|
|
8
9
|
import b4a from 'b4a';
|
|
9
10
|
import { isRecord } from '../guards.js';
|
|
@@ -76,6 +77,16 @@ export function senderAttribution(names) {
|
|
|
76
77
|
const distinct = [...new Set(names.values())].sort();
|
|
77
78
|
return distinct.length > 1 ? distinct.join('\u0000') : '';
|
|
78
79
|
}
|
|
80
|
+
function derivedAttachments(chunks) {
|
|
81
|
+
const derived = new Map();
|
|
82
|
+
for (const chunk of chunks) {
|
|
83
|
+
const { sourceId, attachment } = chunk;
|
|
84
|
+
if (sourceId && attachment && !attachment.deletedAt && !derived.has(sourceId)) {
|
|
85
|
+
derived.set(sourceId, attachment);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return derived;
|
|
89
|
+
}
|
|
79
90
|
export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(), sentRuns = new Map() } = {}) {
|
|
80
91
|
const attemptStarts = latestAttemptStarts(chunks);
|
|
81
92
|
const shared = new Set(senderNames.values()).size > 1;
|
|
@@ -86,7 +97,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
|
|
|
86
97
|
const attachments = [];
|
|
87
98
|
const userMessages = [];
|
|
88
99
|
for (const chunk of source) {
|
|
89
|
-
if (chunk.type === 'attachment') {
|
|
100
|
+
if (chunk.type === 'attachment' && !chunk.sourceId) {
|
|
90
101
|
attachments.push(chunk);
|
|
91
102
|
}
|
|
92
103
|
else if (chunk.userMessage) {
|
|
@@ -94,6 +105,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
|
|
|
94
105
|
}
|
|
95
106
|
}
|
|
96
107
|
const byMessage = groupAttachmentsByMessage(attachments, userMessages);
|
|
108
|
+
const digests = derivedAttachments(chunks);
|
|
97
109
|
const history = [];
|
|
98
110
|
const replayedSkills = new Set();
|
|
99
111
|
if (folded.summary) {
|
|
@@ -132,7 +144,8 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
|
|
|
132
144
|
.flatMap((attachmentChunk) => attachmentChunk.attachment ?? [])
|
|
133
145
|
.filter((ref) => !ref.deletedAt);
|
|
134
146
|
const images = refs.filter((ref) => IMAGE_MIMES.has(ref.mimeType.toLowerCase()));
|
|
135
|
-
const
|
|
147
|
+
const videos = refs.filter((ref) => isVideoAttachment(ref.fileName, ref.mimeType));
|
|
148
|
+
const audio = refs.filter((ref) => isAudioAttachment(ref.fileName, ref.mimeType) && !videos.includes(ref));
|
|
136
149
|
const documents = refs.filter((ref) => isDocumentAttachment(ref));
|
|
137
150
|
const pdfs = documents.filter(isPdfAttachment);
|
|
138
151
|
if (images.length) {
|
|
@@ -141,6 +154,9 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
|
|
|
141
154
|
if (audio.length) {
|
|
142
155
|
turn.audioAttachments = audio;
|
|
143
156
|
}
|
|
157
|
+
if (videos.length) {
|
|
158
|
+
turn.videos = videos.map((ref) => ({ ...ref, digest: digests.get(ref.id) }));
|
|
159
|
+
}
|
|
144
160
|
if (documents.length) {
|
|
145
161
|
turn.documents = documents;
|
|
146
162
|
}
|
|
@@ -8,6 +8,7 @@ export interface ChatMessage {
|
|
|
8
8
|
}[];
|
|
9
9
|
images?: AttachmentRef[];
|
|
10
10
|
audioAttachments?: AttachmentRef[];
|
|
11
|
+
videos?: VideoRef[];
|
|
11
12
|
documents?: DocumentAttachment[];
|
|
12
13
|
pdfs?: AttachmentRef[];
|
|
13
14
|
}
|
|
@@ -24,6 +25,19 @@ export interface UnheardAudioNoticeOptions {
|
|
|
24
25
|
export declare function unheardAudioNotice(label: string, reason: string, opts?: UnheardAudioNoticeOptions): string;
|
|
25
26
|
export declare function silentAudioNotice(fileName: string): string;
|
|
26
27
|
export declare function audioTranscriptBlock(fileName: string, transcript: string): string;
|
|
28
|
+
export interface VideoFrames {
|
|
29
|
+
readonly fileName: string;
|
|
30
|
+
readonly duration: number;
|
|
31
|
+
readonly times: ReadonlyArray<number>;
|
|
32
|
+
}
|
|
33
|
+
export declare function videoAnnotation({ fileName, duration, times }: VideoFrames): string;
|
|
34
|
+
export declare function videoTranscriptBlock(transcript: string): string;
|
|
35
|
+
export declare function unsyncedVideoNotice(fileName: string): string;
|
|
36
|
+
export declare function unreadVideoNotice(fileName: string): string;
|
|
37
|
+
export declare function lentVideoNotice(fileName: string): string;
|
|
38
|
+
export declare function hiddenVideoNotice(fileName: string): string;
|
|
39
|
+
export declare const VIDEO_NO_AUDIO = "[The video has no audio track.]";
|
|
40
|
+
export declare const VIDEO_SILENT = "[The video's audio contained no detectable speech.]";
|
|
27
41
|
export interface AttachmentRef {
|
|
28
42
|
id: string;
|
|
29
43
|
fileName: string;
|
|
@@ -31,6 +45,9 @@ export interface AttachmentRef {
|
|
|
31
45
|
blobId: string;
|
|
32
46
|
url?: string | null;
|
|
33
47
|
}
|
|
48
|
+
export interface VideoRef extends AttachmentRef {
|
|
49
|
+
digest?: AttachmentRef;
|
|
50
|
+
}
|
|
34
51
|
export interface AttachInput {
|
|
35
52
|
fileName: string;
|
|
36
53
|
mimeType: string;
|
|
@@ -47,6 +47,34 @@ export function audioTranscriptBlock(fileName, transcript) {
|
|
|
47
47
|
transcript
|
|
48
48
|
].join('\n');
|
|
49
49
|
}
|
|
50
|
+
function clock(seconds) {
|
|
51
|
+
const s = Math.round(seconds);
|
|
52
|
+
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`;
|
|
53
|
+
}
|
|
54
|
+
export function videoAnnotation({ fileName, duration, times }) {
|
|
55
|
+
return (`[Video "${fileName}", ${clock(duration)} long. You are seeing ${times.length} frames taken ` +
|
|
56
|
+
`evenly across the whole video, in order, at ${times.map(clock).join(', ')}. You cannot see ` +
|
|
57
|
+
'what happens between frames.]');
|
|
58
|
+
}
|
|
59
|
+
export function videoTranscriptBlock(transcript) {
|
|
60
|
+
return `The video's audio was transcribed on-device. Transcript:\n${transcript}`;
|
|
61
|
+
}
|
|
62
|
+
export function unsyncedVideoNotice(fileName) {
|
|
63
|
+
return `[Attached video "${fileName}" has not reached this device yet — the upload may still be syncing.]`;
|
|
64
|
+
}
|
|
65
|
+
export function unreadVideoNotice(fileName) {
|
|
66
|
+
return (`[Attached video "${fileName}" could not be read on this device: the file may be damaged, ` +
|
|
67
|
+
'or in a format it cannot decode. Tell the user so; do not try to open it another way.]');
|
|
68
|
+
}
|
|
69
|
+
export function lentVideoNotice(fileName) {
|
|
70
|
+
return (`[Attached video "${fileName}" — its frames cannot be sent to a model lent by another ` +
|
|
71
|
+
'device; only its audio is below. Tell the user if they ask what it shows.]');
|
|
72
|
+
}
|
|
73
|
+
export function hiddenVideoNotice(fileName) {
|
|
74
|
+
return `[Attached video "${fileName}" — its frames are not visible to this model]`;
|
|
75
|
+
}
|
|
76
|
+
export const VIDEO_NO_AUDIO = '[The video has no audio track.]';
|
|
77
|
+
export const VIDEO_SILENT = "[The video's audio contained no detectable speech.]";
|
|
50
78
|
const MAX_SENDER_NAME_CHARS = 64;
|
|
51
79
|
function displayableSender(name) {
|
|
52
80
|
const flat = name?.replace(/\s+/g, ' ').trim();
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type Size, type VideoProbe } from '../core/video/frames.ts';
|
|
2
|
+
export declare const VIDEO_FRAME_EDGE = 512;
|
|
3
|
+
export declare const MAX_VIDEO_FRAMES = 32;
|
|
4
|
+
export declare const MIN_VIDEO_FRAMES = 8;
|
|
5
|
+
export interface ContextReading {
|
|
6
|
+
readonly ctxSize?: number;
|
|
7
|
+
readonly cacheTokens?: number;
|
|
8
|
+
}
|
|
9
|
+
export interface FrameBudget extends ContextReading {
|
|
10
|
+
readonly videos: number;
|
|
11
|
+
}
|
|
12
|
+
export declare function frameTokens({ width, height }: Size): number;
|
|
13
|
+
export declare function videoFrameCount(probe: VideoProbe, budget: FrameBudget): number;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { frameSize } from '../core/video/frames.js';
|
|
2
|
+
export const VIDEO_FRAME_EDGE = 512;
|
|
3
|
+
export const MAX_VIDEO_FRAMES = 32;
|
|
4
|
+
export const MIN_VIDEO_FRAMES = 8;
|
|
5
|
+
const ANSWER_SHARE = 0.25;
|
|
6
|
+
const UNREAD_SHARE = 0.5;
|
|
7
|
+
const TILE = 32;
|
|
8
|
+
const FRAME_MARKER_TOKENS = 6;
|
|
9
|
+
export function frameTokens({ width, height }) {
|
|
10
|
+
return Math.ceil(width / TILE) * Math.ceil(height / TILE) + FRAME_MARKER_TOKENS;
|
|
11
|
+
}
|
|
12
|
+
function room({ ctxSize, cacheTokens }) {
|
|
13
|
+
if (!ctxSize) {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
if (cacheTokens === undefined) {
|
|
17
|
+
return ctxSize * UNREAD_SHARE;
|
|
18
|
+
}
|
|
19
|
+
return ctxSize * (1 - ANSWER_SHARE) - cacheTokens;
|
|
20
|
+
}
|
|
21
|
+
export function videoFrameCount(probe, budget) {
|
|
22
|
+
const perSecond = Math.max(1, Math.floor(probe.duration));
|
|
23
|
+
const free = room(budget);
|
|
24
|
+
const fits = free === null
|
|
25
|
+
? MAX_VIDEO_FRAMES
|
|
26
|
+
: Math.floor(free / Math.max(1, budget.videos) / frameTokens(frameSize(probe, VIDEO_FRAME_EDGE)));
|
|
27
|
+
const frames = Math.min(MAX_VIDEO_FRAMES, Math.max(MIN_VIDEO_FRAMES, fits));
|
|
28
|
+
return Math.min(perSecond, frames);
|
|
29
|
+
}
|
|
@@ -32,6 +32,7 @@ export interface AssistantOptions {
|
|
|
32
32
|
borrowHarness?: (deviceId: string) => Duplex;
|
|
33
33
|
blobReadTimeoutMs?: number;
|
|
34
34
|
runIdleTimeoutMs?: number;
|
|
35
|
+
modelReloadDelayMs?: number;
|
|
35
36
|
pdfRasterizer?: PdfRasterizer;
|
|
36
37
|
}
|
|
37
38
|
export interface SkillListOptions extends SkillOptions {
|
|
@@ -53,6 +54,8 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
53
54
|
private readonly _drives?;
|
|
54
55
|
private readonly _blobReadTimeoutMs;
|
|
55
56
|
private readonly _runIdleTimeoutMs;
|
|
57
|
+
private readonly _modelReloadDelayMs;
|
|
58
|
+
private readonly _reloads;
|
|
56
59
|
private readonly _pdfRasterizer;
|
|
57
60
|
private readonly _warmAbort;
|
|
58
61
|
private readonly _unloading;
|
|
@@ -93,6 +96,7 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
93
96
|
private readonly _runReplay;
|
|
94
97
|
private readonly _lifecycle;
|
|
95
98
|
private readonly _audioTranscripts;
|
|
99
|
+
private readonly _videoProbes;
|
|
96
100
|
private readonly _pdfPages;
|
|
97
101
|
private readonly _chats;
|
|
98
102
|
private readonly _runImageCacheNames;
|
|
@@ -101,7 +105,7 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
101
105
|
private readonly _modelCatalog;
|
|
102
106
|
private readonly _onHarnessStatus;
|
|
103
107
|
private readonly _onModelCatalogEvent;
|
|
104
|
-
constructor(engine: Capabilities, harness: Harness, { logger, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform, preloadCurrentAgentModel, drives, borrowHarness, blobReadTimeoutMs, runIdleTimeoutMs, pdfRasterizer: pdfRasterizerOption }?: AssistantOptions);
|
|
108
|
+
constructor(engine: Capabilities, harness: Harness, { logger, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform, preloadCurrentAgentModel, drives, borrowHarness, blobReadTimeoutMs, runIdleTimeoutMs, modelReloadDelayMs, pdfRasterizer: pdfRasterizerOption }?: AssistantOptions);
|
|
105
109
|
private _requireDeviceId;
|
|
106
110
|
_open(): Promise<void>;
|
|
107
111
|
private _triggerModelWarm;
|
|
@@ -141,6 +145,9 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
141
145
|
modelCatalogWatch(): AsyncIterable<AssistantModelCatalogRevision>;
|
|
142
146
|
loadModel(name: string): Promise<AssistantLoadModelResponse>;
|
|
143
147
|
private _isCached;
|
|
148
|
+
private _scheduleModelReload;
|
|
149
|
+
private _reloadLoadedModel;
|
|
150
|
+
private _cancelModelReloads;
|
|
144
151
|
private _agentRunning;
|
|
145
152
|
unloadModel(name: string): Promise<AssistantUnloadModelResponse>;
|
|
146
153
|
private _unloadModel;
|
|
@@ -157,7 +164,7 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
157
164
|
private _supervisedWatch;
|
|
158
165
|
private _reclaimDeletedChats;
|
|
159
166
|
private _dropDeletedChat;
|
|
160
|
-
private
|
|
167
|
+
private _followAgentEdits;
|
|
161
168
|
private _reclaimAgentChatCaches;
|
|
162
169
|
private _reconcileCompaction;
|
|
163
170
|
private _senderNames;
|
|
@@ -201,6 +208,18 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
201
208
|
private _uploadGenerated;
|
|
202
209
|
private _hydrateAudio;
|
|
203
210
|
private _transcribeAttachment;
|
|
211
|
+
private _seesImages;
|
|
212
|
+
private _hydrateVideos;
|
|
213
|
+
private _hydrateVideo;
|
|
214
|
+
private _readVideo;
|
|
215
|
+
private _probeVideo;
|
|
216
|
+
private _storedVideo;
|
|
217
|
+
private _cacheVideo;
|
|
218
|
+
private _sampleVideo;
|
|
219
|
+
private _storeVideoDigests;
|
|
220
|
+
private _uploadVideoDigest;
|
|
221
|
+
private _openVideo;
|
|
222
|
+
private _hearVideo;
|
|
204
223
|
private _hydrateImages;
|
|
205
224
|
private _hydrateTurnImages;
|
|
206
225
|
private _hydrateImage;
|
|
@@ -211,6 +230,7 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
211
230
|
private _renderPdfPage;
|
|
212
231
|
private _compactIfNearFull;
|
|
213
232
|
private _runCompaction;
|
|
233
|
+
private _contextReading;
|
|
214
234
|
private _lastRunMetrics;
|
|
215
235
|
compactChat(chatId: string, options: CompactOptions): Promise<CompactResult>;
|
|
216
236
|
private _summarize;
|