@qvac/core 0.10.3 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CHANGELOG.md +71 -1
  2. package/dist-lib/lib/assistant/history.d.ts +2 -1
  3. package/dist-lib/lib/assistant/history.js +32 -5
  4. package/dist-lib/lib/assistant/message.d.ts +20 -0
  5. package/dist-lib/lib/assistant/message.js +36 -0
  6. package/dist-lib/lib/assistant/pdf-budget.d.ts +3 -0
  7. package/dist-lib/lib/assistant/pdf-budget.js +13 -0
  8. package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
  9. package/dist-lib/lib/assistant/video-budget.js +29 -0
  10. package/dist-lib/lib/assistant-tree.d.ts +1 -0
  11. package/dist-lib/lib/assistant-tree.js +9 -2
  12. package/dist-lib/lib/assistant.d.ts +26 -3
  13. package/dist-lib/lib/assistant.js +471 -47
  14. package/dist-lib/lib/core/chat-fork.js +1 -0
  15. package/dist-lib/lib/core/chat-search.js +1 -1
  16. package/dist-lib/lib/core/chunk.d.ts +2 -2
  17. package/dist-lib/lib/core/chunk.js +5 -1
  18. package/dist-lib/lib/core/departures.js +6 -0
  19. package/dist-lib/lib/core/document-attachment.d.ts +7 -0
  20. package/dist-lib/lib/core/document-attachment.js +9 -0
  21. package/dist-lib/lib/core/files/index.js +4 -4
  22. package/dist-lib/lib/core/files/records/chunks.js +21 -0
  23. package/dist-lib/lib/core/files/records/purge.js +4 -3
  24. package/dist-lib/lib/core/files/records/source.js +3 -5
  25. package/dist-lib/lib/core/files/upload.js +3 -2
  26. package/dist-lib/lib/core/identity.d.ts +2 -0
  27. package/dist-lib/lib/core/identity.js +19 -0
  28. package/dist-lib/lib/core/pdf/rasterize.d.ts +0 -1
  29. package/dist-lib/lib/core/pdf/rasterize.js +0 -1
  30. package/dist-lib/lib/core/video/attachment.d.ts +1 -0
  31. package/dist-lib/lib/core/video/attachment.js +7 -0
  32. package/dist-lib/lib/core/video/digest.d.ts +16 -0
  33. package/dist-lib/lib/core/video/digest.js +57 -0
  34. package/dist-lib/lib/core/video/frames.d.ts +32 -0
  35. package/dist-lib/lib/core/video/frames.js +224 -0
  36. package/dist-lib/lib/core.d.ts +1 -0
  37. package/dist-lib/lib/core.js +44 -7
  38. package/dist-lib/lib/credential-pusher.d.ts +6 -0
  39. package/dist-lib/lib/credential-pusher.js +76 -59
  40. package/dist-lib/lib/errors.d.ts +8 -0
  41. package/dist-lib/lib/errors.js +8 -0
  42. package/dist-lib/lib/oauth/flow-core.js +2 -0
  43. package/dist-lib/lib/oauth/loopback.d.ts +1 -0
  44. package/dist-lib/lib/oauth/loopback.js +31 -1
  45. package/dist-lib/lib/oauth/register.js +33 -18
  46. package/dist-lib/lib/oauth/request.d.ts +10 -0
  47. package/dist-lib/lib/oauth/request.js +22 -0
  48. package/dist-lib/lib/oauth/token-exchange-core.js +29 -32
  49. package/dist-lib/lib/oauth/types.d.ts +3 -0
  50. package/dist-lib/lib/oauth.d.ts +2 -1
  51. package/dist-lib/lib/oauth.js +1 -0
  52. package/dist-lib/lib/run-notices.d.ts +1 -0
  53. package/dist-lib/lib/run-notices.js +2 -1
  54. package/dist-lib/lib/run-replay.d.ts +8 -2
  55. package/dist-lib/lib/run-replay.js +29 -11
  56. package/dist-lib/lib/task.js +1 -1
  57. package/dist-lib/schema/engine.js +4 -2
  58. package/dist-lib/schema/indexes.js +2 -2
  59. package/dist-lib/schema/qvac.js +4 -2
  60. package/dist-lib/spec/hrpc/messages.js +16 -6
  61. package/dist-lib/spec/hyperschema/index.js +16 -6
  62. package/dist-lib/spec/hyperschema/schema.json +10 -0
  63. package/dist-lib/spec/hyperschema/types.d.ts +2 -0
  64. package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
  65. package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
  66. package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
  67. package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
  68. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  69. package/package.json +7 -4
  70. package/tui/app.mjs +34 -1
  71. package/tui/lib/render.mjs +5 -0
  72. package/tui/main.mjs +8 -0
package/CHANGELOG.md CHANGED
@@ -13,6 +13,74 @@ release.
13
13
 
14
14
  ## [Unreleased]
15
15
 
16
+ ## [0.11.0] - 2026-10-23
17
+
18
+ ### Added
19
+
20
+ - `OAuthFlowDeps.onStage` reports `awaiting-callback` and `exchanging-code`, so a
21
+ host can log where a connect flow stopped. (#550)
22
+ - `deliverOAuthCallback(address)` in `@qvac/core/oauth` settles a waiting
23
+ sign-in from a callback address the user pastes, for a browser that cannot
24
+ reach the loopback. (#551)
25
+
26
+ ### Changed
27
+
28
+ - The OAuth loopback done page tells the user to return to QV.AC to finish
29
+ connecting. (#550)
30
+ - The PDF page-image cap scales with the run's context window, between 2 and
31
+ 32 pages, instead of a fixed 10. (#526)
32
+
33
+ ### Fixed
34
+
35
+ - An OAuth connect whose provider stops answering now fails after 30 s instead
36
+ of waiting forever: the code exchange after the browser hands back the code
37
+ with `OAUTH_TOKEN_EXCHANGE_TIMEOUT`, and dynamic client registration, before
38
+ the browser opens, with `OAUTH_REGISTRATION_TIMEOUT`.
39
+ `OAuthFlowDeps.requestTimeoutMs` sets the limit per request. (#555)
40
+ - A PDF cut at the page-image cap tells the model the last page shown. (#525)
41
+ - Each user turn renders the same bytes on every request, so the chat's KV
42
+ cache is reused. (#528)
43
+ - Two devices that take turns to leave the account and pair back can both still
44
+ `meshInvite`. Before, the fourth pairing left a device refused with
45
+ `DEVICE_INVITE_CHAIN_TOO_LONG`. (#513)
46
+ - `joinMesh` no longer hangs when a device pairs back right after it unpaired,
47
+ while the removal its leave left behind is still being delivered. (#513)
48
+
49
+ ## [0.10.4] - 2026-10-22
50
+
51
+ ### Added
52
+
53
+ - A video attachment reaches the chat's model as up to 32 frames, sized to the
54
+ context window, with their times and a transcript of its audio track. The
55
+ first run stores them as an attachment chunk carrying `sourceId`, the video's
56
+ attachment id, which later turns and other devices read instead of the video;
57
+ an app should not render a chunk with `sourceId`. `uploadAttachment` takes
58
+ `sourceId` on its first frame. Core now depends on `bare-ffmpeg`. See
59
+ [attachments](docs/api/attachments.md) and [chunks](docs/api/chunks.md).
60
+ (#496)
61
+ - A `ctxSize`, `kvCacheK` or `kvCacheV` change reloads the agent's model on its
62
+ host 2.5 seconds after the last edit, if it is loaded, instead of on the
63
+ agent's next message. (#495)
64
+
65
+ ### Changed
66
+
67
+ - `fileStatusWatch` reads only the chat's attachments, not every message in the
68
+ chat, on each refresh. (#518)
69
+
70
+ ### Fixed
71
+
72
+ - A device that unpairs itself hands the agents it hosted to its member's
73
+ earliest-paired live device, in the account and every shared mesh, instead of
74
+ leaving them orphaned. A leave from an earlier release is handed over on the
75
+ next boot. (#514)
76
+ - On a harness in its own process, an image sent to a model without a projector
77
+ is no longer dropped silently: the run asks the harness about the chat's
78
+ model, and the turn says the image is not visible to it. (#494)
79
+ - A token saved for an agent reaches `skills` and `skillCatalog` readiness as
80
+ soon as it is written, instead of after the ten-minute credential refresh.
81
+ (#523)
82
+ - `duck` reopens the meshes it had saved when it starts. (#519)
83
+
16
84
  ## [0.10.3] - 2026-10-21
17
85
 
18
86
  ### Added
@@ -1857,7 +1925,9 @@ Published under the `release-0.1` dist-tag.
1857
1925
  await assistant.ready()
1858
1926
  ```
1859
1927
 
1860
- [Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.10.3...HEAD
1928
+ [Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.11.0...HEAD
1929
+ [0.11.0]: https://github.com/tetherto/qvac-core/compare/v0.10.4...v0.11.0
1930
+ [0.10.4]: https://github.com/tetherto/qvac-core/compare/v0.10.3...v0.10.4
1861
1931
  [0.10.3]: https://github.com/tetherto/qvac-core/compare/v0.10.2...v0.10.3
1862
1932
  [0.10.2]: https://github.com/tetherto/qvac-core/compare/v0.10.1...v0.10.2
1863
1933
  [0.10.1]: https://github.com/tetherto/qvac-core/compare/v0.10.0...v0.10.1
@@ -37,8 +37,9 @@ export declare function senderAttribution(names: ReadonlyMap<string, string>): s
37
37
  export interface HistoryReplay {
38
38
  readonly senderNames?: ReadonlyMap<string, string>;
39
39
  readonly sentRuns?: ReadonlyMap<string, ReadonlyArray<ChatMessage>>;
40
+ readonly askedRuns?: ReadonlyMap<string, string>;
40
41
  }
41
- export declare function historyFromChunks(chunks: ReadonlyArray<AssistantChunk>, stale?: Partial<StaleToolRuns>, { senderNames, sentRuns }?: HistoryReplay): ChatMessage[];
42
+ export declare function historyFromChunks(chunks: ReadonlyArray<AssistantChunk>, stale?: Partial<StaleToolRuns>, { senderNames, sentRuns, askedRuns }?: HistoryReplay): ChatMessage[];
42
43
  export declare function inputAvailableToRun(chunk: AssistantChunk, requestSeq: number): boolean;
43
44
  export declare function normalizeQuestion(text: string): string;
44
45
  export declare const GENERIC_ACKNOWLEDGEMENTS: Set<string>;
@@ -4,6 +4,7 @@ import { isAudioAttachment } from '../core/audio-attachment.js';
4
4
  import { isDocumentAttachment, isPdfAttachment } from '../core/document-attachment.js';
5
5
  import { groupAttachmentsByMessage } from '../core/files/index.js';
6
6
  import { IMAGE_MIMES } from '../core/image/normalize.js';
7
+ import { isVideoAttachment } from '../core/video/attachment.js';
7
8
  import { compactionTurn, foldCompactedChunks } from '../compaction.js';
8
9
  import b4a from 'b4a';
9
10
  import { isRecord } from '../guards.js';
@@ -76,7 +77,17 @@ export function senderAttribution(names) {
76
77
  const distinct = [...new Set(names.values())].sort();
77
78
  return distinct.length > 1 ? distinct.join('\u0000') : '';
78
79
  }
79
- export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(), sentRuns = new Map() } = {}) {
80
+ function derivedAttachments(chunks) {
81
+ const derived = new Map();
82
+ for (const chunk of chunks) {
83
+ const { sourceId, attachment } = chunk;
84
+ if (sourceId && attachment && !attachment.deletedAt && !derived.has(sourceId)) {
85
+ derived.set(sourceId, attachment);
86
+ }
87
+ }
88
+ return derived;
89
+ }
90
+ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(), sentRuns = new Map(), askedRuns = new Map() } = {}) {
80
91
  const attemptStarts = latestAttemptStarts(chunks);
81
92
  const shared = new Set(senderNames.values()).size > 1;
82
93
  const staleRunIds = stale.runIds ?? new Set();
@@ -86,7 +97,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
86
97
  const attachments = [];
87
98
  const userMessages = [];
88
99
  for (const chunk of source) {
89
- if (chunk.type === 'attachment') {
100
+ if (chunk.type === 'attachment' && !chunk.sourceId) {
90
101
  attachments.push(chunk);
91
102
  }
92
103
  else if (chunk.userMessage) {
@@ -94,6 +105,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
94
105
  }
95
106
  }
96
107
  const byMessage = groupAttachmentsByMessage(attachments, userMessages);
108
+ const digests = derivedAttachments(chunks);
97
109
  const history = [];
98
110
  const replayedSkills = new Set();
99
111
  if (folded.summary) {
@@ -101,12 +113,18 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
101
113
  }
102
114
  const byRun = new Map();
103
115
  const kept = new Set();
116
+ let asking = null;
104
117
  const anchor = (runId) => {
105
118
  let turn = byRun.get(runId);
106
119
  if (!turn) {
107
120
  turn = { role: 'assistant', content: '' };
108
121
  byRun.set(runId, turn);
109
- const sent = runId && !staleRunIds.has(runId) ? sentRuns.get(runId) : undefined;
122
+ const live = runId && !staleRunIds.has(runId) ? runId : undefined;
123
+ const asked = live ? askedRuns.get(live) : undefined;
124
+ if (asking && asked !== undefined) {
125
+ asking.sentContent = asked;
126
+ }
127
+ const sent = live ? sentRuns.get(live) : undefined;
110
128
  if (sent) {
111
129
  const replayed = sent.map((message) => ({ ...message }));
112
130
  history.push(...replayed);
@@ -124,15 +142,21 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
124
142
  continue;
125
143
  }
126
144
  const sender = shared ? senderNames.get(chunk.id) : undefined;
127
- const turn = { role: 'user', content: chunk.userMessage.text };
145
+ const turn = {
146
+ role: 'user',
147
+ content: chunk.userMessage.text,
148
+ sentAt: chunk.createdAt
149
+ };
128
150
  if (sender) {
129
151
  turn.sender = sender;
130
152
  }
153
+ asking = turn;
131
154
  const refs = (byMessage.get(chunk.id) ?? [])
132
155
  .flatMap((attachmentChunk) => attachmentChunk.attachment ?? [])
133
156
  .filter((ref) => !ref.deletedAt);
134
157
  const images = refs.filter((ref) => IMAGE_MIMES.has(ref.mimeType.toLowerCase()));
135
- const audio = refs.filter((ref) => isAudioAttachment(ref.fileName, ref.mimeType));
158
+ const videos = refs.filter((ref) => isVideoAttachment(ref.fileName, ref.mimeType));
159
+ const audio = refs.filter((ref) => isAudioAttachment(ref.fileName, ref.mimeType) && !videos.includes(ref));
136
160
  const documents = refs.filter((ref) => isDocumentAttachment(ref));
137
161
  const pdfs = documents.filter(isPdfAttachment);
138
162
  if (images.length) {
@@ -141,6 +165,9 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
141
165
  if (audio.length) {
142
166
  turn.audioAttachments = audio;
143
167
  }
168
+ if (videos.length) {
169
+ turn.videos = videos.map((ref) => ({ ...ref, digest: digests.get(ref.id) }));
170
+ }
144
171
  if (documents.length) {
145
172
  turn.documents = documents;
146
173
  }
@@ -3,11 +3,14 @@ export interface ChatMessage {
3
3
  role: 'user' | 'assistant' | 'tool';
4
4
  content: string;
5
5
  sender?: string;
6
+ sentAt?: number;
7
+ sentContent?: string;
6
8
  attachments?: {
7
9
  path: string;
8
10
  }[];
9
11
  images?: AttachmentRef[];
10
12
  audioAttachments?: AttachmentRef[];
13
+ videos?: VideoRef[];
11
14
  documents?: DocumentAttachment[];
12
15
  pdfs?: AttachmentRef[];
13
16
  }
@@ -24,6 +27,19 @@ export interface UnheardAudioNoticeOptions {
24
27
  export declare function unheardAudioNotice(label: string, reason: string, opts?: UnheardAudioNoticeOptions): string;
25
28
  export declare function silentAudioNotice(fileName: string): string;
26
29
  export declare function audioTranscriptBlock(fileName: string, transcript: string): string;
30
+ export interface VideoFrames {
31
+ readonly fileName: string;
32
+ readonly duration: number;
33
+ readonly times: ReadonlyArray<number>;
34
+ }
35
+ export declare function videoAnnotation({ fileName, duration, times }: VideoFrames): string;
36
+ export declare function videoTranscriptBlock(transcript: string): string;
37
+ export declare function unsyncedVideoNotice(fileName: string): string;
38
+ export declare function unreadVideoNotice(fileName: string): string;
39
+ export declare function lentVideoNotice(fileName: string): string;
40
+ export declare function hiddenVideoNotice(fileName: string): string;
41
+ export declare const VIDEO_NO_AUDIO = "[The video has no audio track.]";
42
+ export declare const VIDEO_SILENT = "[The video's audio contained no detectable speech.]";
27
43
  export interface AttachmentRef {
28
44
  id: string;
29
45
  fileName: string;
@@ -31,6 +47,9 @@ export interface AttachmentRef {
31
47
  blobId: string;
32
48
  url?: string | null;
33
49
  }
50
+ export interface VideoRef extends AttachmentRef {
51
+ digest?: AttachmentRef;
52
+ }
34
53
  export interface AttachInput {
35
54
  fileName: string;
36
55
  mimeType: string;
@@ -46,4 +65,5 @@ export interface AttachmentMeta {
46
65
  export interface FetchedAttachment extends AttachmentMeta {
47
66
  bytes: Uint8Array;
48
67
  }
68
+ export declare function restoreSentContent(messages: ChatMessage[]): void;
49
69
  export declare function nameSenders(messages: ReadonlyArray<ChatMessage>): ChatMessage[];
@@ -47,11 +47,47 @@ export function audioTranscriptBlock(fileName, transcript) {
47
47
  transcript
48
48
  ].join('\n');
49
49
  }
50
+ function clock(seconds) {
51
+ const s = Math.round(seconds);
52
+ return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`;
53
+ }
54
+ export function videoAnnotation({ fileName, duration, times }) {
55
+ return (`[Video "${fileName}", ${clock(duration)} long. You are seeing ${times.length} frames taken ` +
56
+ `evenly across the whole video, in order, at ${times.map(clock).join(', ')}. You cannot see ` +
57
+ 'what happens between frames.]');
58
+ }
59
+ export function videoTranscriptBlock(transcript) {
60
+ return `The video's audio was transcribed on-device. Transcript:\n${transcript}`;
61
+ }
62
+ export function unsyncedVideoNotice(fileName) {
63
+ return `[Attached video "${fileName}" has not reached this device yet — the upload may still be syncing.]`;
64
+ }
65
+ export function unreadVideoNotice(fileName) {
66
+ return (`[Attached video "${fileName}" could not be read on this device: the file may be damaged, ` +
67
+ 'or in a format it cannot decode. Tell the user so; do not try to open it another way.]');
68
+ }
69
+ export function lentVideoNotice(fileName) {
70
+ return (`[Attached video "${fileName}" — its frames cannot be sent to a model lent by another ` +
71
+ 'device; only its audio is below. Tell the user if they ask what it shows.]');
72
+ }
73
+ export function hiddenVideoNotice(fileName) {
74
+ return `[Attached video "${fileName}" — its frames are not visible to this model]`;
75
+ }
76
+ export const VIDEO_NO_AUDIO = '[The video has no audio track.]';
77
+ export const VIDEO_SILENT = "[The video's audio contained no detectable speech.]";
50
78
  const MAX_SENDER_NAME_CHARS = 64;
51
79
  function displayableSender(name) {
52
80
  const flat = name?.replace(/\s+/g, ' ').trim();
53
81
  return flat ? Array.from(flat).slice(0, MAX_SENDER_NAME_CHARS).join('') : undefined;
54
82
  }
83
+ export function restoreSentContent(messages) {
84
+ for (const turn of messages) {
85
+ if (turn.sentContent === undefined)
86
+ continue;
87
+ turn.content = turn.sentContent;
88
+ delete turn.sentContent;
89
+ }
90
+ }
55
91
  export function nameSenders(messages) {
56
92
  return messages.map(({ sender, ...turn }) => {
57
93
  const name = displayableSender(sender);
@@ -0,0 +1,3 @@
1
+ export declare const MIN_PDF_PAGE_IMAGES = 2;
2
+ export declare const MAX_PDF_PAGE_IMAGES = 32;
3
+ export declare function pdfPageBudget(ctxSize: number): number;
@@ -0,0 +1,13 @@
1
+ import { MAX_IMAGE_WIDTH } from '../core/image/normalize.js';
2
+ import { frameTokens } from './video-budget.js';
3
+ const PAGE_TOKENS = frameTokens({
4
+ width: MAX_IMAGE_WIDTH,
5
+ height: Math.ceil(MAX_IMAGE_WIDTH * Math.SQRT2)
6
+ });
7
+ const PDF_SHARE = 0.5;
8
+ export const MIN_PDF_PAGE_IMAGES = 2;
9
+ export const MAX_PDF_PAGE_IMAGES = 32;
10
+ export function pdfPageBudget(ctxSize) {
11
+ const fits = Math.floor((ctxSize * PDF_SHARE) / PAGE_TOKENS);
12
+ return Math.min(MAX_PDF_PAGE_IMAGES, Math.max(MIN_PDF_PAGE_IMAGES, fits));
13
+ }
@@ -0,0 +1,13 @@
1
+ import { type Size, type VideoProbe } from '../core/video/frames.ts';
2
+ export declare const VIDEO_FRAME_EDGE = 512;
3
+ export declare const MAX_VIDEO_FRAMES = 32;
4
+ export declare const MIN_VIDEO_FRAMES = 8;
5
+ export interface ContextReading {
6
+ readonly ctxSize?: number;
7
+ readonly cacheTokens?: number;
8
+ }
9
+ export interface FrameBudget extends ContextReading {
10
+ readonly videos: number;
11
+ }
12
+ export declare function frameTokens({ width, height }: Size): number;
13
+ export declare function videoFrameCount(probe: VideoProbe, budget: FrameBudget): number;
@@ -0,0 +1,29 @@
1
+ import { frameSize } from '../core/video/frames.js';
2
+ export const VIDEO_FRAME_EDGE = 512;
3
+ export const MAX_VIDEO_FRAMES = 32;
4
+ export const MIN_VIDEO_FRAMES = 8;
5
+ const ANSWER_SHARE = 0.25;
6
+ const UNREAD_SHARE = 0.5;
7
+ const TILE = 32;
8
+ const FRAME_MARKER_TOKENS = 6;
9
+ export function frameTokens({ width, height }) {
10
+ return Math.ceil(width / TILE) * Math.ceil(height / TILE) + FRAME_MARKER_TOKENS;
11
+ }
12
+ function room({ ctxSize, cacheTokens }) {
13
+ if (!ctxSize) {
14
+ return null;
15
+ }
16
+ if (cacheTokens === undefined) {
17
+ return ctxSize * UNREAD_SHARE;
18
+ }
19
+ return ctxSize * (1 - ANSWER_SHARE) - cacheTokens;
20
+ }
21
+ export function videoFrameCount(probe, budget) {
22
+ const perSecond = Math.max(1, Math.floor(probe.duration));
23
+ const free = room(budget);
24
+ const fits = free === null
25
+ ? MAX_VIDEO_FRAMES
26
+ : Math.floor(free / Math.max(1, budget.videos) / frameTokens(frameSize(probe, VIDEO_FRAME_EDGE)));
27
+ const frames = Math.min(MAX_VIDEO_FRAMES, Math.max(MIN_VIDEO_FRAMES, fits));
28
+ return Math.min(perSecond, frames);
29
+ }
@@ -12,6 +12,7 @@ export type HarnessFactory = (ctx: HarnessContext) => Harness | Promise<Harness>
12
12
  export type EmbedderFactory = (harness: Harness) => IndexingEmbedder | null | undefined;
13
13
  export type OnHarnessDisposer = (() => void | Promise<void>) & {
14
14
  sync?: () => Promise<void>;
15
+ ready?: () => Promise<void>;
15
16
  };
16
17
  export type OnHarness = (harness: Harness, ctx: {
17
18
  engine: Client;
@@ -126,7 +126,10 @@ export function assistantTree(opts) {
126
126
  unload: (name) => assistant.unloadModel(name)
127
127
  });
128
128
  offModelUsers = core.registerModelUsers((name) => assistant.modelUsers(name));
129
- offSkills = core.registerSkills((agentId, modelName, options) => assistant.skills(agentId, modelName, options));
129
+ offSkills = core.registerSkills(async (agentId, modelName, options) => {
130
+ await offHarness?.ready?.();
131
+ return assistant.skills(agentId, modelName, options);
132
+ });
130
133
  const transcribe = harnessTranscribe(harness);
131
134
  offTranscribe = core.registerTranscribe(async (request) => {
132
135
  await assistant.ensureMediaModelUnpaused('transcription', request.model);
@@ -144,7 +147,11 @@ export function assistantTree(opts) {
144
147
  await assistant.ensureMediaModelUnpaused('tts');
145
148
  yield* speakStream(text);
146
149
  })());
147
- offSkillCatalog = core.registerSkillCatalog(harnessSkillCatalog(assistant, () => offHarness?.sync?.()));
150
+ const skillCatalog = harnessSkillCatalog(assistant, () => offHarness?.sync?.());
151
+ offSkillCatalog = core.registerSkillCatalog(async (agentId, modelName, options) => {
152
+ await offHarness?.ready?.();
153
+ return skillCatalog(agentId, modelName, options);
154
+ });
148
155
  offSkillManagement = core.registerSkillManagement(harnessSkillManagement(harness));
149
156
  offTools = core.registerTools(harnessTools(assistant));
150
157
  const emb = embedder?.(harness);
@@ -13,7 +13,7 @@ import type { Skill } from './assistant/skill-encode.ts';
13
13
  import { type CompactOptions, type CompactResult } from './assistant/history.ts';
14
14
  export { harnessClearKvCache, harnessDeleteModel, harnessListModels, harnessPreviewModelSource, harnessSkillCatalog, harnessSkillManagement, harnessDetectLanguage, harnessSpeak, harnessSpeakStream, harnessTools, harnessTranscribe, type DeleteModelSummary, type Harness, type HarnessInfo, type ListModelsOptions, type ModelInfo, type ModelType, type ToolCatalogEntry } from './assistant/harness.ts';
15
15
  export { type Skill } from './assistant/skill-encode.ts';
16
- export { annotateUnreadableImages, unheardAudioNotice, unreadableImagesNotice, DOCUMENT_ONLY_USER_PROMPT, IMAGE_AND_DOCUMENT_USER_PROMPT, IMAGE_ONLY_USER_PROMPT, nameSenders } from './assistant/message.ts';
16
+ export { annotateUnreadableImages, unheardAudioNotice, unreadableImagesNotice, DOCUMENT_ONLY_USER_PROMPT, IMAGE_AND_DOCUMENT_USER_PROMPT, IMAGE_ONLY_USER_PROMPT, nameSenders, restoreSentContent } from './assistant/message.ts';
17
17
  export { couldNameSenders, historyFromChunks, listHistoryChunks, senderAttribution, senderNamesFrom, staleToolRunIds, type CompactOptions, type CompactResult, type CompactionTrigger, type StaleToolRuns } from './assistant/history.ts';
18
18
  export declare const CAPTURE_INFERENCE_CALLS_KEY = "inspector.captureInferenceCalls";
19
19
  export declare const CAPTURE_ON = "on";
@@ -32,6 +32,7 @@ export interface AssistantOptions {
32
32
  borrowHarness?: (deviceId: string) => Duplex;
33
33
  blobReadTimeoutMs?: number;
34
34
  runIdleTimeoutMs?: number;
35
+ modelReloadDelayMs?: number;
35
36
  pdfRasterizer?: PdfRasterizer;
36
37
  }
37
38
  export interface SkillListOptions extends SkillOptions {
@@ -53,6 +54,8 @@ export declare class QvacAssistant extends ReadyResource {
53
54
  private readonly _drives?;
54
55
  private readonly _blobReadTimeoutMs;
55
56
  private readonly _runIdleTimeoutMs;
57
+ private readonly _modelReloadDelayMs;
58
+ private readonly _reloads;
56
59
  private readonly _pdfRasterizer;
57
60
  private readonly _warmAbort;
58
61
  private readonly _unloading;
@@ -93,6 +96,7 @@ export declare class QvacAssistant extends ReadyResource {
93
96
  private readonly _runReplay;
94
97
  private readonly _lifecycle;
95
98
  private readonly _audioTranscripts;
99
+ private readonly _videoProbes;
96
100
  private readonly _pdfPages;
97
101
  private readonly _chats;
98
102
  private readonly _runImageCacheNames;
@@ -101,7 +105,7 @@ export declare class QvacAssistant extends ReadyResource {
101
105
  private readonly _modelCatalog;
102
106
  private readonly _onHarnessStatus;
103
107
  private readonly _onModelCatalogEvent;
104
- constructor(engine: Capabilities, harness: Harness, { logger, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform, preloadCurrentAgentModel, drives, borrowHarness, blobReadTimeoutMs, runIdleTimeoutMs, pdfRasterizer: pdfRasterizerOption }?: AssistantOptions);
108
+ constructor(engine: Capabilities, harness: Harness, { logger, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform, preloadCurrentAgentModel, drives, borrowHarness, blobReadTimeoutMs, runIdleTimeoutMs, modelReloadDelayMs, pdfRasterizer: pdfRasterizerOption }?: AssistantOptions);
105
109
  private _requireDeviceId;
106
110
  _open(): Promise<void>;
107
111
  private _triggerModelWarm;
@@ -141,6 +145,9 @@ export declare class QvacAssistant extends ReadyResource {
141
145
  modelCatalogWatch(): AsyncIterable<AssistantModelCatalogRevision>;
142
146
  loadModel(name: string): Promise<AssistantLoadModelResponse>;
143
147
  private _isCached;
148
+ private _scheduleModelReload;
149
+ private _reloadLoadedModel;
150
+ private _cancelModelReloads;
144
151
  private _agentRunning;
145
152
  unloadModel(name: string): Promise<AssistantUnloadModelResponse>;
146
153
  private _unloadModel;
@@ -157,7 +164,7 @@ export declare class QvacAssistant extends ReadyResource {
157
164
  private _supervisedWatch;
158
165
  private _reclaimDeletedChats;
159
166
  private _dropDeletedChat;
160
- private _reclaimRekeyedAgents;
167
+ private _followAgentEdits;
161
168
  private _reclaimAgentChatCaches;
162
169
  private _reconcileCompaction;
163
170
  private _senderNames;
@@ -201,16 +208,32 @@ export declare class QvacAssistant extends ReadyResource {
201
208
  private _uploadGenerated;
202
209
  private _hydrateAudio;
203
210
  private _transcribeAttachment;
211
+ private _seesImages;
212
+ private _hydrateVideos;
213
+ private _hydrateVideo;
214
+ private _readVideo;
215
+ private _probeVideo;
216
+ private _storedVideo;
217
+ private _cacheVideo;
218
+ private _sampleVideo;
219
+ private _storeVideoDigests;
220
+ private _uploadVideoDigest;
221
+ private _openVideo;
222
+ private _hearVideo;
204
223
  private _hydrateImages;
205
224
  private _hydrateTurnImages;
206
225
  private _hydrateImage;
207
226
  private _hydratePdfPages;
227
+ private _pdfPageBudget;
228
+ private _customModel;
229
+ private _hydrateTurnPdfs;
208
230
  private _tryHydratePdf;
209
231
  private _hydratePdf;
210
232
  private _renderPdf;
211
233
  private _renderPdfPage;
212
234
  private _compactIfNearFull;
213
235
  private _runCompaction;
236
+ private _contextReading;
214
237
  private _lastRunMetrics;
215
238
  compactChat(chatId: string, options: CompactOptions): Promise<CompactResult>;
216
239
  private _summarize;