@qvac/core 0.10.2 → 0.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/CHANGELOG.md +66 -1
  2. package/dist-lib/lib/api/agent-folders.d.ts +22 -0
  3. package/dist-lib/lib/api/agent-folders.js +115 -0
  4. package/dist-lib/lib/api/agents.d.ts +3 -1
  5. package/dist-lib/lib/api/agents.js +4 -2
  6. package/dist-lib/lib/api/index.d.ts +1 -0
  7. package/dist-lib/lib/api/index.js +4 -0
  8. package/dist-lib/lib/assistant/history.js +18 -2
  9. package/dist-lib/lib/assistant/message.d.ts +17 -0
  10. package/dist-lib/lib/assistant/message.js +28 -0
  11. package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
  12. package/dist-lib/lib/assistant/video-budget.js +29 -0
  13. package/dist-lib/lib/assistant-tree.js +2 -1
  14. package/dist-lib/lib/assistant.d.ts +22 -2
  15. package/dist-lib/lib/assistant.js +377 -15
  16. package/dist-lib/lib/core/chat-fork.js +1 -0
  17. package/dist-lib/lib/core/chat-search.js +1 -1
  18. package/dist-lib/lib/core/chat.d.ts +1 -0
  19. package/dist-lib/lib/core/chat.js +2 -1
  20. package/dist-lib/lib/core/chunk.d.ts +2 -2
  21. package/dist-lib/lib/core/chunk.js +5 -1
  22. package/dist-lib/lib/core/document-attachment.js +3 -0
  23. package/dist-lib/lib/core/files/index.js +4 -4
  24. package/dist-lib/lib/core/files/records/chunks.js +21 -0
  25. package/dist-lib/lib/core/files/records/purge.js +4 -3
  26. package/dist-lib/lib/core/files/records/source.js +3 -5
  27. package/dist-lib/lib/core/files/upload.js +3 -2
  28. package/dist-lib/lib/core/video/attachment.d.ts +1 -0
  29. package/dist-lib/lib/core/video/attachment.js +7 -0
  30. package/dist-lib/lib/core/video/digest.d.ts +16 -0
  31. package/dist-lib/lib/core/video/digest.js +57 -0
  32. package/dist-lib/lib/core/video/frames.d.ts +32 -0
  33. package/dist-lib/lib/core/video/frames.js +224 -0
  34. package/dist-lib/lib/core.js +25 -5
  35. package/dist-lib/lib/credential-pusher.d.ts +5 -0
  36. package/dist-lib/lib/credential-pusher.js +42 -11
  37. package/dist-lib/lib/errors.d.ts +20 -0
  38. package/dist-lib/lib/errors.js +20 -0
  39. package/dist-lib/lib/lent-harness.d.ts +2 -0
  40. package/dist-lib/lib/lent-harness.js +51 -0
  41. package/dist-lib/lib/mesh-scoped-engine.js +3 -0
  42. package/dist-lib/lib/run-notices.d.ts +1 -0
  43. package/dist-lib/lib/run-notices.js +2 -1
  44. package/dist-lib/lib/task.js +1 -1
  45. package/dist-lib/schema/engine.js +53 -2
  46. package/dist-lib/schema/indexes.js +2 -2
  47. package/dist-lib/schema/qvac.js +4 -2
  48. package/dist-lib/spec/bind.js +9 -0
  49. package/dist-lib/spec/calls.js +3 -0
  50. package/dist-lib/spec/capabilities.d.ts +6 -0
  51. package/dist-lib/spec/hrpc/hrpc.json +39 -0
  52. package/dist-lib/spec/hrpc/index.js +39 -3
  53. package/dist-lib/spec/hrpc/messages.js +177 -6
  54. package/dist-lib/spec/hyperschema/index.js +177 -6
  55. package/dist-lib/spec/hyperschema/schema.json +126 -0
  56. package/dist-lib/spec/hyperschema/types.d.ts +35 -0
  57. package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
  58. package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
  59. package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
  60. package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
  61. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  62. package/package.json +4 -1
  63. package/tui/README.md +10 -0
  64. package/tui/app.mjs +63 -2
  65. package/tui/lib/folders-command.mjs +92 -0
  66. package/tui/lib/knowledge-import.mjs +3 -21
  67. package/tui/lib/local-path.mjs +21 -0
  68. package/tui/lib/render.mjs +5 -0
  69. package/tui/main.mjs +8 -0
package/CHANGELOG.md CHANGED
@@ -13,6 +13,69 @@ release.
13
13
 
14
14
  ## [Unreleased]
15
15
 
16
+ ## [0.10.4] - 2026-10-22
17
+
18
+ ### Added
19
+
20
+ - A video attachment reaches the chat's model as up to 32 frames, sized to the
21
+ context window, with their times and a transcript of its audio track. The
22
+ first run stores them as an attachment chunk carrying `sourceId`, the video's
23
+ attachment id, which later turns and other devices read instead of the video;
24
+ an app should not render a chunk with `sourceId`. `uploadAttachment` takes
25
+ `sourceId` on its first frame. Core now depends on `bare-ffmpeg`. See
26
+ [attachments](docs/api/attachments.md) and [chunks](docs/api/chunks.md).
27
+ (#496)
28
+ - A `ctxSize`, `kvCacheK` or `kvCacheV` change reloads the agent's model on its
29
+ host 2.5 seconds after the last edit, if it is loaded, instead of on the
30
+ agent's next message. (#495)
31
+
32
+ ### Changed
33
+
34
+ - `fileStatusWatch` reads only the chat's attachments, not every message in the
35
+ chat, on each refresh. (#518)
36
+
37
+ ### Fixed
38
+
39
+ - A device that unpairs itself hands the agents it hosted to its member's
40
+ earliest-paired live device, in the account and every shared mesh, instead of
41
+ leaving them orphaned. A leave from an earlier release is handed over on the
42
+ next boot. (#514)
43
+ - On a harness in its own process, an image sent to a model without a projector
44
+ is no longer dropped silently: the run asks the harness about the chat's
45
+ model, and the turn says the image is not visible to it. (#494)
46
+ - A token saved for an agent reaches `skills` and `skillCatalog` readiness as
47
+ soon as it is written, instead of after the ten-minute credential refresh.
48
+ (#523)
49
+ - `duck` reopens the meshes it had saved when it starts. (#519)
50
+
51
+ ## [0.10.3] - 2026-10-21
52
+
53
+ ### Added
54
+
55
+ - `listAgentFolders`, `addAgentFolder` and `removeAgentFolder` share a folder on
56
+ this device with an agent, read only or read and write, so its file tools have
57
+ somewhere to work. They keep the `fs.<agentId>.roots` and
58
+ `fs.<agentId>.writable` config keys the assistant already reads, so a grant
59
+ written before keeps working. See
60
+ [agent folders](docs/api/agent-folders.md). (#445)
61
+
62
+ ### Changed
63
+
64
+ - The runs a `deleteAgent` with `keepChats` cancels carry
65
+ `reason: 'agent-deleted'`, so an app can tell them from a run its user took
66
+ back and keep the message they answered. (#486)
67
+ - `deleteAgent` forgets the folders shared with the agent on this device. (#445)
68
+
69
+ ### Fixed
70
+
71
+ - A borrowed run (`lenderDeviceId`) no longer acts on the lender's machine. The
72
+ lender serves it with no shared folders, with the `files` and `exec` toolsets
73
+ off, as none of its own agents, and with its KV caches kept apart; a borrowed
74
+ run gets no image or attachment. (#477)
75
+ - A device-invite `joinMesh` no longer lists a mesh it is leaving as closed, so
76
+ the app no longer reopens it and reports `MESH_NOT_FOUND` after pairing.
77
+ (#489)
78
+
16
79
  ## [0.10.2] - 2026-10-20
17
80
 
18
81
  ### Added
@@ -1829,7 +1892,9 @@ Published under the `release-0.1` dist-tag.
1829
1892
  await assistant.ready()
1830
1893
  ```
1831
1894
 
1832
- [Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.10.2...HEAD
1895
+ [Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.10.4...HEAD
1896
+ [0.10.4]: https://github.com/tetherto/qvac-core/compare/v0.10.3...v0.10.4
1897
+ [0.10.3]: https://github.com/tetherto/qvac-core/compare/v0.10.2...v0.10.3
1833
1898
  [0.10.2]: https://github.com/tetherto/qvac-core/compare/v0.10.1...v0.10.2
1834
1899
  [0.10.1]: https://github.com/tetherto/qvac-core/compare/v0.10.0...v0.10.1
1835
1900
  [0.10.0]: https://github.com/tetherto/qvac-core/compare/v0.9.0...v0.10.0
@@ -0,0 +1,22 @@
1
+ import type { AssistantAddAgentFolderRequest, AssistantAgentFolders, AssistantAgentFoldersRequest, AssistantRemoveAgentFolderRequest } from '../../spec/hyperschema/types.d.ts';
2
+ import type { openLocalStore } from '../core/local/index.ts';
3
+ import type { CapabilityHandlers } from '../../spec/capabilities.d.ts';
4
+ type LocalStore = Awaited<ReturnType<typeof openLocalStore>>;
5
+ export interface AgentFoldersOptions {
6
+ readonly storageRoot: string;
7
+ }
8
+ export declare class AgentFolders {
9
+ private readonly _state;
10
+ private readonly _storageRoot;
11
+ private readonly _lock;
12
+ constructor(state: LocalStore, { storageRoot }: AgentFoldersOptions);
13
+ list({ agentId }: AssistantAgentFoldersRequest): Promise<AssistantAgentFolders>;
14
+ add({ agentId, path: given, access }: AssistantAddAgentFolderRequest): Promise<AssistantAgentFolders>;
15
+ remove({ agentId, path: given }: AssistantRemoveAgentFolderRequest): Promise<AssistantAgentFolders>;
16
+ forget(agentId: string): Promise<void>;
17
+ private _read;
18
+ private _update;
19
+ private _shareable;
20
+ }
21
+ export declare function createAgentFoldersApi(folders: AgentFolders): Pick<CapabilityHandlers, 'listAgentFolders' | 'addAgentFolder' | 'removeAgentFolder'>;
22
+ export {};
@@ -0,0 +1,115 @@
1
+ import fs from 'fs';
2
+ import path from 'path';
3
+ import ScopeLock from 'scope-lock';
4
+ import { CoreError } from '../errors.js';
5
+ import { fsGrantKey, fsWriteGrantKey } from '../fs-grant-keys.js';
6
+ import { fsGrantsFrom, grantedDirsFrom } from '../fs-grants.js';
7
+ export class AgentFolders {
8
+ _state;
9
+ _storageRoot;
10
+ _lock = new ScopeLock();
11
+ constructor(state, { storageRoot }) {
12
+ this._state = state;
13
+ this._storageRoot = storageRoot;
14
+ }
15
+ async list({ agentId }) {
16
+ return { agentId, folders: foldersOf(await this._read(agentId)) };
17
+ }
18
+ async add({ agentId, path: given, access }) {
19
+ const folder = await this._shareable(given);
20
+ return this._update(agentId, ({ roots, writable }) => ({
21
+ roots: localPathsOf(roots).includes(folder)
22
+ ? roots
23
+ : [...roots, { kind: 'local', path: folder }],
24
+ writable: access === 'write' ? withDir(writable, folder) : withoutDir(writable, folder)
25
+ }));
26
+ }
27
+ async remove({ agentId, path: given }) {
28
+ const names = await namesOf(given);
29
+ return this._update(agentId, (grants) => {
30
+ if (!foldersOf(grants).some((folder) => names.includes(folder.path))) {
31
+ throw new CoreError('AGENT_FOLDER_NOT_SHARED', [given]);
32
+ }
33
+ return {
34
+ roots: grants.roots.filter((root) => !names.includes(localPathOf(root) ?? '')),
35
+ writable: names.reduce(withoutDir, grants.writable)
36
+ };
37
+ });
38
+ }
39
+ async forget(agentId) {
40
+ await this._lock.lock();
41
+ try {
42
+ await this._state.deleteAll([fsGrantKey(agentId), fsWriteGrantKey(agentId)]);
43
+ }
44
+ finally {
45
+ this._lock.unlock();
46
+ }
47
+ }
48
+ async _read(agentId) {
49
+ const [roots, writable] = await Promise.all([
50
+ this._state.get(fsGrantKey(agentId)),
51
+ this._state.get(fsWriteGrantKey(agentId))
52
+ ]);
53
+ return { roots: Array.isArray(roots) ? roots : [], writable: grantedDirsFrom(writable) };
54
+ }
55
+ async _update(agentId, change) {
56
+ await this._lock.lock();
57
+ try {
58
+ const next = change(await this._read(agentId));
59
+ await this._state.setAll([
60
+ { key: fsGrantKey(agentId), value: next.roots },
61
+ { key: fsWriteGrantKey(agentId), value: next.writable }
62
+ ]);
63
+ return { agentId, folders: foldersOf(next) };
64
+ }
65
+ finally {
66
+ this._lock.unlock();
67
+ }
68
+ }
69
+ async _shareable(given) {
70
+ if (!path.isAbsolute(given)) {
71
+ throw new CoreError('AGENT_FOLDER_NOT_ABSOLUTE', [given]);
72
+ }
73
+ const folder = await fs.promises.realpath(given).catch(() => null);
74
+ if (!folder) {
75
+ throw new CoreError('AGENT_FOLDER_MISSING', [given]);
76
+ }
77
+ if (!(await fs.promises.stat(folder)).isDirectory()) {
78
+ throw new CoreError('AGENT_FOLDER_IS_FILE', [given]);
79
+ }
80
+ const storage = await fs.promises.realpath(this._storageRoot);
81
+ if (folder === storage || folder.startsWith(storage + path.sep)) {
82
+ throw new CoreError('AGENT_FOLDER_IN_STORAGE', [given]);
83
+ }
84
+ return folder;
85
+ }
86
+ }
87
+ export function createAgentFoldersApi(folders) {
88
+ return {
89
+ listAgentFolders: (request) => folders.list(request),
90
+ addAgentFolder: (request) => folders.add(request),
91
+ removeAgentFolder: (request) => folders.remove(request)
92
+ };
93
+ }
94
+ async function namesOf(given) {
95
+ const resolved = path.resolve(given);
96
+ const real = await fs.promises.realpath(resolved).catch(() => resolved);
97
+ return [...new Set([resolved, real])];
98
+ }
99
+ function localPathOf(root) {
100
+ const [grant] = fsGrantsFrom([root]);
101
+ return grant?.kind === 'local' ? grant.path : null;
102
+ }
103
+ function localPathsOf(roots) {
104
+ return roots.map(localPathOf).filter((dir) => dir !== null);
105
+ }
106
+ function withDir(dirs, dir) {
107
+ return dirs.includes(dir) ? dirs : [...dirs, dir];
108
+ }
109
+ function withoutDir(dirs, dir) {
110
+ return dirs.filter((other) => other !== dir);
111
+ }
112
+ function foldersOf({ roots, writable }) {
113
+ const paths = [...new Set([...localPathsOf(roots), ...writable])];
114
+ return paths.map((dir) => ({ path: dir, access: writable.includes(dir) ? 'write' : 'read' }));
115
+ }
@@ -1,11 +1,13 @@
1
1
  import type { LoggerInterface } from '@qvac/logging';
2
2
  import { type MeshPool } from './current-mesh.ts';
3
3
  import type { CapabilityHandlers } from '../../spec/capabilities.d.ts';
4
+ import type { AgentFolders } from './agent-folders.ts';
4
5
  type AgentCapabilities = Pick<CapabilityHandlers, 'createAgent' | 'listAgents' | 'getAgent' | 'getDefaultAgent' | 'updateAgent' | 'deleteAgent' | 'agentsWatch'>;
5
6
  export interface CreateAgentsApiOptions {
6
7
  readonly deviceId: Buffer | null;
7
8
  readonly computeLender: (() => Promise<Buffer | null>) | null;
9
+ readonly folders: AgentFolders | null;
8
10
  readonly log: LoggerInterface | null;
9
11
  }
10
- export declare function createAgentsApi(pool: MeshPool, { deviceId, computeLender, log }: CreateAgentsApiOptions): AgentCapabilities;
12
+ export declare function createAgentsApi(pool: MeshPool, { deviceId, computeLender, folders, log }: CreateAgentsApiOptions): AgentCapabilities;
11
13
  export {};
@@ -1,7 +1,7 @@
1
1
  import { createAgent, deleteAgent, getDefaultAgent, isOrphaned, listAgents, liveProviderIds, mustGetActiveAgent, updateAgent } from '../core/agent.js';
2
2
  import { meshWatchable, requireMesh } from './current-mesh.js';
3
3
  import b4a from 'b4a';
4
- export function createAgentsApi(pool, { deviceId, computeLender, log }) {
4
+ export function createAgentsApi(pool, { deviceId, computeLender, folders, log }) {
5
5
  const withOrphaned = async (mesh, agent) => ({
6
6
  ...agent,
7
7
  orphaned: await isOrphaned(mesh, agent)
@@ -43,7 +43,9 @@ export function createAgentsApi(pool, { deviceId, computeLender, log }) {
43
43
  },
44
44
  deleteAgent: async ({ agentId, meshId, keepChats }) => {
45
45
  const mesh = await requireMesh(pool, meshId);
46
- return withOrphaned(mesh, await deleteAgent(mesh, deviceId, { id: agentId, keepChats }));
46
+ const deleted = await deleteAgent(mesh, deviceId, { id: agentId, keepChats });
47
+ await folders?.forget(agentId);
48
+ return withOrphaned(mesh, deleted);
47
49
  },
48
50
  agentsWatch: meshWatchable(pool, (request, mesh) => list(mesh, request.providerDeviceId ?? undefined), { name: 'agentsWatch', log })
49
51
  };
@@ -60,6 +60,7 @@ interface ApiDeps {
60
60
  platform?: (() => string | null) | null;
61
61
  computeLender?: (() => Promise<Buffer | null>) | null;
62
62
  pushSkillSecret?: ((request: AssistantPushSkillSecretRequest) => Promise<void>) | null;
63
+ storageRoot?: string | null;
63
64
  skills?: ((agentId?: string | null, modelName?: string | null, options?: SkillOptions) => Promise<unknown>) | null;
64
65
  skillCatalog?: ((agentId?: string | null, modelName?: string | null, options?: {
65
66
  readonly refresh?: boolean;
@@ -2,6 +2,7 @@ import { toAsyncIterable } from '../core/async-iterable.js';
2
2
  import { normalizeImageForModel } from '../core/image/normalize.js';
3
3
  import { detectImageMime } from '../core/image/sniff.js';
4
4
  import { createAgentsApi } from './agents.js';
5
+ import { AgentFolders, createAgentFoldersApi } from './agent-folders.js';
5
6
  import { createApprovalRulesApi } from './approval-rules.js';
6
7
  import { createAttachmentsApi } from './attachments.js';
7
8
  import { createChatApi } from './chat.js';
@@ -38,6 +39,7 @@ export function createApi(deps = {}) {
38
39
  const state = deps.state ?? null;
39
40
  const log = deps.log ?? null;
40
41
  const members = mesh && deps.meshStatus ? createMembersApi(mesh, deps.meshStatus, { state, log }) : null;
42
+ const folders = state && deps.storageRoot ? new AgentFolders(state, { storageRoot: deps.storageRoot }) : null;
41
43
  const handlers = {
42
44
  ...(state ? createConfigApi(state, log) : null),
43
45
  ...(state ? createAppConfigApi(state, log) : null),
@@ -49,10 +51,12 @@ export function createApi(deps = {}) {
49
51
  log
50
52
  })
51
53
  : null),
54
+ ...(folders ? createAgentFoldersApi(folders) : null),
52
55
  ...(mesh
53
56
  ? createAgentsApi(mesh, {
54
57
  deviceId: deps.deviceId ?? null,
55
58
  computeLender: deps.computeLender ?? null,
59
+ folders,
56
60
  log
57
61
  })
58
62
  : null),
@@ -4,6 +4,7 @@ import { isAudioAttachment } from '../core/audio-attachment.js';
4
4
  import { isDocumentAttachment, isPdfAttachment } from '../core/document-attachment.js';
5
5
  import { groupAttachmentsByMessage } from '../core/files/index.js';
6
6
  import { IMAGE_MIMES } from '../core/image/normalize.js';
7
+ import { isVideoAttachment } from '../core/video/attachment.js';
7
8
  import { compactionTurn, foldCompactedChunks } from '../compaction.js';
8
9
  import b4a from 'b4a';
9
10
  import { isRecord } from '../guards.js';
@@ -76,6 +77,16 @@ export function senderAttribution(names) {
76
77
  const distinct = [...new Set(names.values())].sort();
77
78
  return distinct.length > 1 ? distinct.join('\u0000') : '';
78
79
  }
80
+ function derivedAttachments(chunks) {
81
+ const derived = new Map();
82
+ for (const chunk of chunks) {
83
+ const { sourceId, attachment } = chunk;
84
+ if (sourceId && attachment && !attachment.deletedAt && !derived.has(sourceId)) {
85
+ derived.set(sourceId, attachment);
86
+ }
87
+ }
88
+ return derived;
89
+ }
79
90
  export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(), sentRuns = new Map() } = {}) {
80
91
  const attemptStarts = latestAttemptStarts(chunks);
81
92
  const shared = new Set(senderNames.values()).size > 1;
@@ -86,7 +97,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
86
97
  const attachments = [];
87
98
  const userMessages = [];
88
99
  for (const chunk of source) {
89
- if (chunk.type === 'attachment') {
100
+ if (chunk.type === 'attachment' && !chunk.sourceId) {
90
101
  attachments.push(chunk);
91
102
  }
92
103
  else if (chunk.userMessage) {
@@ -94,6 +105,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
94
105
  }
95
106
  }
96
107
  const byMessage = groupAttachmentsByMessage(attachments, userMessages);
108
+ const digests = derivedAttachments(chunks);
97
109
  const history = [];
98
110
  const replayedSkills = new Set();
99
111
  if (folded.summary) {
@@ -132,7 +144,8 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
132
144
  .flatMap((attachmentChunk) => attachmentChunk.attachment ?? [])
133
145
  .filter((ref) => !ref.deletedAt);
134
146
  const images = refs.filter((ref) => IMAGE_MIMES.has(ref.mimeType.toLowerCase()));
135
- const audio = refs.filter((ref) => isAudioAttachment(ref.fileName, ref.mimeType));
147
+ const videos = refs.filter((ref) => isVideoAttachment(ref.fileName, ref.mimeType));
148
+ const audio = refs.filter((ref) => isAudioAttachment(ref.fileName, ref.mimeType) && !videos.includes(ref));
136
149
  const documents = refs.filter((ref) => isDocumentAttachment(ref));
137
150
  const pdfs = documents.filter(isPdfAttachment);
138
151
  if (images.length) {
@@ -141,6 +154,9 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
141
154
  if (audio.length) {
142
155
  turn.audioAttachments = audio;
143
156
  }
157
+ if (videos.length) {
158
+ turn.videos = videos.map((ref) => ({ ...ref, digest: digests.get(ref.id) }));
159
+ }
144
160
  if (documents.length) {
145
161
  turn.documents = documents;
146
162
  }
@@ -8,6 +8,7 @@ export interface ChatMessage {
8
8
  }[];
9
9
  images?: AttachmentRef[];
10
10
  audioAttachments?: AttachmentRef[];
11
+ videos?: VideoRef[];
11
12
  documents?: DocumentAttachment[];
12
13
  pdfs?: AttachmentRef[];
13
14
  }
@@ -24,6 +25,19 @@ export interface UnheardAudioNoticeOptions {
24
25
  export declare function unheardAudioNotice(label: string, reason: string, opts?: UnheardAudioNoticeOptions): string;
25
26
  export declare function silentAudioNotice(fileName: string): string;
26
27
  export declare function audioTranscriptBlock(fileName: string, transcript: string): string;
28
+ export interface VideoFrames {
29
+ readonly fileName: string;
30
+ readonly duration: number;
31
+ readonly times: ReadonlyArray<number>;
32
+ }
33
+ export declare function videoAnnotation({ fileName, duration, times }: VideoFrames): string;
34
+ export declare function videoTranscriptBlock(transcript: string): string;
35
+ export declare function unsyncedVideoNotice(fileName: string): string;
36
+ export declare function unreadVideoNotice(fileName: string): string;
37
+ export declare function lentVideoNotice(fileName: string): string;
38
+ export declare function hiddenVideoNotice(fileName: string): string;
39
+ export declare const VIDEO_NO_AUDIO = "[The video has no audio track.]";
40
+ export declare const VIDEO_SILENT = "[The video's audio contained no detectable speech.]";
27
41
  export interface AttachmentRef {
28
42
  id: string;
29
43
  fileName: string;
@@ -31,6 +45,9 @@ export interface AttachmentRef {
31
45
  blobId: string;
32
46
  url?: string | null;
33
47
  }
48
+ export interface VideoRef extends AttachmentRef {
49
+ digest?: AttachmentRef;
50
+ }
34
51
  export interface AttachInput {
35
52
  fileName: string;
36
53
  mimeType: string;
@@ -47,6 +47,34 @@ export function audioTranscriptBlock(fileName, transcript) {
47
47
  transcript
48
48
  ].join('\n');
49
49
  }
50
+ function clock(seconds) {
51
+ const s = Math.round(seconds);
52
+ return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`;
53
+ }
54
+ export function videoAnnotation({ fileName, duration, times }) {
55
+ return (`[Video "${fileName}", ${clock(duration)} long. You are seeing ${times.length} frames taken ` +
56
+ `evenly across the whole video, in order, at ${times.map(clock).join(', ')}. You cannot see ` +
57
+ 'what happens between frames.]');
58
+ }
59
+ export function videoTranscriptBlock(transcript) {
60
+ return `The video's audio was transcribed on-device. Transcript:\n${transcript}`;
61
+ }
62
+ export function unsyncedVideoNotice(fileName) {
63
+ return `[Attached video "${fileName}" has not reached this device yet — the upload may still be syncing.]`;
64
+ }
65
+ export function unreadVideoNotice(fileName) {
66
+ return (`[Attached video "${fileName}" could not be read on this device: the file may be damaged, ` +
67
+ 'or in a format it cannot decode. Tell the user so; do not try to open it another way.]');
68
+ }
69
+ export function lentVideoNotice(fileName) {
70
+ return (`[Attached video "${fileName}" — its frames cannot be sent to a model lent by another ` +
71
+ 'device; only its audio is below. Tell the user if they ask what it shows.]');
72
+ }
73
+ export function hiddenVideoNotice(fileName) {
74
+ return `[Attached video "${fileName}" — its frames are not visible to this model]`;
75
+ }
76
+ export const VIDEO_NO_AUDIO = '[The video has no audio track.]';
77
+ export const VIDEO_SILENT = "[The video's audio contained no detectable speech.]";
50
78
  const MAX_SENDER_NAME_CHARS = 64;
51
79
  function displayableSender(name) {
52
80
  const flat = name?.replace(/\s+/g, ' ').trim();
@@ -0,0 +1,13 @@
1
+ import { type Size, type VideoProbe } from '../core/video/frames.ts';
2
+ export declare const VIDEO_FRAME_EDGE = 512;
3
+ export declare const MAX_VIDEO_FRAMES = 32;
4
+ export declare const MIN_VIDEO_FRAMES = 8;
5
+ export interface ContextReading {
6
+ readonly ctxSize?: number;
7
+ readonly cacheTokens?: number;
8
+ }
9
+ export interface FrameBudget extends ContextReading {
10
+ readonly videos: number;
11
+ }
12
+ export declare function frameTokens({ width, height }: Size): number;
13
+ export declare function videoFrameCount(probe: VideoProbe, budget: FrameBudget): number;
@@ -0,0 +1,29 @@
1
+ import { frameSize } from '../core/video/frames.js';
2
+ export const VIDEO_FRAME_EDGE = 512;
3
+ export const MAX_VIDEO_FRAMES = 32;
4
+ export const MIN_VIDEO_FRAMES = 8;
5
+ const ANSWER_SHARE = 0.25;
6
+ const UNREAD_SHARE = 0.5;
7
+ const TILE = 32;
8
+ const FRAME_MARKER_TOKENS = 6;
9
+ export function frameTokens({ width, height }) {
10
+ return Math.ceil(width / TILE) * Math.ceil(height / TILE) + FRAME_MARKER_TOKENS;
11
+ }
12
+ function room({ ctxSize, cacheTokens }) {
13
+ if (!ctxSize) {
14
+ return null;
15
+ }
16
+ if (cacheTokens === undefined) {
17
+ return ctxSize * UNREAD_SHARE;
18
+ }
19
+ return ctxSize * (1 - ANSWER_SHARE) - cacheTokens;
20
+ }
21
+ export function videoFrameCount(probe, budget) {
22
+ const perSecond = Math.max(1, Math.floor(probe.duration));
23
+ const free = room(budget);
24
+ const fits = free === null
25
+ ? MAX_VIDEO_FRAMES
26
+ : Math.floor(free / Math.max(1, budget.videos) / frameTokens(frameSize(probe, VIDEO_FRAME_EDGE)));
27
+ const frames = Math.min(MAX_VIDEO_FRAMES, Math.max(MIN_VIDEO_FRAMES, fits));
28
+ return Math.min(perSecond, frames);
29
+ }
@@ -8,6 +8,7 @@ import { startModelMemoryCap } from './model-memory-cap.js';
8
8
  import { startDownloadsAllowedPusher } from './downloads-allowed-pusher.js';
9
9
  import { serveHarness } from '@qvac/harness';
10
10
  import { duplexPair } from '@qvac/harness/transport';
11
+ import { lentHarness } from './lent-harness.js';
11
12
  export function assistantTree(opts) {
12
13
  const { core, harness, logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, embedder, extractors, onHarness, platform, preloadCurrentAgentModel, fakeModels = false } = opts;
13
14
  const sup = new Supervisor();
@@ -118,7 +119,7 @@ export function assistantTree(opts) {
118
119
  },
119
120
  logger
120
121
  });
121
- offLoanableHarness = core.registerLoanableHarness((stream) => serveHarness(stream, harness));
122
+ offLoanableHarness = core.registerLoanableHarness((stream) => serveHarness(stream, lentHarness(harness)));
122
123
  offRetryModelDownload = core.registerRetryModelDownload((name) => assistant.retryModelDownload(name));
123
124
  offModelResidency = core.registerModelResidency({
124
125
  load: (name) => assistant.loadModel(name),
@@ -32,6 +32,7 @@ export interface AssistantOptions {
32
32
  borrowHarness?: (deviceId: string) => Duplex;
33
33
  blobReadTimeoutMs?: number;
34
34
  runIdleTimeoutMs?: number;
35
+ modelReloadDelayMs?: number;
35
36
  pdfRasterizer?: PdfRasterizer;
36
37
  }
37
38
  export interface SkillListOptions extends SkillOptions {
@@ -53,6 +54,8 @@ export declare class QvacAssistant extends ReadyResource {
53
54
  private readonly _drives?;
54
55
  private readonly _blobReadTimeoutMs;
55
56
  private readonly _runIdleTimeoutMs;
57
+ private readonly _modelReloadDelayMs;
58
+ private readonly _reloads;
56
59
  private readonly _pdfRasterizer;
57
60
  private readonly _warmAbort;
58
61
  private readonly _unloading;
@@ -93,6 +96,7 @@ export declare class QvacAssistant extends ReadyResource {
93
96
  private readonly _runReplay;
94
97
  private readonly _lifecycle;
95
98
  private readonly _audioTranscripts;
99
+ private readonly _videoProbes;
96
100
  private readonly _pdfPages;
97
101
  private readonly _chats;
98
102
  private readonly _runImageCacheNames;
@@ -101,7 +105,7 @@ export declare class QvacAssistant extends ReadyResource {
101
105
  private readonly _modelCatalog;
102
106
  private readonly _onHarnessStatus;
103
107
  private readonly _onModelCatalogEvent;
104
- constructor(engine: Capabilities, harness: Harness, { logger, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform, preloadCurrentAgentModel, drives, borrowHarness, blobReadTimeoutMs, runIdleTimeoutMs, pdfRasterizer: pdfRasterizerOption }?: AssistantOptions);
108
+ constructor(engine: Capabilities, harness: Harness, { logger, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform, preloadCurrentAgentModel, drives, borrowHarness, blobReadTimeoutMs, runIdleTimeoutMs, modelReloadDelayMs, pdfRasterizer: pdfRasterizerOption }?: AssistantOptions);
105
109
  private _requireDeviceId;
106
110
  _open(): Promise<void>;
107
111
  private _triggerModelWarm;
@@ -141,6 +145,9 @@ export declare class QvacAssistant extends ReadyResource {
141
145
  modelCatalogWatch(): AsyncIterable<AssistantModelCatalogRevision>;
142
146
  loadModel(name: string): Promise<AssistantLoadModelResponse>;
143
147
  private _isCached;
148
+ private _scheduleModelReload;
149
+ private _reloadLoadedModel;
150
+ private _cancelModelReloads;
144
151
  private _agentRunning;
145
152
  unloadModel(name: string): Promise<AssistantUnloadModelResponse>;
146
153
  private _unloadModel;
@@ -157,7 +164,7 @@ export declare class QvacAssistant extends ReadyResource {
157
164
  private _supervisedWatch;
158
165
  private _reclaimDeletedChats;
159
166
  private _dropDeletedChat;
160
- private _reclaimRekeyedAgents;
167
+ private _followAgentEdits;
161
168
  private _reclaimAgentChatCaches;
162
169
  private _reconcileCompaction;
163
170
  private _senderNames;
@@ -201,6 +208,18 @@ export declare class QvacAssistant extends ReadyResource {
201
208
  private _uploadGenerated;
202
209
  private _hydrateAudio;
203
210
  private _transcribeAttachment;
211
+ private _seesImages;
212
+ private _hydrateVideos;
213
+ private _hydrateVideo;
214
+ private _readVideo;
215
+ private _probeVideo;
216
+ private _storedVideo;
217
+ private _cacheVideo;
218
+ private _sampleVideo;
219
+ private _storeVideoDigests;
220
+ private _uploadVideoDigest;
221
+ private _openVideo;
222
+ private _hearVideo;
204
223
  private _hydrateImages;
205
224
  private _hydrateTurnImages;
206
225
  private _hydrateImage;
@@ -211,6 +230,7 @@ export declare class QvacAssistant extends ReadyResource {
211
230
  private _renderPdfPage;
212
231
  private _compactIfNearFull;
213
232
  private _runCompaction;
233
+ private _contextReading;
214
234
  private _lastRunMetrics;
215
235
  compactChat(chatId: string, options: CompactOptions): Promise<CompactResult>;
216
236
  private _summarize;