@qvac/core 0.10.2 → 0.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -1
- package/dist-lib/lib/api/agent-folders.d.ts +22 -0
- package/dist-lib/lib/api/agent-folders.js +115 -0
- package/dist-lib/lib/api/agents.d.ts +3 -1
- package/dist-lib/lib/api/agents.js +4 -2
- package/dist-lib/lib/api/index.d.ts +1 -0
- package/dist-lib/lib/api/index.js +4 -0
- package/dist-lib/lib/assistant/history.js +18 -2
- package/dist-lib/lib/assistant/message.d.ts +17 -0
- package/dist-lib/lib/assistant/message.js +28 -0
- package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
- package/dist-lib/lib/assistant/video-budget.js +29 -0
- package/dist-lib/lib/assistant-tree.js +2 -1
- package/dist-lib/lib/assistant.d.ts +22 -2
- package/dist-lib/lib/assistant.js +377 -15
- package/dist-lib/lib/core/chat-fork.js +1 -0
- package/dist-lib/lib/core/chat-search.js +1 -1
- package/dist-lib/lib/core/chat.d.ts +1 -0
- package/dist-lib/lib/core/chat.js +2 -1
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +5 -1
- package/dist-lib/lib/core/document-attachment.js +3 -0
- package/dist-lib/lib/core/files/index.js +4 -4
- package/dist-lib/lib/core/files/records/chunks.js +21 -0
- package/dist-lib/lib/core/files/records/purge.js +4 -3
- package/dist-lib/lib/core/files/records/source.js +3 -5
- package/dist-lib/lib/core/files/upload.js +3 -2
- package/dist-lib/lib/core/video/attachment.d.ts +1 -0
- package/dist-lib/lib/core/video/attachment.js +7 -0
- package/dist-lib/lib/core/video/digest.d.ts +16 -0
- package/dist-lib/lib/core/video/digest.js +57 -0
- package/dist-lib/lib/core/video/frames.d.ts +32 -0
- package/dist-lib/lib/core/video/frames.js +224 -0
- package/dist-lib/lib/core.js +25 -5
- package/dist-lib/lib/credential-pusher.d.ts +5 -0
- package/dist-lib/lib/credential-pusher.js +42 -11
- package/dist-lib/lib/errors.d.ts +20 -0
- package/dist-lib/lib/errors.js +20 -0
- package/dist-lib/lib/lent-harness.d.ts +2 -0
- package/dist-lib/lib/lent-harness.js +51 -0
- package/dist-lib/lib/mesh-scoped-engine.js +3 -0
- package/dist-lib/lib/run-notices.d.ts +1 -0
- package/dist-lib/lib/run-notices.js +2 -1
- package/dist-lib/lib/task.js +1 -1
- package/dist-lib/schema/engine.js +53 -2
- package/dist-lib/schema/indexes.js +2 -2
- package/dist-lib/schema/qvac.js +4 -2
- package/dist-lib/spec/bind.js +9 -0
- package/dist-lib/spec/calls.js +3 -0
- package/dist-lib/spec/capabilities.d.ts +6 -0
- package/dist-lib/spec/hrpc/hrpc.json +39 -0
- package/dist-lib/spec/hrpc/index.js +39 -3
- package/dist-lib/spec/hrpc/messages.js +177 -6
- package/dist-lib/spec/hyperschema/index.js +177 -6
- package/dist-lib/spec/hyperschema/schema.json +126 -0
- package/dist-lib/spec/hyperschema/types.d.ts +35 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
- package/package.json +4 -1
- package/tui/README.md +10 -0
- package/tui/app.mjs +63 -2
- package/tui/lib/folders-command.mjs +92 -0
- package/tui/lib/knowledge-import.mjs +3 -21
- package/tui/lib/local-path.mjs +21 -0
- package/tui/lib/render.mjs +5 -0
- package/tui/main.mjs +8 -0
package/CHANGELOG.md
CHANGED
|
@@ -13,6 +13,69 @@ release.
|
|
|
13
13
|
|
|
14
14
|
## [Unreleased]
|
|
15
15
|
|
|
16
|
+
## [0.10.4] - 2026-10-22
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- A video attachment reaches the chat's model as up to 32 frames, sized to the
|
|
21
|
+
context window, with their times and a transcript of its audio track. The
|
|
22
|
+
first run stores them as an attachment chunk carrying `sourceId`, the video's
|
|
23
|
+
attachment id, which later turns and other devices read instead of the video;
|
|
24
|
+
an app should not render a chunk with `sourceId`. `uploadAttachment` takes
|
|
25
|
+
`sourceId` on its first frame. Core now depends on `bare-ffmpeg`. See
|
|
26
|
+
[attachments](docs/api/attachments.md) and [chunks](docs/api/chunks.md).
|
|
27
|
+
(#496)
|
|
28
|
+
- A `ctxSize`, `kvCacheK` or `kvCacheV` change reloads the agent's model on its
|
|
29
|
+
host 2.5 seconds after the last edit, if it is loaded, instead of on the
|
|
30
|
+
agent's next message. (#495)
|
|
31
|
+
|
|
32
|
+
### Changed
|
|
33
|
+
|
|
34
|
+
- `fileStatusWatch` reads only the chat's attachments, not every message in the
|
|
35
|
+
chat, on each refresh. (#518)
|
|
36
|
+
|
|
37
|
+
### Fixed
|
|
38
|
+
|
|
39
|
+
- A device that unpairs itself hands the agents it hosted to its member's
|
|
40
|
+
earliest-paired live device, in the account and every shared mesh, instead of
|
|
41
|
+
leaving them orphaned. A leave from an earlier release is handed over on the
|
|
42
|
+
next boot. (#514)
|
|
43
|
+
- On a harness in its own process, an image sent to a model without a projector
|
|
44
|
+
is no longer dropped silently: the run asks the harness about the chat's
|
|
45
|
+
model, and the turn says the image is not visible to it. (#494)
|
|
46
|
+
- A token saved for an agent reaches `skills` and `skillCatalog` readiness as
|
|
47
|
+
soon as it is written, instead of after the ten-minute credential refresh.
|
|
48
|
+
(#523)
|
|
49
|
+
- `duck` reopens the meshes it had saved when it starts. (#519)
|
|
50
|
+
|
|
51
|
+
## [0.10.3] - 2026-10-21
|
|
52
|
+
|
|
53
|
+
### Added
|
|
54
|
+
|
|
55
|
+
- `listAgentFolders`, `addAgentFolder` and `removeAgentFolder` share a folder on
|
|
56
|
+
this device with an agent, read only or read and write, so its file tools have
|
|
57
|
+
somewhere to work. They keep the `fs.<agentId>.roots` and
|
|
58
|
+
`fs.<agentId>.writable` config keys the assistant already reads, so a grant
|
|
59
|
+
written before keeps working. See
|
|
60
|
+
[agent folders](docs/api/agent-folders.md). (#445)
|
|
61
|
+
|
|
62
|
+
### Changed
|
|
63
|
+
|
|
64
|
+
- The runs a `deleteAgent` with `keepChats` cancels carry
|
|
65
|
+
`reason: 'agent-deleted'`, so an app can tell them from a run its user took
|
|
66
|
+
back and keep the message they answered. (#486)
|
|
67
|
+
- `deleteAgent` forgets the folders shared with the agent on this device. (#445)
|
|
68
|
+
|
|
69
|
+
### Fixed
|
|
70
|
+
|
|
71
|
+
- A borrowed run (`lenderDeviceId`) no longer acts on the lender's machine. The
|
|
72
|
+
lender serves it with no shared folders, with the `files` and `exec` toolsets
|
|
73
|
+
off, as none of its own agents, and with its KV caches kept apart; a borrowed
|
|
74
|
+
run gets no image or attachment. (#477)
|
|
75
|
+
- A device-invite `joinMesh` no longer lists a mesh it is leaving as closed, so
|
|
76
|
+
the app no longer reopens it and reports `MESH_NOT_FOUND` after pairing.
|
|
77
|
+
(#489)
|
|
78
|
+
|
|
16
79
|
## [0.10.2] - 2026-10-20
|
|
17
80
|
|
|
18
81
|
### Added
|
|
@@ -1829,7 +1892,9 @@ Published under the `release-0.1` dist-tag.
|
|
|
1829
1892
|
await assistant.ready()
|
|
1830
1893
|
```
|
|
1831
1894
|
|
|
1832
|
-
[Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.10.
|
|
1895
|
+
[Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.10.4...HEAD
|
|
1896
|
+
[0.10.4]: https://github.com/tetherto/qvac-core/compare/v0.10.3...v0.10.4
|
|
1897
|
+
[0.10.3]: https://github.com/tetherto/qvac-core/compare/v0.10.2...v0.10.3
|
|
1833
1898
|
[0.10.2]: https://github.com/tetherto/qvac-core/compare/v0.10.1...v0.10.2
|
|
1834
1899
|
[0.10.1]: https://github.com/tetherto/qvac-core/compare/v0.10.0...v0.10.1
|
|
1835
1900
|
[0.10.0]: https://github.com/tetherto/qvac-core/compare/v0.9.0...v0.10.0
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { AssistantAddAgentFolderRequest, AssistantAgentFolders, AssistantAgentFoldersRequest, AssistantRemoveAgentFolderRequest } from '../../spec/hyperschema/types.d.ts';
|
|
2
|
+
import type { openLocalStore } from '../core/local/index.ts';
|
|
3
|
+
import type { CapabilityHandlers } from '../../spec/capabilities.d.ts';
|
|
4
|
+
type LocalStore = Awaited<ReturnType<typeof openLocalStore>>;
|
|
5
|
+
export interface AgentFoldersOptions {
|
|
6
|
+
readonly storageRoot: string;
|
|
7
|
+
}
|
|
8
|
+
export declare class AgentFolders {
|
|
9
|
+
private readonly _state;
|
|
10
|
+
private readonly _storageRoot;
|
|
11
|
+
private readonly _lock;
|
|
12
|
+
constructor(state: LocalStore, { storageRoot }: AgentFoldersOptions);
|
|
13
|
+
list({ agentId }: AssistantAgentFoldersRequest): Promise<AssistantAgentFolders>;
|
|
14
|
+
add({ agentId, path: given, access }: AssistantAddAgentFolderRequest): Promise<AssistantAgentFolders>;
|
|
15
|
+
remove({ agentId, path: given }: AssistantRemoveAgentFolderRequest): Promise<AssistantAgentFolders>;
|
|
16
|
+
forget(agentId: string): Promise<void>;
|
|
17
|
+
private _read;
|
|
18
|
+
private _update;
|
|
19
|
+
private _shareable;
|
|
20
|
+
}
|
|
21
|
+
export declare function createAgentFoldersApi(folders: AgentFolders): Pick<CapabilityHandlers, 'listAgentFolders' | 'addAgentFolder' | 'removeAgentFolder'>;
|
|
22
|
+
export {};
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import ScopeLock from 'scope-lock';
|
|
4
|
+
import { CoreError } from '../errors.js';
|
|
5
|
+
import { fsGrantKey, fsWriteGrantKey } from '../fs-grant-keys.js';
|
|
6
|
+
import { fsGrantsFrom, grantedDirsFrom } from '../fs-grants.js';
|
|
7
|
+
export class AgentFolders {
|
|
8
|
+
_state;
|
|
9
|
+
_storageRoot;
|
|
10
|
+
_lock = new ScopeLock();
|
|
11
|
+
constructor(state, { storageRoot }) {
|
|
12
|
+
this._state = state;
|
|
13
|
+
this._storageRoot = storageRoot;
|
|
14
|
+
}
|
|
15
|
+
async list({ agentId }) {
|
|
16
|
+
return { agentId, folders: foldersOf(await this._read(agentId)) };
|
|
17
|
+
}
|
|
18
|
+
async add({ agentId, path: given, access }) {
|
|
19
|
+
const folder = await this._shareable(given);
|
|
20
|
+
return this._update(agentId, ({ roots, writable }) => ({
|
|
21
|
+
roots: localPathsOf(roots).includes(folder)
|
|
22
|
+
? roots
|
|
23
|
+
: [...roots, { kind: 'local', path: folder }],
|
|
24
|
+
writable: access === 'write' ? withDir(writable, folder) : withoutDir(writable, folder)
|
|
25
|
+
}));
|
|
26
|
+
}
|
|
27
|
+
async remove({ agentId, path: given }) {
|
|
28
|
+
const names = await namesOf(given);
|
|
29
|
+
return this._update(agentId, (grants) => {
|
|
30
|
+
if (!foldersOf(grants).some((folder) => names.includes(folder.path))) {
|
|
31
|
+
throw new CoreError('AGENT_FOLDER_NOT_SHARED', [given]);
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
roots: grants.roots.filter((root) => !names.includes(localPathOf(root) ?? '')),
|
|
35
|
+
writable: names.reduce(withoutDir, grants.writable)
|
|
36
|
+
};
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
async forget(agentId) {
|
|
40
|
+
await this._lock.lock();
|
|
41
|
+
try {
|
|
42
|
+
await this._state.deleteAll([fsGrantKey(agentId), fsWriteGrantKey(agentId)]);
|
|
43
|
+
}
|
|
44
|
+
finally {
|
|
45
|
+
this._lock.unlock();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
async _read(agentId) {
|
|
49
|
+
const [roots, writable] = await Promise.all([
|
|
50
|
+
this._state.get(fsGrantKey(agentId)),
|
|
51
|
+
this._state.get(fsWriteGrantKey(agentId))
|
|
52
|
+
]);
|
|
53
|
+
return { roots: Array.isArray(roots) ? roots : [], writable: grantedDirsFrom(writable) };
|
|
54
|
+
}
|
|
55
|
+
async _update(agentId, change) {
|
|
56
|
+
await this._lock.lock();
|
|
57
|
+
try {
|
|
58
|
+
const next = change(await this._read(agentId));
|
|
59
|
+
await this._state.setAll([
|
|
60
|
+
{ key: fsGrantKey(agentId), value: next.roots },
|
|
61
|
+
{ key: fsWriteGrantKey(agentId), value: next.writable }
|
|
62
|
+
]);
|
|
63
|
+
return { agentId, folders: foldersOf(next) };
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
this._lock.unlock();
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
async _shareable(given) {
|
|
70
|
+
if (!path.isAbsolute(given)) {
|
|
71
|
+
throw new CoreError('AGENT_FOLDER_NOT_ABSOLUTE', [given]);
|
|
72
|
+
}
|
|
73
|
+
const folder = await fs.promises.realpath(given).catch(() => null);
|
|
74
|
+
if (!folder) {
|
|
75
|
+
throw new CoreError('AGENT_FOLDER_MISSING', [given]);
|
|
76
|
+
}
|
|
77
|
+
if (!(await fs.promises.stat(folder)).isDirectory()) {
|
|
78
|
+
throw new CoreError('AGENT_FOLDER_IS_FILE', [given]);
|
|
79
|
+
}
|
|
80
|
+
const storage = await fs.promises.realpath(this._storageRoot);
|
|
81
|
+
if (folder === storage || folder.startsWith(storage + path.sep)) {
|
|
82
|
+
throw new CoreError('AGENT_FOLDER_IN_STORAGE', [given]);
|
|
83
|
+
}
|
|
84
|
+
return folder;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
export function createAgentFoldersApi(folders) {
|
|
88
|
+
return {
|
|
89
|
+
listAgentFolders: (request) => folders.list(request),
|
|
90
|
+
addAgentFolder: (request) => folders.add(request),
|
|
91
|
+
removeAgentFolder: (request) => folders.remove(request)
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
async function namesOf(given) {
|
|
95
|
+
const resolved = path.resolve(given);
|
|
96
|
+
const real = await fs.promises.realpath(resolved).catch(() => resolved);
|
|
97
|
+
return [...new Set([resolved, real])];
|
|
98
|
+
}
|
|
99
|
+
function localPathOf(root) {
|
|
100
|
+
const [grant] = fsGrantsFrom([root]);
|
|
101
|
+
return grant?.kind === 'local' ? grant.path : null;
|
|
102
|
+
}
|
|
103
|
+
function localPathsOf(roots) {
|
|
104
|
+
return roots.map(localPathOf).filter((dir) => dir !== null);
|
|
105
|
+
}
|
|
106
|
+
function withDir(dirs, dir) {
|
|
107
|
+
return dirs.includes(dir) ? dirs : [...dirs, dir];
|
|
108
|
+
}
|
|
109
|
+
function withoutDir(dirs, dir) {
|
|
110
|
+
return dirs.filter((other) => other !== dir);
|
|
111
|
+
}
|
|
112
|
+
function foldersOf({ roots, writable }) {
|
|
113
|
+
const paths = [...new Set([...localPathsOf(roots), ...writable])];
|
|
114
|
+
return paths.map((dir) => ({ path: dir, access: writable.includes(dir) ? 'write' : 'read' }));
|
|
115
|
+
}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import type { LoggerInterface } from '@qvac/logging';
|
|
2
2
|
import { type MeshPool } from './current-mesh.ts';
|
|
3
3
|
import type { CapabilityHandlers } from '../../spec/capabilities.d.ts';
|
|
4
|
+
import type { AgentFolders } from './agent-folders.ts';
|
|
4
5
|
type AgentCapabilities = Pick<CapabilityHandlers, 'createAgent' | 'listAgents' | 'getAgent' | 'getDefaultAgent' | 'updateAgent' | 'deleteAgent' | 'agentsWatch'>;
|
|
5
6
|
export interface CreateAgentsApiOptions {
|
|
6
7
|
readonly deviceId: Buffer | null;
|
|
7
8
|
readonly computeLender: (() => Promise<Buffer | null>) | null;
|
|
9
|
+
readonly folders: AgentFolders | null;
|
|
8
10
|
readonly log: LoggerInterface | null;
|
|
9
11
|
}
|
|
10
|
-
export declare function createAgentsApi(pool: MeshPool, { deviceId, computeLender, log }: CreateAgentsApiOptions): AgentCapabilities;
|
|
12
|
+
export declare function createAgentsApi(pool: MeshPool, { deviceId, computeLender, folders, log }: CreateAgentsApiOptions): AgentCapabilities;
|
|
11
13
|
export {};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createAgent, deleteAgent, getDefaultAgent, isOrphaned, listAgents, liveProviderIds, mustGetActiveAgent, updateAgent } from '../core/agent.js';
|
|
2
2
|
import { meshWatchable, requireMesh } from './current-mesh.js';
|
|
3
3
|
import b4a from 'b4a';
|
|
4
|
-
export function createAgentsApi(pool, { deviceId, computeLender, log }) {
|
|
4
|
+
export function createAgentsApi(pool, { deviceId, computeLender, folders, log }) {
|
|
5
5
|
const withOrphaned = async (mesh, agent) => ({
|
|
6
6
|
...agent,
|
|
7
7
|
orphaned: await isOrphaned(mesh, agent)
|
|
@@ -43,7 +43,9 @@ export function createAgentsApi(pool, { deviceId, computeLender, log }) {
|
|
|
43
43
|
},
|
|
44
44
|
deleteAgent: async ({ agentId, meshId, keepChats }) => {
|
|
45
45
|
const mesh = await requireMesh(pool, meshId);
|
|
46
|
-
|
|
46
|
+
const deleted = await deleteAgent(mesh, deviceId, { id: agentId, keepChats });
|
|
47
|
+
await folders?.forget(agentId);
|
|
48
|
+
return withOrphaned(mesh, deleted);
|
|
47
49
|
},
|
|
48
50
|
agentsWatch: meshWatchable(pool, (request, mesh) => list(mesh, request.providerDeviceId ?? undefined), { name: 'agentsWatch', log })
|
|
49
51
|
};
|
|
@@ -60,6 +60,7 @@ interface ApiDeps {
|
|
|
60
60
|
platform?: (() => string | null) | null;
|
|
61
61
|
computeLender?: (() => Promise<Buffer | null>) | null;
|
|
62
62
|
pushSkillSecret?: ((request: AssistantPushSkillSecretRequest) => Promise<void>) | null;
|
|
63
|
+
storageRoot?: string | null;
|
|
63
64
|
skills?: ((agentId?: string | null, modelName?: string | null, options?: SkillOptions) => Promise<unknown>) | null;
|
|
64
65
|
skillCatalog?: ((agentId?: string | null, modelName?: string | null, options?: {
|
|
65
66
|
readonly refresh?: boolean;
|
|
@@ -2,6 +2,7 @@ import { toAsyncIterable } from '../core/async-iterable.js';
|
|
|
2
2
|
import { normalizeImageForModel } from '../core/image/normalize.js';
|
|
3
3
|
import { detectImageMime } from '../core/image/sniff.js';
|
|
4
4
|
import { createAgentsApi } from './agents.js';
|
|
5
|
+
import { AgentFolders, createAgentFoldersApi } from './agent-folders.js';
|
|
5
6
|
import { createApprovalRulesApi } from './approval-rules.js';
|
|
6
7
|
import { createAttachmentsApi } from './attachments.js';
|
|
7
8
|
import { createChatApi } from './chat.js';
|
|
@@ -38,6 +39,7 @@ export function createApi(deps = {}) {
|
|
|
38
39
|
const state = deps.state ?? null;
|
|
39
40
|
const log = deps.log ?? null;
|
|
40
41
|
const members = mesh && deps.meshStatus ? createMembersApi(mesh, deps.meshStatus, { state, log }) : null;
|
|
42
|
+
const folders = state && deps.storageRoot ? new AgentFolders(state, { storageRoot: deps.storageRoot }) : null;
|
|
41
43
|
const handlers = {
|
|
42
44
|
...(state ? createConfigApi(state, log) : null),
|
|
43
45
|
...(state ? createAppConfigApi(state, log) : null),
|
|
@@ -49,10 +51,12 @@ export function createApi(deps = {}) {
|
|
|
49
51
|
log
|
|
50
52
|
})
|
|
51
53
|
: null),
|
|
54
|
+
...(folders ? createAgentFoldersApi(folders) : null),
|
|
52
55
|
...(mesh
|
|
53
56
|
? createAgentsApi(mesh, {
|
|
54
57
|
deviceId: deps.deviceId ?? null,
|
|
55
58
|
computeLender: deps.computeLender ?? null,
|
|
59
|
+
folders,
|
|
56
60
|
log
|
|
57
61
|
})
|
|
58
62
|
: null),
|
|
@@ -4,6 +4,7 @@ import { isAudioAttachment } from '../core/audio-attachment.js';
|
|
|
4
4
|
import { isDocumentAttachment, isPdfAttachment } from '../core/document-attachment.js';
|
|
5
5
|
import { groupAttachmentsByMessage } from '../core/files/index.js';
|
|
6
6
|
import { IMAGE_MIMES } from '../core/image/normalize.js';
|
|
7
|
+
import { isVideoAttachment } from '../core/video/attachment.js';
|
|
7
8
|
import { compactionTurn, foldCompactedChunks } from '../compaction.js';
|
|
8
9
|
import b4a from 'b4a';
|
|
9
10
|
import { isRecord } from '../guards.js';
|
|
@@ -76,6 +77,16 @@ export function senderAttribution(names) {
|
|
|
76
77
|
const distinct = [...new Set(names.values())].sort();
|
|
77
78
|
return distinct.length > 1 ? distinct.join('\u0000') : '';
|
|
78
79
|
}
|
|
80
|
+
function derivedAttachments(chunks) {
|
|
81
|
+
const derived = new Map();
|
|
82
|
+
for (const chunk of chunks) {
|
|
83
|
+
const { sourceId, attachment } = chunk;
|
|
84
|
+
if (sourceId && attachment && !attachment.deletedAt && !derived.has(sourceId)) {
|
|
85
|
+
derived.set(sourceId, attachment);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return derived;
|
|
89
|
+
}
|
|
79
90
|
export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(), sentRuns = new Map() } = {}) {
|
|
80
91
|
const attemptStarts = latestAttemptStarts(chunks);
|
|
81
92
|
const shared = new Set(senderNames.values()).size > 1;
|
|
@@ -86,7 +97,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
|
|
|
86
97
|
const attachments = [];
|
|
87
98
|
const userMessages = [];
|
|
88
99
|
for (const chunk of source) {
|
|
89
|
-
if (chunk.type === 'attachment') {
|
|
100
|
+
if (chunk.type === 'attachment' && !chunk.sourceId) {
|
|
90
101
|
attachments.push(chunk);
|
|
91
102
|
}
|
|
92
103
|
else if (chunk.userMessage) {
|
|
@@ -94,6 +105,7 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
|
|
|
94
105
|
}
|
|
95
106
|
}
|
|
96
107
|
const byMessage = groupAttachmentsByMessage(attachments, userMessages);
|
|
108
|
+
const digests = derivedAttachments(chunks);
|
|
97
109
|
const history = [];
|
|
98
110
|
const replayedSkills = new Set();
|
|
99
111
|
if (folded.summary) {
|
|
@@ -132,7 +144,8 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
|
|
|
132
144
|
.flatMap((attachmentChunk) => attachmentChunk.attachment ?? [])
|
|
133
145
|
.filter((ref) => !ref.deletedAt);
|
|
134
146
|
const images = refs.filter((ref) => IMAGE_MIMES.has(ref.mimeType.toLowerCase()));
|
|
135
|
-
const
|
|
147
|
+
const videos = refs.filter((ref) => isVideoAttachment(ref.fileName, ref.mimeType));
|
|
148
|
+
const audio = refs.filter((ref) => isAudioAttachment(ref.fileName, ref.mimeType) && !videos.includes(ref));
|
|
136
149
|
const documents = refs.filter((ref) => isDocumentAttachment(ref));
|
|
137
150
|
const pdfs = documents.filter(isPdfAttachment);
|
|
138
151
|
if (images.length) {
|
|
@@ -141,6 +154,9 @@ export function historyFromChunks(chunks, stale = {}, { senderNames = new Map(),
|
|
|
141
154
|
if (audio.length) {
|
|
142
155
|
turn.audioAttachments = audio;
|
|
143
156
|
}
|
|
157
|
+
if (videos.length) {
|
|
158
|
+
turn.videos = videos.map((ref) => ({ ...ref, digest: digests.get(ref.id) }));
|
|
159
|
+
}
|
|
144
160
|
if (documents.length) {
|
|
145
161
|
turn.documents = documents;
|
|
146
162
|
}
|
|
@@ -8,6 +8,7 @@ export interface ChatMessage {
|
|
|
8
8
|
}[];
|
|
9
9
|
images?: AttachmentRef[];
|
|
10
10
|
audioAttachments?: AttachmentRef[];
|
|
11
|
+
videos?: VideoRef[];
|
|
11
12
|
documents?: DocumentAttachment[];
|
|
12
13
|
pdfs?: AttachmentRef[];
|
|
13
14
|
}
|
|
@@ -24,6 +25,19 @@ export interface UnheardAudioNoticeOptions {
|
|
|
24
25
|
export declare function unheardAudioNotice(label: string, reason: string, opts?: UnheardAudioNoticeOptions): string;
|
|
25
26
|
export declare function silentAudioNotice(fileName: string): string;
|
|
26
27
|
export declare function audioTranscriptBlock(fileName: string, transcript: string): string;
|
|
28
|
+
export interface VideoFrames {
|
|
29
|
+
readonly fileName: string;
|
|
30
|
+
readonly duration: number;
|
|
31
|
+
readonly times: ReadonlyArray<number>;
|
|
32
|
+
}
|
|
33
|
+
export declare function videoAnnotation({ fileName, duration, times }: VideoFrames): string;
|
|
34
|
+
export declare function videoTranscriptBlock(transcript: string): string;
|
|
35
|
+
export declare function unsyncedVideoNotice(fileName: string): string;
|
|
36
|
+
export declare function unreadVideoNotice(fileName: string): string;
|
|
37
|
+
export declare function lentVideoNotice(fileName: string): string;
|
|
38
|
+
export declare function hiddenVideoNotice(fileName: string): string;
|
|
39
|
+
export declare const VIDEO_NO_AUDIO = "[The video has no audio track.]";
|
|
40
|
+
export declare const VIDEO_SILENT = "[The video's audio contained no detectable speech.]";
|
|
27
41
|
export interface AttachmentRef {
|
|
28
42
|
id: string;
|
|
29
43
|
fileName: string;
|
|
@@ -31,6 +45,9 @@ export interface AttachmentRef {
|
|
|
31
45
|
blobId: string;
|
|
32
46
|
url?: string | null;
|
|
33
47
|
}
|
|
48
|
+
export interface VideoRef extends AttachmentRef {
|
|
49
|
+
digest?: AttachmentRef;
|
|
50
|
+
}
|
|
34
51
|
export interface AttachInput {
|
|
35
52
|
fileName: string;
|
|
36
53
|
mimeType: string;
|
|
@@ -47,6 +47,34 @@ export function audioTranscriptBlock(fileName, transcript) {
|
|
|
47
47
|
transcript
|
|
48
48
|
].join('\n');
|
|
49
49
|
}
|
|
50
|
+
function clock(seconds) {
|
|
51
|
+
const s = Math.round(seconds);
|
|
52
|
+
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`;
|
|
53
|
+
}
|
|
54
|
+
export function videoAnnotation({ fileName, duration, times }) {
|
|
55
|
+
return (`[Video "${fileName}", ${clock(duration)} long. You are seeing ${times.length} frames taken ` +
|
|
56
|
+
`evenly across the whole video, in order, at ${times.map(clock).join(', ')}. You cannot see ` +
|
|
57
|
+
'what happens between frames.]');
|
|
58
|
+
}
|
|
59
|
+
export function videoTranscriptBlock(transcript) {
|
|
60
|
+
return `The video's audio was transcribed on-device. Transcript:\n${transcript}`;
|
|
61
|
+
}
|
|
62
|
+
export function unsyncedVideoNotice(fileName) {
|
|
63
|
+
return `[Attached video "${fileName}" has not reached this device yet — the upload may still be syncing.]`;
|
|
64
|
+
}
|
|
65
|
+
export function unreadVideoNotice(fileName) {
|
|
66
|
+
return (`[Attached video "${fileName}" could not be read on this device: the file may be damaged, ` +
|
|
67
|
+
'or in a format it cannot decode. Tell the user so; do not try to open it another way.]');
|
|
68
|
+
}
|
|
69
|
+
export function lentVideoNotice(fileName) {
|
|
70
|
+
return (`[Attached video "${fileName}" — its frames cannot be sent to a model lent by another ` +
|
|
71
|
+
'device; only its audio is below. Tell the user if they ask what it shows.]');
|
|
72
|
+
}
|
|
73
|
+
export function hiddenVideoNotice(fileName) {
|
|
74
|
+
return `[Attached video "${fileName}" — its frames are not visible to this model]`;
|
|
75
|
+
}
|
|
76
|
+
export const VIDEO_NO_AUDIO = '[The video has no audio track.]';
|
|
77
|
+
export const VIDEO_SILENT = "[The video's audio contained no detectable speech.]";
|
|
50
78
|
const MAX_SENDER_NAME_CHARS = 64;
|
|
51
79
|
function displayableSender(name) {
|
|
52
80
|
const flat = name?.replace(/\s+/g, ' ').trim();
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type Size, type VideoProbe } from '../core/video/frames.ts';
|
|
2
|
+
export declare const VIDEO_FRAME_EDGE = 512;
|
|
3
|
+
export declare const MAX_VIDEO_FRAMES = 32;
|
|
4
|
+
export declare const MIN_VIDEO_FRAMES = 8;
|
|
5
|
+
export interface ContextReading {
|
|
6
|
+
readonly ctxSize?: number;
|
|
7
|
+
readonly cacheTokens?: number;
|
|
8
|
+
}
|
|
9
|
+
export interface FrameBudget extends ContextReading {
|
|
10
|
+
readonly videos: number;
|
|
11
|
+
}
|
|
12
|
+
export declare function frameTokens({ width, height }: Size): number;
|
|
13
|
+
export declare function videoFrameCount(probe: VideoProbe, budget: FrameBudget): number;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { frameSize } from '../core/video/frames.js';
|
|
2
|
+
export const VIDEO_FRAME_EDGE = 512;
|
|
3
|
+
export const MAX_VIDEO_FRAMES = 32;
|
|
4
|
+
export const MIN_VIDEO_FRAMES = 8;
|
|
5
|
+
const ANSWER_SHARE = 0.25;
|
|
6
|
+
const UNREAD_SHARE = 0.5;
|
|
7
|
+
const TILE = 32;
|
|
8
|
+
const FRAME_MARKER_TOKENS = 6;
|
|
9
|
+
export function frameTokens({ width, height }) {
|
|
10
|
+
return Math.ceil(width / TILE) * Math.ceil(height / TILE) + FRAME_MARKER_TOKENS;
|
|
11
|
+
}
|
|
12
|
+
function room({ ctxSize, cacheTokens }) {
|
|
13
|
+
if (!ctxSize) {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
if (cacheTokens === undefined) {
|
|
17
|
+
return ctxSize * UNREAD_SHARE;
|
|
18
|
+
}
|
|
19
|
+
return ctxSize * (1 - ANSWER_SHARE) - cacheTokens;
|
|
20
|
+
}
|
|
21
|
+
export function videoFrameCount(probe, budget) {
|
|
22
|
+
const perSecond = Math.max(1, Math.floor(probe.duration));
|
|
23
|
+
const free = room(budget);
|
|
24
|
+
const fits = free === null
|
|
25
|
+
? MAX_VIDEO_FRAMES
|
|
26
|
+
: Math.floor(free / Math.max(1, budget.videos) / frameTokens(frameSize(probe, VIDEO_FRAME_EDGE)));
|
|
27
|
+
const frames = Math.min(MAX_VIDEO_FRAMES, Math.max(MIN_VIDEO_FRAMES, fits));
|
|
28
|
+
return Math.min(perSecond, frames);
|
|
29
|
+
}
|
|
@@ -8,6 +8,7 @@ import { startModelMemoryCap } from './model-memory-cap.js';
|
|
|
8
8
|
import { startDownloadsAllowedPusher } from './downloads-allowed-pusher.js';
|
|
9
9
|
import { serveHarness } from '@qvac/harness';
|
|
10
10
|
import { duplexPair } from '@qvac/harness/transport';
|
|
11
|
+
import { lentHarness } from './lent-harness.js';
|
|
11
12
|
export function assistantTree(opts) {
|
|
12
13
|
const { core, harness, logger = null, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, embedder, extractors, onHarness, platform, preloadCurrentAgentModel, fakeModels = false } = opts;
|
|
13
14
|
const sup = new Supervisor();
|
|
@@ -118,7 +119,7 @@ export function assistantTree(opts) {
|
|
|
118
119
|
},
|
|
119
120
|
logger
|
|
120
121
|
});
|
|
121
|
-
offLoanableHarness = core.registerLoanableHarness((stream) => serveHarness(stream, harness));
|
|
122
|
+
offLoanableHarness = core.registerLoanableHarness((stream) => serveHarness(stream, lentHarness(harness)));
|
|
122
123
|
offRetryModelDownload = core.registerRetryModelDownload((name) => assistant.retryModelDownload(name));
|
|
123
124
|
offModelResidency = core.registerModelResidency({
|
|
124
125
|
load: (name) => assistant.loadModel(name),
|
|
@@ -32,6 +32,7 @@ export interface AssistantOptions {
|
|
|
32
32
|
borrowHarness?: (deviceId: string) => Duplex;
|
|
33
33
|
blobReadTimeoutMs?: number;
|
|
34
34
|
runIdleTimeoutMs?: number;
|
|
35
|
+
modelReloadDelayMs?: number;
|
|
35
36
|
pdfRasterizer?: PdfRasterizer;
|
|
36
37
|
}
|
|
37
38
|
export interface SkillListOptions extends SkillOptions {
|
|
@@ -53,6 +54,8 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
53
54
|
private readonly _drives?;
|
|
54
55
|
private readonly _blobReadTimeoutMs;
|
|
55
56
|
private readonly _runIdleTimeoutMs;
|
|
57
|
+
private readonly _modelReloadDelayMs;
|
|
58
|
+
private readonly _reloads;
|
|
56
59
|
private readonly _pdfRasterizer;
|
|
57
60
|
private readonly _warmAbort;
|
|
58
61
|
private readonly _unloading;
|
|
@@ -93,6 +96,7 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
93
96
|
private readonly _runReplay;
|
|
94
97
|
private readonly _lifecycle;
|
|
95
98
|
private readonly _audioTranscripts;
|
|
99
|
+
private readonly _videoProbes;
|
|
96
100
|
private readonly _pdfPages;
|
|
97
101
|
private readonly _chats;
|
|
98
102
|
private readonly _runImageCacheNames;
|
|
@@ -101,7 +105,7 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
101
105
|
private readonly _modelCatalog;
|
|
102
106
|
private readonly _onHarnessStatus;
|
|
103
107
|
private readonly _onModelCatalogEvent;
|
|
104
|
-
constructor(engine: Capabilities, harness: Harness, { logger, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform, preloadCurrentAgentModel, drives, borrowHarness, blobReadTimeoutMs, runIdleTimeoutMs, pdfRasterizer: pdfRasterizerOption }?: AssistantOptions);
|
|
108
|
+
constructor(engine: Capabilities, harness: Harness, { logger, imageCacheDir, inferenceCallsDir, runReplayDir, imageCacheMaxBytes, imageCacheMaxAgeMs, downloadRetryMs, platform, preloadCurrentAgentModel, drives, borrowHarness, blobReadTimeoutMs, runIdleTimeoutMs, modelReloadDelayMs, pdfRasterizer: pdfRasterizerOption }?: AssistantOptions);
|
|
105
109
|
private _requireDeviceId;
|
|
106
110
|
_open(): Promise<void>;
|
|
107
111
|
private _triggerModelWarm;
|
|
@@ -141,6 +145,9 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
141
145
|
modelCatalogWatch(): AsyncIterable<AssistantModelCatalogRevision>;
|
|
142
146
|
loadModel(name: string): Promise<AssistantLoadModelResponse>;
|
|
143
147
|
private _isCached;
|
|
148
|
+
private _scheduleModelReload;
|
|
149
|
+
private _reloadLoadedModel;
|
|
150
|
+
private _cancelModelReloads;
|
|
144
151
|
private _agentRunning;
|
|
145
152
|
unloadModel(name: string): Promise<AssistantUnloadModelResponse>;
|
|
146
153
|
private _unloadModel;
|
|
@@ -157,7 +164,7 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
157
164
|
private _supervisedWatch;
|
|
158
165
|
private _reclaimDeletedChats;
|
|
159
166
|
private _dropDeletedChat;
|
|
160
|
-
private
|
|
167
|
+
private _followAgentEdits;
|
|
161
168
|
private _reclaimAgentChatCaches;
|
|
162
169
|
private _reconcileCompaction;
|
|
163
170
|
private _senderNames;
|
|
@@ -201,6 +208,18 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
201
208
|
private _uploadGenerated;
|
|
202
209
|
private _hydrateAudio;
|
|
203
210
|
private _transcribeAttachment;
|
|
211
|
+
private _seesImages;
|
|
212
|
+
private _hydrateVideos;
|
|
213
|
+
private _hydrateVideo;
|
|
214
|
+
private _readVideo;
|
|
215
|
+
private _probeVideo;
|
|
216
|
+
private _storedVideo;
|
|
217
|
+
private _cacheVideo;
|
|
218
|
+
private _sampleVideo;
|
|
219
|
+
private _storeVideoDigests;
|
|
220
|
+
private _uploadVideoDigest;
|
|
221
|
+
private _openVideo;
|
|
222
|
+
private _hearVideo;
|
|
204
223
|
private _hydrateImages;
|
|
205
224
|
private _hydrateTurnImages;
|
|
206
225
|
private _hydrateImage;
|
|
@@ -211,6 +230,7 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
211
230
|
private _renderPdfPage;
|
|
212
231
|
private _compactIfNearFull;
|
|
213
232
|
private _runCompaction;
|
|
233
|
+
private _contextReading;
|
|
214
234
|
private _lastRunMetrics;
|
|
215
235
|
compactChat(chatId: string, options: CompactOptions): Promise<CompactResult>;
|
|
216
236
|
private _summarize;
|