@qvac/core 1.0.0-rc.2 → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +158 -2
- package/dist-lib/index.d.ts +1 -0
- package/dist-lib/index.js +1 -0
- package/dist-lib/lib/api/chunk-delta.d.ts +7 -2
- package/dist-lib/lib/api/chunk-delta.js +89 -7
- package/dist-lib/lib/api/chunks.js +14 -8
- package/dist-lib/lib/api/config.d.ts +4 -1
- package/dist-lib/lib/api/config.js +45 -4
- package/dist-lib/lib/api/custom-models.js +3 -5
- package/dist-lib/lib/api/devices.js +3 -1
- package/dist-lib/lib/api/files.d.ts +1 -1
- package/dist-lib/lib/api/files.js +65 -8
- package/dist-lib/lib/api/instrument.js +12 -4
- package/dist-lib/lib/api/members.js +7 -13
- package/dist-lib/lib/api/mesh.d.ts +1 -3
- package/dist-lib/lib/api/mesh.js +2 -2
- package/dist-lib/lib/api/notifications.js +8 -4
- package/dist-lib/lib/api/projects.js +5 -1
- package/dist-lib/lib/api/rag.js +32 -19
- package/dist-lib/lib/api/served.js +2 -1
- package/dist-lib/lib/api/workspace.js +0 -1
- package/dist-lib/lib/assistant/harness.d.ts +8 -2
- package/dist-lib/lib/assistant/harness.js +17 -4
- package/dist-lib/lib/assistant/history.d.ts +1 -1
- package/dist-lib/lib/assistant/history.js +19 -13
- package/dist-lib/lib/assistant/message.d.ts +6 -1
- package/dist-lib/lib/assistant/message.js +22 -2
- package/dist-lib/lib/assistant/model-round.d.ts +5 -0
- package/dist-lib/lib/assistant/model-round.js +46 -18
- package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
- package/dist-lib/lib/assistant/pdf-budget.js +10 -11
- package/dist-lib/lib/assistant/pdf-pages.d.ts +25 -0
- package/dist-lib/lib/assistant/pdf-pages.js +75 -0
- package/dist-lib/lib/assistant/round-host.js +5 -3
- package/dist-lib/lib/assistant-client.d.ts +2 -1
- package/dist-lib/lib/assistant-client.js +24 -6
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +8 -5
- package/dist-lib/lib/assistant.d.ts +20 -7
- package/dist-lib/lib/assistant.js +468 -240
- package/dist-lib/lib/client.js +10 -1
- package/dist-lib/lib/compaction.d.ts +3 -1
- package/dist-lib/lib/compaction.js +18 -4
- package/dist-lib/lib/constants.d.ts +3 -0
- package/dist-lib/lib/constants.js +3 -0
- package/dist-lib/lib/core/chat-activity.js +13 -16
- package/dist-lib/lib/core/chat-copy.js +1 -1
- package/dist-lib/lib/core/chat-fork.js +2 -1
- package/dist-lib/lib/core/chat-move.d.ts +4 -0
- package/dist-lib/lib/core/chat-move.js +42 -11
- package/dist-lib/lib/core/chat.d.ts +0 -1
- package/dist-lib/lib/core/chat.js +1 -1
- package/dist-lib/lib/core/chunk.d.ts +14 -3
- package/dist-lib/lib/core/chunk.js +112 -22
- package/dist-lib/lib/core/collections.d.ts +12 -1
- package/dist-lib/lib/core/collections.js +12 -1
- package/dist-lib/lib/core/config-value.d.ts +6 -0
- package/dist-lib/lib/core/config-value.js +14 -0
- package/dist-lib/lib/core/contract/activity.d.ts +3 -2
- package/dist-lib/lib/core/contract/activity.js +39 -21
- package/dist-lib/lib/core/contract/agent-join.js +15 -19
- package/dist-lib/lib/core/contract/agents.js +8 -7
- package/dist-lib/lib/core/contract/chat.d.ts +3 -0
- package/dist-lib/lib/core/contract/chat.js +12 -1
- package/dist-lib/lib/core/contract/chunks.js +16 -6
- package/dist-lib/lib/core/contract/devices.js +6 -4
- package/dist-lib/lib/core/contract/index.js +2 -1
- package/dist-lib/lib/core/contract/invites.js +12 -31
- package/dist-lib/lib/core/contract/joined-meshes.js +2 -1
- package/dist-lib/lib/core/contract/mesh-rules.js +4 -0
- package/dist-lib/lib/core/contract/model-requests.js +7 -6
- package/dist-lib/lib/core/contract/project-device-keys.js +15 -6
- package/dist-lib/lib/core/contract/project-host-keys.js +5 -4
- package/dist-lib/lib/core/contract/projects.js +12 -13
- package/dist-lib/lib/core/contract/public-seeds.js +5 -5
- package/dist-lib/lib/core/contract/roles.js +2 -1
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -1
- package/dist-lib/lib/core/contract/run-requests.js +19 -13
- package/dist-lib/lib/core/contract/shipped-rows.d.ts +2 -0
- package/dist-lib/lib/core/contract/shipped-rows.js +22 -0
- package/dist-lib/lib/core/contract/workspace.js +4 -10
- package/dist-lib/lib/core/contract/writers.js +20 -6
- package/dist-lib/lib/core/devices.d.ts +2 -1
- package/dist-lib/lib/core/devices.js +4 -2
- package/dist-lib/lib/core/document-attachment.d.ts +6 -3
- package/dist-lib/lib/core/document-attachment.js +10 -5
- package/dist-lib/lib/core/files/attachments.js +3 -3
- package/dist-lib/lib/core/files/index.d.ts +8 -5
- package/dist-lib/lib/core/files/index.js +109 -18
- package/dist-lib/lib/core/files/processing/index.d.ts +1 -1
- package/dist-lib/lib/core/files/processing/index.js +8 -2
- package/dist-lib/lib/core/files/processing/source.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/source.js +3 -0
- package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/state.js +7 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/build.js +14 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/index.js +1 -1
- package/dist-lib/lib/core/files/rag/produce.js +1 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +7 -1
- package/dist-lib/lib/core/first-event.d.ts +6 -0
- package/dist-lib/lib/core/first-event.js +13 -0
- package/dist-lib/lib/core/invite.d.ts +2 -2
- package/dist-lib/lib/core/invite.js +5 -4
- package/dist-lib/lib/core/local/index.d.ts +10 -1
- package/dist-lib/lib/core/local/index.js +105 -10
- package/dist-lib/lib/core/mesh.d.ts +15 -4
- package/dist-lib/lib/core/mesh.js +128 -57
- package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
- package/dist-lib/lib/core/migrations/chat-chunks.js +47 -0
- package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
- package/dist-lib/lib/core/migrations/file-index.js +6 -2
- package/dist-lib/lib/core/migrations/index.d.ts +2 -1
- package/dist-lib/lib/core/migrations/index.js +66 -9
- package/dist-lib/lib/core/notification.js +96 -23
- package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
- package/dist-lib/lib/core/pdf/figure.js +64 -0
- package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
- package/dist-lib/lib/core/pdf/rasterize.js +72 -22
- package/dist-lib/lib/core/peer-duplex.js +57 -5
- package/dist-lib/lib/core/public-mesh.js +8 -2
- package/dist-lib/lib/core/run-requests.js +2 -3
- package/dist-lib/lib/core/senders.d.ts +7 -6
- package/dist-lib/lib/core/senders.js +23 -30
- package/dist-lib/lib/core/shared-agents.d.ts +1 -0
- package/dist-lib/lib/core/shared-agents.js +47 -19
- package/dist-lib/lib/core/sync/background.d.ts +3 -15
- package/dist-lib/lib/core/sync/background.js +25 -102
- package/dist-lib/lib/core/sync/devices.js +6 -3
- package/dist-lib/lib/core/sync/index.js +12 -2
- package/dist-lib/lib/core/sync-status.js +3 -3
- package/dist-lib/lib/core.d.ts +59 -10
- package/dist-lib/lib/core.js +706 -174
- package/dist-lib/lib/credential-pusher.d.ts +11 -7
- package/dist-lib/lib/credential-pusher.js +40 -8
- package/dist-lib/lib/device-tree.d.ts +18 -0
- package/dist-lib/lib/device-tree.js +32 -0
- package/dist-lib/lib/doctor/redact.js +1 -1
- package/dist-lib/lib/errors.d.ts +4 -0
- package/dist-lib/lib/errors.js +4 -0
- package/dist-lib/lib/fs-drive-bridge.d.ts +7 -0
- package/dist-lib/lib/fs-drive-bridge.js +308 -30
- package/dist-lib/lib/integrations/setup-routes.js +7 -5
- package/dist-lib/lib/knowledge-search.d.ts +1 -1
- package/dist-lib/lib/knowledge-search.js +5 -46
- package/dist-lib/lib/kv-cache-retention.d.ts +35 -0
- package/dist-lib/lib/kv-cache-retention.js +98 -0
- package/dist-lib/lib/latest-frame.d.ts +10 -0
- package/dist-lib/lib/latest-frame.js +48 -0
- package/dist-lib/lib/model-download.d.ts +6 -1
- package/dist-lib/lib/model-download.js +32 -17
- package/dist-lib/lib/oauth/resolve.js +13 -5
- package/dist-lib/lib/reconnecting-watch.js +19 -11
- package/dist-lib/lib/run-notices.d.ts +2 -0
- package/dist-lib/lib/run-notices.js +5 -0
- package/dist-lib/lib/run-replay.d.ts +13 -1
- package/dist-lib/lib/run-replay.js +58 -21
- package/dist-lib/schema/build.js +66 -57
- package/dist-lib/schema/content.js +2 -1
- package/dist-lib/schema/engine.js +37 -5
- package/dist-lib/schema/field-order.js +7 -1
- package/dist-lib/schema/indexes.d.ts +4 -0
- package/dist-lib/schema/indexes.js +18 -0
- package/dist-lib/schema/local.js +9 -2
- package/dist-lib/schema/qvac.js +112 -4
- package/dist-lib/spec/bind.js +3 -0
- package/dist-lib/spec/calls.js +1 -0
- package/dist-lib/spec/capabilities.d.ts +2 -0
- package/dist-lib/spec/hrpc/hrpc.json +13 -0
- package/dist-lib/spec/hrpc/index.js +15 -3
- package/dist-lib/spec/hrpc/messages.js +1475 -1392
- package/dist-lib/spec/hyperschema/index.js +1475 -1392
- package/dist-lib/spec/hyperschema/schema.json +86 -1
- package/dist-lib/spec/hyperschema/types.d.ts +21 -1
- package/dist-lib/spec/local/hyperdb/messages.js +110 -54
- package/dist-lib/spec/local/hyperschema/index.js +56 -28
- package/dist-lib/spec/local/hyperschema/schema.json +55 -20
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -1
- package/dist-lib/spec/mesh/hyperdb/db.json +255 -51
- package/dist-lib/spec/mesh/hyperdb/index.js +1063 -404
- package/dist-lib/spec/mesh/hyperdb/messages.js +1647 -1214
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +62 -56
- package/dist-lib/spec/mesh/hyperdispatch/index.js +156 -139
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +787 -629
- package/dist-lib/spec/mesh/hyperschema/index.js +787 -629
- package/dist-lib/spec/mesh/hyperschema/schema.json +283 -174
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +29 -1
- package/package.json +23 -17
- package/worker/assistant-sidecar.mjs +3 -0
- package/dist-lib/lib/core/files/processing/settle.d.ts +0 -13
- package/dist-lib/lib/core/files/processing/settle.js +0 -86
- package/dist-lib/lib/core/migrations/device-indexes.d.ts +0 -2
- package/dist-lib/lib/core/migrations/device-indexes.js +0 -7
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
|
-
import {
|
|
2
|
+
import { isRecord } from '../guards.js';
|
|
3
|
+
import { coreErrorCodeOf } from '../errors.js';
|
|
3
4
|
const SCOPE = ['api'];
|
|
4
|
-
const MAX_STRING = 40;
|
|
5
5
|
export function instrument(handlers, log) {
|
|
6
6
|
const out = { ...handlers };
|
|
7
7
|
for (const [name, handler] of Object.entries(out)) {
|
|
@@ -47,9 +47,17 @@ function logFailure(log, name, { request, started, err }) {
|
|
|
47
47
|
log?.debug(SCOPE, `${name} failed`, {
|
|
48
48
|
request,
|
|
49
49
|
ms: Date.now() - started,
|
|
50
|
-
err:
|
|
50
|
+
err: errorType(err)
|
|
51
51
|
});
|
|
52
52
|
}
|
|
53
|
+
function errorType(err) {
|
|
54
|
+
const type = err instanceof Error ? err.name : typeof err;
|
|
55
|
+
const code = coreErrorCodeOf(err);
|
|
56
|
+
if (code) {
|
|
57
|
+
return `${type} (code ${code})`;
|
|
58
|
+
}
|
|
59
|
+
return type;
|
|
60
|
+
}
|
|
53
61
|
function isPromise(value) {
|
|
54
62
|
return isRecord(value) && typeof value.then === 'function';
|
|
55
63
|
}
|
|
@@ -85,7 +93,7 @@ function scalar(value) {
|
|
|
85
93
|
return `[${value.length}]`;
|
|
86
94
|
}
|
|
87
95
|
if (typeof value === 'string') {
|
|
88
|
-
return
|
|
96
|
+
return `<str:${value.length}>`;
|
|
89
97
|
}
|
|
90
98
|
if (typeof value === 'object') {
|
|
91
99
|
return '{}';
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { WORKSPACE_ROLES, memberRole } from '../core/contract/workspace.js';
|
|
1
|
+
import { WORKSPACE_ROLES, liveMemberIds, memberRole } from '../core/contract/workspace.js';
|
|
2
|
+
import { liveDevicesOf } from '../core/contract/activity.js';
|
|
2
3
|
import { meshWatchable, requireMesh } from './current-mesh.js';
|
|
3
4
|
import b4a from 'b4a';
|
|
4
|
-
import {
|
|
5
|
+
import { MEMBERS } from '../core/collections.js';
|
|
5
6
|
import { CoreError } from '../errors.js';
|
|
6
7
|
const RESOLVE_MEMBER_MS = 10_000;
|
|
7
8
|
async function resolveOurMember(mesh) {
|
|
@@ -19,6 +20,9 @@ async function resolveOurMember(mesh) {
|
|
|
19
20
|
clearTimeout(timer);
|
|
20
21
|
}
|
|
21
22
|
}
|
|
23
|
+
if (!mesh.ourMemberId && !mesh.writable) {
|
|
24
|
+
throw new CoreError('MESH_STILL_JOINING');
|
|
25
|
+
}
|
|
22
26
|
}
|
|
23
27
|
export async function requireOurMember(mesh) {
|
|
24
28
|
if (!mesh.ourMemberId) {
|
|
@@ -48,16 +52,6 @@ async function toEngineMember(reader, { mesh, member, removed }) {
|
|
|
48
52
|
const role = mesh.role === 'workspace' ? await memberRole(reader, member.id) : undefined;
|
|
49
53
|
return { id: member.id, name: member.name, joinedAt: member.joinedAt, removed, role };
|
|
50
54
|
}
|
|
51
|
-
async function liveMemberIds(snap) {
|
|
52
|
-
const live = new Set();
|
|
53
|
-
for await (const device of snap.find(DEVICES)) {
|
|
54
|
-
const owner = device.revoked ? null : (device.memberId ?? device.claimedMemberId);
|
|
55
|
-
if (owner) {
|
|
56
|
-
live.add(b4a.toHex(owner));
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
return live;
|
|
60
|
-
}
|
|
61
55
|
export function createMembersApi(pool, self, { state, log }) {
|
|
62
56
|
const mustGetMember = async (reader, id) => {
|
|
63
57
|
const member = await reader.get(MEMBERS, { id });
|
|
@@ -85,7 +79,7 @@ export function createMembersApi(pool, self, { state, log }) {
|
|
|
85
79
|
const snap = mesh.view.snapshot();
|
|
86
80
|
try {
|
|
87
81
|
const member = await mustGetMember(snap, id);
|
|
88
|
-
const removed =
|
|
82
|
+
const removed = (await liveDevicesOf(snap, id)).length === 0;
|
|
89
83
|
return await toEngineMember(snap, { mesh, member, removed });
|
|
90
84
|
}
|
|
91
85
|
finally {
|
|
@@ -26,9 +26,7 @@ export interface MeshStatusSource {
|
|
|
26
26
|
close(meshId: string): Promise<void>;
|
|
27
27
|
label(meshId: string, label: string): Promise<void>;
|
|
28
28
|
name(meshId: string, name: string): Promise<void>;
|
|
29
|
-
createMesh(
|
|
30
|
-
compute?: boolean;
|
|
31
|
-
}): Promise<string>;
|
|
29
|
+
createMesh(): Promise<string>;
|
|
32
30
|
createWorkspace(opts?: {
|
|
33
31
|
name?: string;
|
|
34
32
|
icon?: string;
|
package/dist-lib/lib/api/mesh.js
CHANGED
|
@@ -41,8 +41,8 @@ export function createMeshApi(status, updateMember, log) {
|
|
|
41
41
|
await status.name(meshId, name ?? '');
|
|
42
42
|
return { ok: true };
|
|
43
43
|
},
|
|
44
|
-
createMesh: async (
|
|
45
|
-
meshId: await status.createMesh(
|
|
44
|
+
createMesh: async () => ({
|
|
45
|
+
meshId: await status.createMesh()
|
|
46
46
|
}),
|
|
47
47
|
createRoom: async () => {
|
|
48
48
|
throw new CoreError('ROUTE_RETIRED', ['createRoom', 'createMesh']);
|
|
@@ -52,7 +52,7 @@ async function notificationSnapshot(mesh, local, { limit: requestedLimit, device
|
|
|
52
52
|
if (!mesh.closing && !mesh.closed) {
|
|
53
53
|
try {
|
|
54
54
|
chats = await deriveNotifications(mesh, local, {
|
|
55
|
-
limit:
|
|
55
|
+
limit: requestedLimit || 200,
|
|
56
56
|
deviceId
|
|
57
57
|
});
|
|
58
58
|
}
|
|
@@ -62,7 +62,12 @@ async function notificationSnapshot(mesh, local, { limit: requestedLimit, device
|
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
|
-
const
|
|
65
|
+
const limit = requestedLimit || 200;
|
|
66
|
+
const modelRows = (await local.getModelNotifications())
|
|
67
|
+
.filter((row) => row.notificationId && row.completedAt && !row.acknowledged)
|
|
68
|
+
.sort((left, right) => (right.completedAt ?? 0) - (left.completedAt ?? 0));
|
|
69
|
+
const modelsTruncated = modelRows.length > limit;
|
|
70
|
+
const models = modelRows.slice(0, limit);
|
|
66
71
|
if (mesh.closing || mesh.closed) {
|
|
67
72
|
chats = { notifications: [] };
|
|
68
73
|
}
|
|
@@ -79,9 +84,8 @@ async function notificationSnapshot(mesh, local, { limit: requestedLimit, device
|
|
|
79
84
|
]
|
|
80
85
|
: [])
|
|
81
86
|
].sort((left, right) => right.createdAt - left.createdAt || (right.seq ?? 0) - (left.seq ?? 0));
|
|
82
|
-
const limit = requestedLimit || 200;
|
|
83
87
|
return {
|
|
84
88
|
notifications: notifications.slice(0, limit),
|
|
85
|
-
truncated: notifications.length > limit || undefined
|
|
89
|
+
truncated: chats.truncated || modelsTruncated || notifications.length > limit || undefined
|
|
86
90
|
};
|
|
87
91
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { PROJECT_ORDER, PROJECTS } from '../core/collections.js';
|
|
2
2
|
import { listedProject, liveMember, readProject } from '../core/contract/projects.js';
|
|
3
|
+
import { unpickMemberAgents } from '../core/shared-agents.js';
|
|
3
4
|
import { dueProjectKey } from '../core/project-keys.js';
|
|
4
5
|
import { MAX_MESH_NAME_CHARS, MAX_PROJECT_ORDER_IDS, meshNameWithinBounds } from '../../schema/qvac.js';
|
|
5
6
|
import { meshWatchable, requireMesh } from './current-mesh.js';
|
|
@@ -112,7 +113,7 @@ async function addMember(pool, { projectOf, nameOf }, { meshId, projectId, membe
|
|
|
112
113
|
await batch.flush();
|
|
113
114
|
return reread(scope, nameOf);
|
|
114
115
|
}
|
|
115
|
-
async function removeMember(pool, { projectOf, nameOf }, { meshId, projectId, memberId }) {
|
|
116
|
+
async function removeMember(pool, { projectOf, nameOf, log }, { meshId, projectId, memberId }) {
|
|
116
117
|
const scope = await ourProject(pool, meshId, projectId);
|
|
117
118
|
if (b4a.equals(scope.ours, memberId)) {
|
|
118
119
|
throw new CoreError('MEMBER_REMOVE_SELF');
|
|
@@ -124,6 +125,9 @@ async function removeMember(pool, { projectOf, nameOf }, { meshId, projectId, me
|
|
|
124
125
|
memberId,
|
|
125
126
|
removed: true
|
|
126
127
|
});
|
|
128
|
+
await unpickMemberAgents(scope.workspace, project, memberId).catch((err) => {
|
|
129
|
+
log?.warn(['api', 'projects'], 'unpicking the removed member agents failed', err);
|
|
130
|
+
});
|
|
127
131
|
return reread(scope, nameOf);
|
|
128
132
|
}
|
|
129
133
|
async function remove(pool, { meshId, projectId }) {
|
package/dist-lib/lib/api/rag.js
CHANGED
|
@@ -3,7 +3,7 @@ import { getAgent } from '../core/agent.js';
|
|
|
3
3
|
import { runDefinition } from '../core/shared-agents.js';
|
|
4
4
|
import { getChat } from '../core/chat.js';
|
|
5
5
|
import { chatProject } from '../core/chat-project.js';
|
|
6
|
-
import { isKnowledgeRoutedAttachment, fullTextBudgetChars, fullTextHits, joinedIndex, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from '../core/files/index.js';
|
|
6
|
+
import { isKnowledgeRoutedAttachment, fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from '../core/files/index.js';
|
|
7
7
|
import { heldMesh, requireMesh } from './current-mesh.js';
|
|
8
8
|
import { errorMessage } from '../guards.js';
|
|
9
9
|
import b4a from 'b4a';
|
|
@@ -16,11 +16,17 @@ function clampLimit(limit) {
|
|
|
16
16
|
}
|
|
17
17
|
export function createRagApi(pool, { index, embedder, effectiveCtxSize = null }) {
|
|
18
18
|
return {
|
|
19
|
-
ragSearch: async ({ chatId, query, mode, limit, attachmentIds, meshId }) => {
|
|
19
|
+
ragSearch: async ({ chatId, query, mode, limit, attachmentIds, meshId, immediate }) => {
|
|
20
20
|
let pendingFiles = [];
|
|
21
21
|
let projectFilesNotReady = 0;
|
|
22
|
+
let chatFilesIndexingHere = 0;
|
|
22
23
|
const respond = (overrides) => {
|
|
23
|
-
const result = response({
|
|
24
|
+
const result = response({
|
|
25
|
+
pendingFiles,
|
|
26
|
+
projectFilesNotReady,
|
|
27
|
+
chatFilesIndexingHere,
|
|
28
|
+
...overrides
|
|
29
|
+
});
|
|
24
30
|
return { ...result, chatFilesNotReady: result.pendingFiles?.length ?? 0 };
|
|
25
31
|
};
|
|
26
32
|
const mesh = await requireMesh(pool, meshId);
|
|
@@ -35,10 +41,14 @@ export function createRagApi(pool, { index, embedder, effectiveCtxSize = null })
|
|
|
35
41
|
const kept = (id) => !off.has('chat') || scopeIds.has(id);
|
|
36
42
|
const chatKnowledge = async () => (await mesh.files.listKnowledge(chatId)).filter((row) => kept(row.id));
|
|
37
43
|
const chatFiles = async () => (await mesh.files.status(chatId)).files.filter((file) => kept(file.fileId));
|
|
38
|
-
|
|
44
|
+
if (!immediate) {
|
|
45
|
+
await mesh.files.indexingHere(chatId);
|
|
46
|
+
}
|
|
47
|
+
const chatRows = await chatKnowledge();
|
|
39
48
|
const initialFiles = await chatFiles();
|
|
40
49
|
pendingFiles = pendingFileNames(initialFiles);
|
|
41
|
-
|
|
50
|
+
chatFilesIndexingHere = initialFiles.filter((file) => mesh.files.indexesHere(file)).length;
|
|
51
|
+
const scopeRows = chatRows.filter((row) => scopeIds.has(row.id));
|
|
42
52
|
const project = off.has('project')
|
|
43
53
|
? null
|
|
44
54
|
: await chatProject(mesh, chatId, (id) => heldMesh(pool, id));
|
|
@@ -52,8 +62,11 @@ export function createRagApi(pool, { index, embedder, effectiveCtxSize = null })
|
|
|
52
62
|
head: !agentAllowsRagSearch(agent)
|
|
53
63
|
});
|
|
54
64
|
if (fullText) {
|
|
65
|
+
const read = new Set(scopeRows.map((row) => row.id));
|
|
66
|
+
const unread = initialFiles.filter((file) => !read.has(file.fileId));
|
|
55
67
|
return respond({
|
|
56
|
-
pendingFiles: pendingFileNames(initialFiles,
|
|
68
|
+
pendingFiles: pendingFileNames(initialFiles, read),
|
|
69
|
+
chatFilesIndexingHere: unread.filter((file) => mesh.files.indexesHere(file)).length,
|
|
57
70
|
searchedCount: fullText.length,
|
|
58
71
|
indexEmpty: false,
|
|
59
72
|
hasKnowledge: true,
|
|
@@ -68,17 +81,6 @@ export function createRagApi(pool, { index, embedder, effectiveCtxSize = null })
|
|
|
68
81
|
if (chatRows.length === 0 && kbRows.length === 0) {
|
|
69
82
|
return respond({});
|
|
70
83
|
}
|
|
71
|
-
const waiting = scopeRows.filter((row) => !mesh.files.hydratedArtifact(row.id));
|
|
72
|
-
if (waiting.length) {
|
|
73
|
-
try {
|
|
74
|
-
await mesh.files.settle(waiting.map((row) => row.id));
|
|
75
|
-
}
|
|
76
|
-
catch (err) {
|
|
77
|
-
mesh.log?.warn('rag', 'indexing settle failed', { err });
|
|
78
|
-
}
|
|
79
|
-
chatRows = await chatKnowledge();
|
|
80
|
-
scopeRows = chatRows.filter((row) => scopeIds.has(row.id));
|
|
81
|
-
}
|
|
82
84
|
const rows = [...chatRows, ...kbRows];
|
|
83
85
|
const hasKnowledge = rows.length > 0;
|
|
84
86
|
const chatIndex = index(meshId);
|
|
@@ -107,10 +109,11 @@ export function createRagApi(pool, { index, embedder, effectiveCtxSize = null })
|
|
|
107
109
|
const availableFiles = [...new Set(hydratedDocuments.map((document) => document.title))];
|
|
108
110
|
const failedRows = new Set();
|
|
109
111
|
const fileStates = await chatFiles();
|
|
112
|
+
const statusById = new Map(fileStates.map((file) => [file.fileId, file]));
|
|
110
113
|
for (const row of rows) {
|
|
111
114
|
if (artifactByRow.get(row))
|
|
112
115
|
continue;
|
|
113
|
-
if (
|
|
116
|
+
if (statusById.get(row.id)?.state === 'failed') {
|
|
114
117
|
failedRows.add(row);
|
|
115
118
|
}
|
|
116
119
|
}
|
|
@@ -136,6 +139,14 @@ export function createRagApi(pool, { index, embedder, effectiveCtxSize = null })
|
|
|
136
139
|
failedFiles
|
|
137
140
|
});
|
|
138
141
|
}
|
|
142
|
+
if (scopeIds.size > 0 && scopeDocumentIds.length === 0) {
|
|
143
|
+
return respond({
|
|
144
|
+
indexEmpty,
|
|
145
|
+
hasKnowledge,
|
|
146
|
+
availableFiles,
|
|
147
|
+
failedFiles
|
|
148
|
+
});
|
|
149
|
+
}
|
|
139
150
|
const pendingReferenced = scopeDocumentIds.length === 0 &&
|
|
140
151
|
referencesPendingFile(query, rows.map((row) => ({
|
|
141
152
|
title: row.fileName,
|
|
@@ -194,12 +205,14 @@ export function createRagApi(pool, { index, embedder, effectiveCtxSize = null })
|
|
|
194
205
|
}
|
|
195
206
|
}
|
|
196
207
|
try {
|
|
208
|
+
const ctxSize = effectiveCtxSize ? await effectiveCtxSize(agent) : 0;
|
|
197
209
|
const result = await retrieveRagKnowledge({
|
|
198
210
|
query,
|
|
199
211
|
chatId,
|
|
200
212
|
documents: hydratedDocuments,
|
|
201
213
|
liveSourceIds: rows.map((row) => row.id),
|
|
202
|
-
scopeDocumentIds: scopeDocumentIds.length > 0 ? scopeDocumentIds : undefined
|
|
214
|
+
scopeDocumentIds: scopeDocumentIds.length > 0 ? scopeDocumentIds : undefined,
|
|
215
|
+
chunkBudgetTokens: retrievalBudgetTokens(ctxSize)
|
|
203
216
|
}, {
|
|
204
217
|
index: vectorIndex,
|
|
205
218
|
embed: (texts) => embed.embed({ texts }),
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readChunk } from '../core/chunk.js';
|
|
1
2
|
import { listModelRequests } from '../core/model-requests.js';
|
|
2
3
|
import { servedChats } from '../core/public-mesh.js';
|
|
3
4
|
import { createChunksApi } from './chunks.js';
|
|
@@ -27,7 +28,7 @@ export function createServedApi({ pool, local, foreign, status, list, deviceId,
|
|
|
27
28
|
const rounds = watchable(({ meshId }) => requireMesh(pool, meshId, { follow: true, wide: true }), async (_request, mesh) => {
|
|
28
29
|
const chats = new Set((deviceId ? await servedChats(mesh, deviceId) : []).map(({ id }) => id));
|
|
29
30
|
const live = await listModelRequests(mesh);
|
|
30
|
-
return { chunks: live.filter((round) => chats.has(round.chatId)) };
|
|
31
|
+
return { chunks: live.filter((round) => chats.has(round.chatId)).map(readChunk) };
|
|
31
32
|
}, { name: 'modelRequestsWatch', log });
|
|
32
33
|
const { chatsWatch: ownChats = chatsWatch, appendChunk: ownAppend = chunks.appendChunk } = local;
|
|
33
34
|
return {
|
|
@@ -22,7 +22,6 @@ export function createWorkspaceApi(pool, status, log) {
|
|
|
22
22
|
workspaceWatch: meshWatchable(pool, ({ meshId }, mesh) => read(mesh, meshId), { name: 'workspaceWatch', log }),
|
|
23
23
|
updateWorkspace: async (request) => update(await requireMesh(pool, request.meshId), request, status),
|
|
24
24
|
leaveWorkspace: async ({ meshId }) => {
|
|
25
|
-
requireWorkspace(await requireMesh(pool, meshId), meshId);
|
|
26
25
|
await status.leaveWorkspace(meshId);
|
|
27
26
|
return { ok: true };
|
|
28
27
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import type { ApprovalCall, CompletionEvent, DeleteModelSummary, DriveOpFn, GrantedDrive, HostRunLog, ManagedSkill, MemoryBudget, ModelRoundIo, ModelRoundPort, ModelRoundRequest, ModelRoundResult, ModelSourcePreview, PreviewModelSourceContext, SkillFileContent } from '@qvac/harness';
|
|
3
3
|
import type { SkillCatalogOptions, SkillManagementHooks } from '../core.ts';
|
|
4
|
-
import type { AssistantAgent, AssistantCapabilityProfile, AssistantChunkContentAttachment, AssistantCustomModelPreview, AssistantDeleteModelRequest, AssistantDetectLanguageRequest, AssistantDetectLanguageResponse, AssistantDeleteModelResponse, AssistantEnhanceMode, AssistantKvCacheDeletionResponse, AssistantListModelsRequest, AssistantModelInfoList, AssistantPreviewCustomModelRequest, AssistantSkillCatalogResponse, AssistantSpeakRequest, AssistantSpeakResponse, AssistantToolCatalogResponse, AssistantToolReportEntry, AssistantTranscribeRequest, AssistantTranscribeResponse } from '../../spec/hyperschema/types.d.ts';
|
|
4
|
+
import type { AssistantAgent, AssistantCapabilityProfile, AssistantChunkContentAttachment, AssistantCustomModelPreview, AssistantCustomModelReportEntry, AssistantDeleteModelRequest, AssistantDetectLanguageRequest, AssistantDetectLanguageResponse, AssistantDeleteModelResponse, AssistantEnhanceMode, AssistantKvCacheDeletionResponse, AssistantListModelsRequest, AssistantModelInfoList, AssistantPreviewCustomModelRequest, AssistantSkillCatalogResponse, AssistantSpeakRequest, AssistantSpeakResponse, AssistantToolCatalogResponse, AssistantToolReportEntry, AssistantTranscribeRequest, AssistantTranscribeResponse } from '../../spec/hyperschema/types.d.ts';
|
|
5
5
|
import type { HarnessDiagnostics } from '../diagnostics.ts';
|
|
6
6
|
import type { DownloadArgs, DownloadEventStream, ImageDownloadObservation } from '../model-download.ts';
|
|
7
7
|
import type { RunInput } from '../run-input.ts';
|
|
@@ -49,6 +49,7 @@ export interface ModelInfo {
|
|
|
49
49
|
actualSize?: number;
|
|
50
50
|
taskType?: string;
|
|
51
51
|
memoryBytes?: number;
|
|
52
|
+
label?: string;
|
|
52
53
|
}
|
|
53
54
|
export interface ListModelsOptions {
|
|
54
55
|
type?: ModelType;
|
|
@@ -266,10 +267,15 @@ export declare function harnessPreviewModelSource(harness: Pick<Harness, 'previe
|
|
|
266
267
|
export declare function harnessTranscribe(harness: Pick<Harness, 'transcribe'>): (request: AssistantTranscribeRequest) => Promise<AssistantTranscribeResponse>;
|
|
267
268
|
export declare function harnessDetectLanguage(harness: Pick<Harness, 'detectLanguage'>): (request: AssistantDetectLanguageRequest) => Promise<AssistantDetectLanguageResponse>;
|
|
268
269
|
export declare function harnessSpeak(harness: Pick<Harness, 'speak'>): (request: AssistantSpeakRequest) => Promise<AssistantSpeakResponse>;
|
|
269
|
-
|
|
270
|
+
type SkillsRead<O, R> = (agentId?: string | null, modelName?: string | null, options?: O) => Promise<R>;
|
|
271
|
+
export declare function syncedSkills<O extends {
|
|
272
|
+
readonly refresh?: boolean;
|
|
273
|
+
}, R>(read: SkillsRead<O, R>, syncCredentials?: () => Promise<void> | undefined): SkillsRead<O, R>;
|
|
274
|
+
export declare function harnessSkillCatalog(harness: Pick<Harness, 'skills'>, syncCredentials?: () => Promise<void> | undefined): SkillsRead<SkillCatalogOptions, AssistantSkillCatalogResponse>;
|
|
270
275
|
export declare function harnessSkillManagement(harness: Pick<Harness, 'listManagedSkills' | 'readSkillFile' | 'writeSkillFile' | 'copyBaseForEdit' | 'deleteOverlaySkill'>): SkillManagementHooks;
|
|
271
276
|
export declare function harnessTools(harness: Pick<Harness, 'tools'>): () => Promise<AssistantToolCatalogResponse>;
|
|
272
277
|
export declare function toolReportEntry(tool: ToolCatalogEntry): AssistantToolReportEntry;
|
|
278
|
+
export declare function customModelReportEntries(models: ReadonlyArray<ModelInfo>): AssistantCustomModelReportEntry[];
|
|
273
279
|
export declare function harnessSpeakStream(harness: Pick<Harness, 'speakStream'>): (text: AsyncIterable<string>) => AsyncIterable<AssistantSpeakResponse>;
|
|
274
280
|
export declare function harnessDeathReason(status: unknown): string;
|
|
275
281
|
export declare function harnessDied(status: unknown): boolean;
|
|
@@ -77,16 +77,19 @@ export function harnessSpeak(harness) {
|
|
|
77
77
|
return { audio, sampleRate };
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
|
-
export function
|
|
80
|
+
export function syncedSkills(read, syncCredentials) {
|
|
81
81
|
return async (agentId, modelName, options) => {
|
|
82
82
|
if (options?.refresh) {
|
|
83
83
|
await syncCredentials?.();
|
|
84
84
|
}
|
|
85
|
-
return
|
|
86
|
-
skills: ((await harness.skills?.(agentId, modelName, options)) ?? []).map(toEngineSkill)
|
|
87
|
-
};
|
|
85
|
+
return read(agentId, modelName, options);
|
|
88
86
|
};
|
|
89
87
|
}
|
|
88
|
+
export function harnessSkillCatalog(harness, syncCredentials) {
|
|
89
|
+
return syncedSkills(async (agentId, modelName, options) => ({
|
|
90
|
+
skills: ((await harness.skills?.(agentId, modelName, options)) ?? []).map(toEngineSkill)
|
|
91
|
+
}), syncCredentials);
|
|
92
|
+
}
|
|
90
93
|
export function harnessSkillManagement(harness) {
|
|
91
94
|
return {
|
|
92
95
|
async listManagedSkills() {
|
|
@@ -135,6 +138,16 @@ export function toolReportEntry(tool) {
|
|
|
135
138
|
writeRootsRequired: tool.writeRootsRequired === true
|
|
136
139
|
};
|
|
137
140
|
}
|
|
141
|
+
export function customModelReportEntries(models) {
|
|
142
|
+
return models
|
|
143
|
+
.filter((model) => model.registrySource === 'custom')
|
|
144
|
+
.map((model) => ({
|
|
145
|
+
name: model.name,
|
|
146
|
+
label: model.label || model.name,
|
|
147
|
+
taskType: model.taskType ?? null,
|
|
148
|
+
expectedSize: model.expectedSize
|
|
149
|
+
}));
|
|
150
|
+
}
|
|
138
151
|
export function harnessSpeakStream(harness) {
|
|
139
152
|
return async function* (text) {
|
|
140
153
|
if (!harness.speakStream) {
|
|
@@ -52,7 +52,7 @@ export interface RoundReplay {
|
|
|
52
52
|
export declare function replayFromChunks(rounds: ReadonlyArray<AssistantChunk>): RoundReplay;
|
|
53
53
|
export declare function sentTurn(chunk: AssistantChunk): string | undefined;
|
|
54
54
|
export declare function runView(chunks: ReadonlyArray<AssistantChunk>, { request, runId }: RunAt): AssistantChunk[];
|
|
55
|
-
export declare function foldableView(chunks: ReadonlyArray<AssistantChunk>, run
|
|
55
|
+
export declare function foldableView(chunks: ReadonlyArray<AssistantChunk>, run?: RunAt): AssistantChunk[];
|
|
56
56
|
export declare function viewOutgrewCache(chunks: ReadonlyArray<AssistantChunk>, run: RunAt, cachedRuns: ReadonlyMap<string, unknown>): boolean;
|
|
57
57
|
export declare function normalizeQuestion(text: string): string;
|
|
58
58
|
export declare const GENERIC_ACKNOWLEDGEMENTS: Set<string>;
|
|
@@ -5,7 +5,7 @@ import { isDocumentAttachment, isPdfAttachment } from '../core/document-attachme
|
|
|
5
5
|
import { groupAttachmentsByMessage } from '../core/files/index.js';
|
|
6
6
|
import { IMAGE_MIMES } from '../core/image/normalize.js';
|
|
7
7
|
import { isVideoAttachment } from '../core/video/attachment.js';
|
|
8
|
-
import { compactionTurn, foldCompactedChunks } from '../compaction.js';
|
|
8
|
+
import { compactionTurn, foldCompactedChunks, foldPosition } from '../compaction.js';
|
|
9
9
|
import { attemptStarts, hiddenTranscriptText } from '../core/transcript-visibility.js';
|
|
10
10
|
import { acceptedRepairCount, historyBranches, isPromptChunk } from '../core/prompt-history.js';
|
|
11
11
|
import { isChatMessage } from '../run-replay.js';
|
|
@@ -114,6 +114,13 @@ export function historyFromChunks(chunks, stale = {}, { sentRuns = new Map(), as
|
|
|
114
114
|
}
|
|
115
115
|
return turn;
|
|
116
116
|
};
|
|
117
|
+
const afterRun = (runId) => {
|
|
118
|
+
let at = history.indexOf(anchor(runId)) + 1;
|
|
119
|
+
while (history[at]?.role === 'tool') {
|
|
120
|
+
at++;
|
|
121
|
+
}
|
|
122
|
+
return at;
|
|
123
|
+
};
|
|
117
124
|
for (const chunk of source) {
|
|
118
125
|
if (chunk.userMessage) {
|
|
119
126
|
if (staleQuestionChunkIds.has(chunk.id) || steered.has(chunk.id)) {
|
|
@@ -150,7 +157,7 @@ export function historyFromChunks(chunks, stale = {}, { sentRuns = new Map(), as
|
|
|
150
157
|
turn.documents = documents;
|
|
151
158
|
}
|
|
152
159
|
if (pdfs.length) {
|
|
153
|
-
turn.pdfs = pdfs;
|
|
160
|
+
turn.pdfs = pdfs.map((ref) => ({ ...ref, pages: digests.get(ref.id) }));
|
|
154
161
|
}
|
|
155
162
|
history.push(turn);
|
|
156
163
|
continue;
|
|
@@ -176,8 +183,7 @@ export function historyFromChunks(chunks, stale = {}, { sentRuns = new Map(), as
|
|
|
176
183
|
const loaded = isRecord(result) ? result.name : undefined;
|
|
177
184
|
if (typeof loaded === 'string' && !replayedSkills.has(loaded)) {
|
|
178
185
|
replayedSkills.add(loaded);
|
|
179
|
-
|
|
180
|
-
history.splice(at, 0, { role: 'tool', content: skillLoadedTurn(loaded) });
|
|
186
|
+
history.splice(afterRun(chunk.runId), 0, { role: 'tool', content: skillLoadedTurn(loaded) });
|
|
181
187
|
}
|
|
182
188
|
continue;
|
|
183
189
|
}
|
|
@@ -190,7 +196,7 @@ export function historyFromChunks(chunks, stale = {}, { sentRuns = new Map(), as
|
|
|
190
196
|
}
|
|
191
197
|
const produced = attachmentTurn(chunk.toolResult.name, chunk.toolResult.result);
|
|
192
198
|
if (produced) {
|
|
193
|
-
history.
|
|
199
|
+
history.splice(afterRun(chunk.runId), 0, { role: 'tool', content: produced });
|
|
194
200
|
}
|
|
195
201
|
continue;
|
|
196
202
|
}
|
|
@@ -287,15 +293,15 @@ export function runView(chunks, { request, runId }) {
|
|
|
287
293
|
});
|
|
288
294
|
}
|
|
289
295
|
export function foldableView(chunks, run) {
|
|
290
|
-
const view = runView(chunks, run);
|
|
296
|
+
const view = run ? runView(chunks, run) : chunks;
|
|
291
297
|
const kept = new Set(view);
|
|
292
|
-
const
|
|
293
|
-
const
|
|
294
|
-
|
|
295
|
-
chunk.runId !== run
|
|
296
|
-
!(chunk
|
|
297
|
-
const limit = chunks.reduce((low, chunk) => (caps(chunk) ? Math.min(low, chunk
|
|
298
|
-
return view.filter((chunk) => chunk.compaction || chunk
|
|
298
|
+
const settled = new Set(chunks.flatMap((chunk) => chunk.runId && TERMINAL_STATES.has(chunk.runStatus?.state ?? '') ? [chunk.runId] : []));
|
|
299
|
+
const at = foldPosition(chunks);
|
|
300
|
+
const caps = (chunk) => !chunk.compaction &&
|
|
301
|
+
chunk.runId !== run?.runId &&
|
|
302
|
+
(!kept.has(chunk) || (!!chunk.runId && !settled.has(chunk.runId)));
|
|
303
|
+
const limit = chunks.reduce((low, chunk) => (caps(chunk) ? Math.min(low, at(chunk)) : low), Infinity);
|
|
304
|
+
return view.filter((chunk) => chunk.compaction || at(chunk) < limit);
|
|
299
305
|
}
|
|
300
306
|
export function viewOutgrewCache(chunks, run, cachedRuns) {
|
|
301
307
|
const cached = chunks.findLast((chunk) => chunk.type === 'run-request' &&
|
|
@@ -13,8 +13,10 @@ export interface ChatMessage {
|
|
|
13
13
|
audioAttachments?: AttachmentRef[];
|
|
14
14
|
videos?: VideoRef[];
|
|
15
15
|
documents?: DocumentAttachment[];
|
|
16
|
-
pdfs?:
|
|
16
|
+
pdfs?: PdfRef[];
|
|
17
17
|
}
|
|
18
|
+
export declare function earlierImages(messages: ReadonlyArray<ChatMessage>): Set<string>;
|
|
19
|
+
export declare function claimNewImages(messages: unknown, sent: Set<string>): boolean;
|
|
18
20
|
export declare const IMAGE_ONLY_USER_PROMPT = "The user sent one or more images without any accompanying text. Describe what the images show and respond to anything notable in them.";
|
|
19
21
|
export declare const DOCUMENT_ONLY_USER_PROMPT = "The user sent one or more documents without any accompanying text. Summarize the attached documents.";
|
|
20
22
|
export declare const IMAGE_AND_DOCUMENT_USER_PROMPT = "The user sent one or more images together with one or more documents, and no accompanying text. Describe what the images show and summarize the attached documents.";
|
|
@@ -50,6 +52,9 @@ export interface AttachmentRef {
|
|
|
50
52
|
export interface VideoRef extends AttachmentRef {
|
|
51
53
|
digest?: AttachmentRef;
|
|
52
54
|
}
|
|
55
|
+
export interface PdfRef extends AttachmentRef {
|
|
56
|
+
pages?: AttachmentRef;
|
|
57
|
+
}
|
|
53
58
|
export interface AttachInput {
|
|
54
59
|
fileName: string;
|
|
55
60
|
mimeType: string;
|
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
import { isRecord } from '../guards.js';
|
|
2
|
+
export function earlierImages(messages) {
|
|
3
|
+
const question = messages.findLastIndex((turn) => turn.role === 'user');
|
|
4
|
+
return new Set(messages.slice(0, Math.max(question, 0)).flatMap((turn) => imagePaths([turn])));
|
|
5
|
+
}
|
|
6
|
+
export function claimNewImages(messages, sent) {
|
|
7
|
+
let fresh = false;
|
|
8
|
+
for (const path of imagePaths(messages)) {
|
|
9
|
+
fresh ||= !sent.has(path);
|
|
10
|
+
sent.add(path);
|
|
11
|
+
}
|
|
12
|
+
return fresh;
|
|
13
|
+
}
|
|
14
|
+
function imagePaths(messages) {
|
|
15
|
+
if (!Array.isArray(messages))
|
|
16
|
+
return [];
|
|
17
|
+
return messages.flatMap((turn) => isRecord(turn) && Array.isArray(turn.attachments)
|
|
18
|
+
? turn.attachments.flatMap((file) => isRecord(file) && typeof file.path === 'string' ? [file.path] : [])
|
|
19
|
+
: []);
|
|
20
|
+
}
|
|
1
21
|
export const IMAGE_ONLY_USER_PROMPT = 'The user sent one or more images without any accompanying text. Describe what the images show and respond to anything notable in them.';
|
|
2
22
|
export const DOCUMENT_ONLY_USER_PROMPT = 'The user sent one or more documents without any accompanying text. Summarize the attached documents.';
|
|
3
23
|
export const IMAGE_AND_DOCUMENT_USER_PROMPT = 'The user sent one or more images together with one or more documents, and no accompanying text. Describe what the images show and summarize the attached documents.';
|
|
@@ -42,8 +62,8 @@ export function silentAudioNotice(fileName) {
|
|
|
42
62
|
}
|
|
43
63
|
export function audioTranscriptBlock(fileName, transcript) {
|
|
44
64
|
return [
|
|
45
|
-
`The audio file "${fileName}" has already been transcribed on-device. You cannot open the file itself
|
|
46
|
-
'Transcript:',
|
|
65
|
+
`The audio file "${fileName}" has already been transcribed on-device for its speech. You cannot open the file itself, but a tool that takes an audio attachment still reads the audio.`,
|
|
66
|
+
'Transcript of the speech in it, which is its words and not any music:',
|
|
47
67
|
transcript
|
|
48
68
|
].join('\n');
|
|
49
69
|
}
|
|
@@ -3,10 +3,12 @@ import type { AbortSignal } from 'bare-abort-controller';
|
|
|
3
3
|
import type { AssistantChunk, AssistantChunkContentModelRequest, AssistantChunkDelta } from '../../spec/hyperschema/types.d.ts';
|
|
4
4
|
import type { ScopedCapabilities } from '../mesh-scoped-engine.ts';
|
|
5
5
|
import { type AttachmentRef } from './message.ts';
|
|
6
|
+
import type { PdfBounds } from '../core/pdf/figure.ts';
|
|
6
7
|
import { type ProbeKind } from '../probe-rounds.ts';
|
|
7
8
|
export interface MediaRef {
|
|
8
9
|
readonly id: string;
|
|
9
10
|
readonly page?: number | undefined;
|
|
11
|
+
readonly clip?: PdfBounds | undefined;
|
|
10
12
|
readonly frame?: number | undefined;
|
|
11
13
|
readonly frames?: number | undefined;
|
|
12
14
|
}
|
|
@@ -24,6 +26,7 @@ export interface OwnerRoundsOptions {
|
|
|
24
26
|
readonly deviceId: Buffer;
|
|
25
27
|
readonly from: number;
|
|
26
28
|
readonly media?: ((file: string) => MediaRef | undefined) | undefined;
|
|
29
|
+
readonly inspect?: ((call: Record<string, unknown>) => void) | undefined;
|
|
27
30
|
}
|
|
28
31
|
interface Echo {
|
|
29
32
|
readonly type: 'content' | 'thinking';
|
|
@@ -50,8 +53,10 @@ export declare class OwnerRounds {
|
|
|
50
53
|
private _contextSent;
|
|
51
54
|
private _rounds;
|
|
52
55
|
private _retakes;
|
|
56
|
+
private readonly _takers;
|
|
53
57
|
constructor(opts: OwnerRoundsOptions);
|
|
54
58
|
get rounds(): number;
|
|
59
|
+
ranHere(round: number | undefined): boolean;
|
|
55
60
|
retook(): boolean;
|
|
56
61
|
echoed(type: string, text: string): boolean;
|
|
57
62
|
private _round;
|