@qvac/core 1.0.0-rc.7 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +241 -3
- package/README.md +1 -0
- package/dist-lib/lib/api/attachments.d.ts +1 -1
- package/dist-lib/lib/api/attachments.js +11 -0
- package/dist-lib/lib/api/chunks.d.ts +4 -2
- package/dist-lib/lib/api/chunks.js +6 -3
- package/dist-lib/lib/api/device-routes.d.ts +6 -1
- package/dist-lib/lib/api/device-routes.js +20 -1
- package/dist-lib/lib/api/devices.js +6 -18
- package/dist-lib/lib/api/execution.d.ts +6 -2
- package/dist-lib/lib/api/execution.js +29 -3
- package/dist-lib/lib/api/host-run-log.d.ts +7 -1
- package/dist-lib/lib/api/host-run-log.js +3 -1
- package/dist-lib/lib/api/index.d.ts +3 -1
- package/dist-lib/lib/api/index.js +17 -6
- package/dist-lib/lib/api/members.js +13 -29
- package/dist-lib/lib/api/mesh.d.ts +2 -1
- package/dist-lib/lib/api/projects.d.ts +1 -1
- package/dist-lib/lib/api/projects.js +9 -5
- package/dist-lib/lib/api/rag-label.d.ts +4 -0
- package/dist-lib/lib/api/rag-label.js +21 -0
- package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
- package/dist-lib/lib/api/rag-lexical.js +1058 -0
- package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
- package/dist-lib/lib/api/rag-manifest.js +46 -0
- package/dist-lib/lib/api/rag-outline.d.ts +36 -0
- package/dist-lib/lib/api/rag-outline.js +426 -0
- package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
- package/dist-lib/lib/api/rag-plugin.js +70 -0
- package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
- package/dist-lib/lib/api/rag-synthesis.js +102 -0
- package/dist-lib/lib/api/rag-targets.d.ts +23 -0
- package/dist-lib/lib/api/rag-targets.js +215 -0
- package/dist-lib/lib/api/rag-window.d.ts +5 -0
- package/dist-lib/lib/api/rag-window.js +46 -0
- package/dist-lib/lib/api/rag.d.ts +3 -2
- package/dist-lib/lib/api/rag.js +421 -184
- package/dist-lib/lib/api/served.js +8 -14
- package/dist-lib/lib/api/workspace.d.ts +8 -1
- package/dist-lib/lib/api/workspace.js +27 -13
- package/dist-lib/lib/assistant/harness.d.ts +9 -1
- package/dist-lib/lib/assistant/history.d.ts +11 -2
- package/dist-lib/lib/assistant/history.js +74 -4
- package/dist-lib/lib/assistant/media-round.d.ts +61 -0
- package/dist-lib/lib/assistant/media-round.js +111 -0
- package/dist-lib/lib/assistant/message.d.ts +7 -0
- package/dist-lib/lib/assistant/model-round.d.ts +19 -1
- package/dist-lib/lib/assistant/model-round.js +163 -28
- package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
- package/dist-lib/lib/assistant/pdf-pages.js +67 -3
- package/dist-lib/lib/assistant/round-host.d.ts +22 -0
- package/dist-lib/lib/assistant/round-host.js +142 -8
- package/dist-lib/lib/assistant/upload.d.ts +12 -0
- package/dist-lib/lib/assistant/upload.js +36 -0
- package/dist-lib/lib/assistant-client.d.ts +1 -0
- package/dist-lib/lib/assistant-client.js +40 -19
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +48 -1
- package/dist-lib/lib/assistant.d.ts +18 -1
- package/dist-lib/lib/assistant.js +484 -95
- package/dist-lib/lib/client.d.ts +1 -0
- package/dist-lib/lib/client.js +4 -1
- package/dist-lib/lib/compaction.js +2 -2
- package/dist-lib/lib/core/agent.js +12 -45
- package/dist-lib/lib/core/approval-rules.js +1 -8
- package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
- package/dist-lib/lib/core/archived-mesh.js +12 -8
- package/dist-lib/lib/core/chat-copy.d.ts +18 -3
- package/dist-lib/lib/core/chat-copy.js +74 -25
- package/dist-lib/lib/core/chat-fork.js +15 -6
- package/dist-lib/lib/core/chat-group.js +16 -24
- package/dist-lib/lib/core/chat-move.d.ts +9 -0
- package/dist-lib/lib/core/chat-move.js +94 -3
- package/dist-lib/lib/core/chat-order.js +12 -18
- package/dist-lib/lib/core/chat.js +38 -31
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +18 -25
- package/dist-lib/lib/core/collections.d.ts +2 -0
- package/dist-lib/lib/core/collections.js +2 -0
- package/dist-lib/lib/core/contract/activity.d.ts +4 -0
- package/dist-lib/lib/core/contract/activity.js +34 -4
- package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
- package/dist-lib/lib/core/contract/chat-group.js +21 -1
- package/dist-lib/lib/core/contract/chat.d.ts +9 -0
- package/dist-lib/lib/core/contract/chat.js +127 -5
- package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
- package/dist-lib/lib/core/contract/chunks.js +37 -6
- package/dist-lib/lib/core/contract/features.d.ts +21 -0
- package/dist-lib/lib/core/contract/features.js +22 -0
- package/dist-lib/lib/core/contract/index.js +2 -0
- package/dist-lib/lib/core/contract/indexing.js +4 -2
- package/dist-lib/lib/core/contract/knowledge.js +5 -4
- package/dist-lib/lib/core/contract/members.js +16 -10
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
- package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
- package/dist-lib/lib/core/contract/projects.d.ts +5 -0
- package/dist-lib/lib/core/contract/projects.js +13 -5
- package/dist-lib/lib/core/contract/roles.js +32 -4
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
- package/dist-lib/lib/core/contract/run-requests.js +35 -2
- package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
- package/dist-lib/lib/core/contract/tool-configs.js +3 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
- package/dist-lib/lib/core/contract/workspace.js +40 -7
- package/dist-lib/lib/core/contract/writers.js +2 -2
- package/dist-lib/lib/core/departures.d.ts +3 -1
- package/dist-lib/lib/core/departures.js +3 -0
- package/dist-lib/lib/core/devices.js +1 -8
- package/dist-lib/lib/core/files/attachments.d.ts +2 -0
- package/dist-lib/lib/core/files/attachments.js +19 -1
- package/dist-lib/lib/core/files/index.d.ts +20 -4
- package/dist-lib/lib/core/files/index.js +132 -21
- package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
- package/dist-lib/lib/core/files/processing/document.js +7 -0
- package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
- package/dist-lib/lib/core/files/processing/index.js +144 -12
- package/dist-lib/lib/core/files/processing/protocol.js +13 -34
- package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
- package/dist-lib/lib/core/files/rag/build.js +123 -19
- package/dist-lib/lib/core/files/rag/failure.js +2 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
- package/dist-lib/lib/core/files/rag/index.js +13 -1
- package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
- package/dist-lib/lib/core/files/rag/load.js +27 -6
- package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
- package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
- package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/produce.js +104 -16
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
- package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
- package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
- package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
- package/dist-lib/lib/core/files/records/knowledge.js +7 -40
- package/dist-lib/lib/core/files/records/reclaim.js +5 -10
- package/dist-lib/lib/core/files/upload.js +68 -41
- package/dist-lib/lib/core/fs-drives.js +14 -26
- package/dist-lib/lib/core/invite.d.ts +1 -0
- package/dist-lib/lib/core/invite.js +4 -2
- package/dist-lib/lib/core/local/files.d.ts +1 -1
- package/dist-lib/lib/core/local/files.js +5 -5
- package/dist-lib/lib/core/local/index.d.ts +13 -6
- package/dist-lib/lib/core/local/index.js +103 -69
- package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
- package/dist-lib/lib/core/local/run-logs.js +5 -5
- package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
- package/dist-lib/lib/core/mesh-processor.js +87 -0
- package/dist-lib/lib/core/mesh.d.ts +48 -5
- package/dist-lib/lib/core/mesh.js +318 -129
- package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
- package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
- package/dist-lib/lib/core/migrations/writer-links.js +32 -0
- package/dist-lib/lib/core/project-keys.d.ts +6 -0
- package/dist-lib/lib/core/project-keys.js +11 -0
- package/dist-lib/lib/core/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/public-mesh.js +10 -8
- package/dist-lib/lib/core/run-requests.d.ts +3 -0
- package/dist-lib/lib/core/run-requests.js +21 -1
- package/dist-lib/lib/core/skill-availability.d.ts +33 -0
- package/dist-lib/lib/core/skill-availability.js +67 -0
- package/dist-lib/lib/core/sync/background.d.ts +0 -1
- package/dist-lib/lib/core/sync/background.js +2 -1
- package/dist-lib/lib/core/sync/changed.d.ts +4 -2
- package/dist-lib/lib/core/sync/changed.js +1 -1
- package/dist-lib/lib/core/sync/index.js +52 -10
- package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
- package/dist-lib/lib/core/sync/view-updates.js +82 -91
- package/dist-lib/lib/core/sync-status.js +97 -41
- package/dist-lib/lib/core/tool-activity.js +6 -12
- package/dist-lib/lib/core/tool-configs.js +17 -29
- package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
- package/dist-lib/lib/core/upgrade-summary.js +67 -0
- package/dist-lib/lib/core/view-read.d.ts +4 -0
- package/dist-lib/lib/core/view-read.js +23 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
- package/dist-lib/lib/core/workspace-upgrade.js +27 -16
- package/dist-lib/lib/core.d.ts +45 -42
- package/dist-lib/lib/core.js +675 -640
- package/dist-lib/lib/errors.d.ts +24 -0
- package/dist-lib/lib/errors.js +24 -0
- package/dist-lib/lib/llm-model-names.d.ts +1 -1
- package/dist-lib/lib/llm-model-names.js +1 -1
- package/dist-lib/lib/mesh-scoped-engine.js +4 -0
- package/dist-lib/lib/model-context.js +1 -2
- package/dist-lib/lib/model-download.d.ts +6 -1
- package/dist-lib/lib/model-download.js +16 -3
- package/dist-lib/lib/oauth/resolve.js +29 -8
- package/dist-lib/lib/oauth/token-exchange-core.js +11 -0
- package/dist-lib/lib/probe-rounds.d.ts +7 -2
- package/dist-lib/lib/probe-rounds.js +6 -1
- package/dist-lib/lib/run-transcript.d.ts +4 -1
- package/dist-lib/lib/run-transcript.js +15 -6
- package/dist-lib/schema/engine.js +83 -6
- package/dist-lib/schema/indexes.d.ts +3 -0
- package/dist-lib/schema/indexes.js +5 -0
- package/dist-lib/schema/local.js +11 -2
- package/dist-lib/schema/qvac.js +49 -5
- package/dist-lib/spec/bind.js +34 -0
- package/dist-lib/spec/calls.js +5 -0
- package/dist-lib/spec/capabilities.d.ts +17 -0
- package/dist-lib/spec/hrpc/hrpc.json +65 -0
- package/dist-lib/spec/hrpc/index.js +68 -5
- package/dist-lib/spec/hrpc/messages.js +260 -23
- package/dist-lib/spec/hyperschema/index.js +260 -23
- package/dist-lib/spec/hyperschema/schema.json +181 -0
- package/dist-lib/spec/hyperschema/types.d.ts +48 -0
- package/dist-lib/spec/local/hyperdb/messages.js +270 -224
- package/dist-lib/spec/local/hyperschema/index.js +73 -38
- package/dist-lib/spec/local/hyperschema/schema.json +31 -0
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
- package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
- package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
- package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
- package/package.json +82 -71
- package/tui/README.md +52 -0
- package/tui/app.mjs +69 -12
- package/tui/boot.mjs +27 -6
- package/tui/cli.mjs +93 -0
- package/tui/index.mjs +2 -87
- package/tui/lib/attachment-files.mjs +82 -0
- package/tui/lib/reopen-meshes.mjs +15 -0
- package/tui/lib/updater.mjs +86 -0
- package/tui/main.mjs +29 -57
- package/worker/assistant-sidecar.mjs +10 -3
- package/worker/stow-entry.mjs +3 -52
- package/dist-lib/lib/core/chat-project.d.ts +0 -2
- package/dist-lib/lib/core/chat-project.js +0 -15
|
@@ -7,6 +7,7 @@ import { historyFromChunks, listHistoryChunks, replayFromChunks, runPromptView,
|
|
|
7
7
|
import { nameSenders, restoreSentContent } from './message.js';
|
|
8
8
|
import { PdfBoundsSchema } from './pdf-pages.js';
|
|
9
9
|
import { isProbeRound, PROBE_KINDS } from '../probe-rounds.js';
|
|
10
|
+
import { MODEL_MISSING, mediaOptions, mediaOutputFrom, putBlob } from './media-round.js';
|
|
10
11
|
const message = z.custom((value) => isRecord(value) && typeof value.role === 'string' && typeof value.content === 'string');
|
|
11
12
|
const record = () => z.custom((value) => isRecord(value));
|
|
12
13
|
export function wholeImageId(ref) {
|
|
@@ -32,23 +33,46 @@ const roundOptions = z.object({
|
|
|
32
33
|
responseFormat: record().optional(),
|
|
33
34
|
toolDialect: z.custom().optional(),
|
|
34
35
|
unforced: record().optional(),
|
|
35
|
-
media: z.array(z.object({ at: z.number(), refs: z.array(mediaRef) })).optional()
|
|
36
|
+
media: z.array(z.object({ at: z.number(), refs: z.array(mediaRef) })).optional(),
|
|
37
|
+
mediaRound: mediaOptions.optional()
|
|
36
38
|
});
|
|
37
39
|
const roundResult = z.custom((value) => isRecord(value) && typeof value.contentText === 'string' && isRecord(value.final));
|
|
40
|
+
export class Echoes {
|
|
41
|
+
_pending = { content: '', thinking: '' };
|
|
42
|
+
forwarded({ type, text }) {
|
|
43
|
+
this._pending[type] += text;
|
|
44
|
+
}
|
|
45
|
+
own(type, text) {
|
|
46
|
+
const pending = this._pending[type];
|
|
47
|
+
if (pending.startsWith(text)) {
|
|
48
|
+
this._pending[type] = pending.slice(text.length);
|
|
49
|
+
return '';
|
|
50
|
+
}
|
|
51
|
+
if (pending && text.startsWith(pending)) {
|
|
52
|
+
this._pending[type] = '';
|
|
53
|
+
return text.slice(pending.length);
|
|
54
|
+
}
|
|
55
|
+
return text;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
38
58
|
export class OwnerRounds {
|
|
39
59
|
port;
|
|
60
|
+
media;
|
|
40
61
|
inputLength = 0;
|
|
62
|
+
roundBase = 0;
|
|
41
63
|
share = null;
|
|
42
64
|
_opts;
|
|
43
|
-
_echoes =
|
|
65
|
+
_echoes = new Echoes();
|
|
44
66
|
_known = null;
|
|
45
67
|
_contextSent = false;
|
|
46
68
|
_rounds = 0;
|
|
47
69
|
_retakes = 0;
|
|
48
70
|
_takers = new Map();
|
|
71
|
+
_generated = [];
|
|
49
72
|
constructor(opts) {
|
|
50
73
|
this._opts = opts;
|
|
51
74
|
this.port = (request, io) => this._round(request, io);
|
|
75
|
+
this.media = (request, io) => this._generate(request, io);
|
|
52
76
|
}
|
|
53
77
|
get rounds() {
|
|
54
78
|
return this._rounds;
|
|
@@ -65,13 +89,8 @@ export class OwnerRounds {
|
|
|
65
89
|
this._retakes--;
|
|
66
90
|
return true;
|
|
67
91
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
if (at === -1) {
|
|
71
|
-
return false;
|
|
72
|
-
}
|
|
73
|
-
this._echoes.splice(at, 1);
|
|
74
|
-
return true;
|
|
92
|
+
unechoed(type, text) {
|
|
93
|
+
return this._echoes.own(type, text);
|
|
75
94
|
}
|
|
76
95
|
async retire() {
|
|
77
96
|
const { engine, chatId, runId } = this._opts;
|
|
@@ -138,7 +157,7 @@ export class OwnerRounds {
|
|
|
138
157
|
const { modelId: asked, generationParams } = options;
|
|
139
158
|
const call = { kind, modelId: asked, messages, generationParams, responseFormat };
|
|
140
159
|
try {
|
|
141
|
-
const { contentText } = await this._probed({ id, agentId }, signal);
|
|
160
|
+
const { contentText } = await this._probed({ id, agentId }, signal, (result) => roundResult.parse(result));
|
|
142
161
|
this._opts.inspect?.({ ...call, outcome: 'completed', result: contentText });
|
|
143
162
|
return contentText;
|
|
144
163
|
}
|
|
@@ -147,7 +166,56 @@ export class OwnerRounds {
|
|
|
147
166
|
throw err;
|
|
148
167
|
}
|
|
149
168
|
}
|
|
150
|
-
async
|
|
169
|
+
async _generate(request, io) {
|
|
170
|
+
if (io.signal.aborted)
|
|
171
|
+
throw new AbortedError();
|
|
172
|
+
const { engine, chatId, runId, deviceId, meshId, local } = this._opts;
|
|
173
|
+
const agentId = (await this._switchedTo()) ?? this._opts.agentId;
|
|
174
|
+
const { modelChoice, params } = request;
|
|
175
|
+
const refs = this._named(request.sources);
|
|
176
|
+
const sources = refs || !meshId
|
|
177
|
+
? undefined
|
|
178
|
+
: await Promise.all((request.images ?? []).map((bytes) => putBlob(engine, meshId, bytes)));
|
|
179
|
+
const options = {
|
|
180
|
+
kind: request.kind,
|
|
181
|
+
attempt: 0,
|
|
182
|
+
modelId: modelChoice ?? request.kind,
|
|
183
|
+
continuationNudge: '',
|
|
184
|
+
deliveredNudge: '',
|
|
185
|
+
mediaRound: { modelChoice, params, sources, refs }
|
|
186
|
+
};
|
|
187
|
+
const { id } = await engine.appendChunk({
|
|
188
|
+
chatId,
|
|
189
|
+
runId,
|
|
190
|
+
type: 'model-request',
|
|
191
|
+
originDeviceId: deviceId,
|
|
192
|
+
modelRequest: { round: 0, agents: [agentId], options, state: 'open' }
|
|
193
|
+
});
|
|
194
|
+
const settled = await this._probed({ id, agentId }, io.signal, (result) => result);
|
|
195
|
+
const { result, generated } = await mediaOutputFrom(engine, settled, io.signal);
|
|
196
|
+
if ('error' in result && result.code === MODEL_MISSING && local) {
|
|
197
|
+
return local(request, io);
|
|
198
|
+
}
|
|
199
|
+
if (generated) {
|
|
200
|
+
this._generated.push(generated);
|
|
201
|
+
}
|
|
202
|
+
return result;
|
|
203
|
+
}
|
|
204
|
+
generated(bytes) {
|
|
205
|
+
const at = this._generated.findIndex((output) => b4a.equals(output.bytes, bytes));
|
|
206
|
+
if (at === -1) {
|
|
207
|
+
return undefined;
|
|
208
|
+
}
|
|
209
|
+
return this._generated.splice(at, 1)[0].blobId;
|
|
210
|
+
}
|
|
211
|
+
_named(sources) {
|
|
212
|
+
const refs = (sources ?? []).flatMap(({ attachmentId, path, width, height }) => {
|
|
213
|
+
const id = attachmentId ?? (path ? wholeImageId(this._opts.media?.(path)) : undefined);
|
|
214
|
+
return id ? [{ attachmentId: id, width, height }] : [];
|
|
215
|
+
});
|
|
216
|
+
return refs.length && refs.length === sources?.length ? refs : undefined;
|
|
217
|
+
}
|
|
218
|
+
async _probed(asked, signal, parse) {
|
|
151
219
|
const { engine, chatId, from } = this._opts;
|
|
152
220
|
const watch = engine.chunksDeltaWatch({ chatId, gte: from || undefined });
|
|
153
221
|
const agents = engine.agentsWatch({});
|
|
@@ -160,9 +228,8 @@ export class OwnerRounds {
|
|
|
160
228
|
for await (const frame of watch) {
|
|
161
229
|
round = frame.chunks?.find((chunk) => chunk.id === asked.id)?.modelRequest ?? round;
|
|
162
230
|
ended = !!round && round.state !== 'open' && round.state !== 'taken';
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
return result;
|
|
231
|
+
if (roundDone(round)) {
|
|
232
|
+
return parse(round?.result);
|
|
166
233
|
}
|
|
167
234
|
}
|
|
168
235
|
throw new AbortedError();
|
|
@@ -172,22 +239,27 @@ export class OwnerRounds {
|
|
|
172
239
|
watch.destroy();
|
|
173
240
|
agents.destroy();
|
|
174
241
|
if (!ended) {
|
|
242
|
+
this._opts.log?.info('rounds', 'probe cancelled', {
|
|
243
|
+
roundId: asked.id,
|
|
244
|
+
reason: signal.aborted ? 'aborted' : 'watch ended'
|
|
245
|
+
});
|
|
175
246
|
await engine.cancelModelRequest({ chunkId: asked.id }).catch(noop);
|
|
176
247
|
}
|
|
177
248
|
}
|
|
178
249
|
}
|
|
179
250
|
async _dropOnHostLoss(agents, asked, round) {
|
|
180
|
-
let reachable = false;
|
|
181
251
|
for await (const _frame of agents) {
|
|
182
252
|
const agent = await this._opts.engine.getAgent({ agentId: asked.agentId }).catch(() => null);
|
|
183
253
|
if (!agent) {
|
|
184
254
|
continue;
|
|
185
255
|
}
|
|
186
|
-
const was = reachable;
|
|
187
|
-
reachable = !!agent.hostReachable;
|
|
188
256
|
const host = round()?.hostDeviceId;
|
|
189
257
|
const moved = !!host && !b4a.equals(agent.providerDeviceId, host);
|
|
190
|
-
if (moved ||
|
|
258
|
+
if (moved || !agent.hostReachable) {
|
|
259
|
+
this._opts.log?.info('rounds', 'probe cancelled', {
|
|
260
|
+
roundId: asked.id,
|
|
261
|
+
reason: moved ? 'provider moved' : 'host lost'
|
|
262
|
+
});
|
|
191
263
|
await this._opts.engine.cancelModelRequest({ chunkId: asked.id }).catch(noop);
|
|
192
264
|
return;
|
|
193
265
|
}
|
|
@@ -205,7 +277,7 @@ export class OwnerRounds {
|
|
|
205
277
|
type: 'model-request',
|
|
206
278
|
originDeviceId: deviceId,
|
|
207
279
|
modelRequest: {
|
|
208
|
-
round: sent.round,
|
|
280
|
+
round: sent.round + this.roundBase,
|
|
209
281
|
agents: [agentId],
|
|
210
282
|
cacheKey: sent.cacheKey,
|
|
211
283
|
promptPrefix: sent.promptPrefix,
|
|
@@ -214,6 +286,14 @@ export class OwnerRounds {
|
|
|
214
286
|
state: 'open'
|
|
215
287
|
}
|
|
216
288
|
});
|
|
289
|
+
this._opts.log?.info('rounds', 'round asked', {
|
|
290
|
+
runId,
|
|
291
|
+
roundId: id,
|
|
292
|
+
chatId,
|
|
293
|
+
mesh: this._opts.meshId?.slice(0, 8),
|
|
294
|
+
agentId,
|
|
295
|
+
round: sent.round + this.roundBase
|
|
296
|
+
});
|
|
217
297
|
return id;
|
|
218
298
|
}
|
|
219
299
|
async _modelOf(agentId, modelId) {
|
|
@@ -238,7 +318,10 @@ export class OwnerRounds {
|
|
|
238
318
|
runId,
|
|
239
319
|
type: 'context',
|
|
240
320
|
originDeviceId: deviceId,
|
|
241
|
-
context: {
|
|
321
|
+
context: {
|
|
322
|
+
turn: { sent: sent?.content, shots: this._pdfShots(sent) },
|
|
323
|
+
toolSetHash: request.toolSet ?? null
|
|
324
|
+
}
|
|
242
325
|
});
|
|
243
326
|
}
|
|
244
327
|
async _options(request, system) {
|
|
@@ -264,6 +347,17 @@ export class OwnerRounds {
|
|
|
264
347
|
}
|
|
265
348
|
return options;
|
|
266
349
|
}
|
|
350
|
+
_pdfShots(turn) {
|
|
351
|
+
return (turn?.attachments ?? []).flatMap(({ path }) => {
|
|
352
|
+
const ref = this._opts.media?.(path);
|
|
353
|
+
if (ref?.page === undefined || ref.frame !== undefined) {
|
|
354
|
+
return [];
|
|
355
|
+
}
|
|
356
|
+
return [
|
|
357
|
+
ref.clip ? { id: ref.id, page: ref.page, clip: ref.clip } : { id: ref.id, page: ref.page }
|
|
358
|
+
];
|
|
359
|
+
});
|
|
360
|
+
}
|
|
267
361
|
_namedImages(turn) {
|
|
268
362
|
const imageIds = (turn.attachments ?? []).flatMap(({ path }) => wholeImageId(this._opts.media?.(path)) ?? []);
|
|
269
363
|
return imageIds.length ? { ...turn, imageIds } : turn;
|
|
@@ -305,17 +399,36 @@ export class OwnerRounds {
|
|
|
305
399
|
io.signal.addEventListener('abort', abort);
|
|
306
400
|
const at = { asked: first, moving: null, host: null };
|
|
307
401
|
const output = new RoundOutput(first.id, io, {
|
|
308
|
-
forwarded: (echo) => this._echoes.
|
|
402
|
+
forwarded: (echo) => this._echoes.forwarded(echo),
|
|
309
403
|
retook: () => this._retakes++
|
|
310
404
|
});
|
|
311
405
|
const move = async (agentId) => {
|
|
312
406
|
at.moving = { agentId, after: output.newest(), emit: io.retake() };
|
|
407
|
+
output.hold();
|
|
313
408
|
await engine.cancelModelRequest({ chunkId: at.asked.id }).catch(noop);
|
|
314
409
|
};
|
|
315
410
|
void this._followHost(agents, { at, output, move }).catch(noop);
|
|
411
|
+
const since = Date.now();
|
|
412
|
+
let takenAt;
|
|
316
413
|
try {
|
|
317
414
|
for await (const frame of watch) {
|
|
318
415
|
const round = output.apply(frame);
|
|
416
|
+
if (round?.state === 'taken' && round.takenBy && round.takenAt !== takenAt) {
|
|
417
|
+
takenAt = round.takenAt;
|
|
418
|
+
this._opts.log?.debug('rounds', 'round taken', {
|
|
419
|
+
roundId: at.asked.id,
|
|
420
|
+
takenBy: shortId(round.takenBy)
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
if (!at.moving && round && round.state !== 'open' && round.state !== 'taken') {
|
|
424
|
+
this._opts.log?.info('rounds', 'round ended', {
|
|
425
|
+
roundId: at.asked.id,
|
|
426
|
+
state: round.state,
|
|
427
|
+
takenBy: round.takenBy ? shortId(round.takenBy) : undefined,
|
|
428
|
+
host: round.hostDeviceId ? shortId(round.hostDeviceId) : undefined,
|
|
429
|
+
ms: Date.now() - since
|
|
430
|
+
});
|
|
431
|
+
}
|
|
319
432
|
const result = at.moving ? undefined : roundEnd(round);
|
|
320
433
|
if (result) {
|
|
321
434
|
this._takers.set(sent.round, round?.takenBy ?? null);
|
|
@@ -360,12 +473,23 @@ export class OwnerRounds {
|
|
|
360
473
|
if (provider !== b4a.toHex(host)) {
|
|
361
474
|
if (at.host.moved !== provider) {
|
|
362
475
|
at.host.moved = provider;
|
|
476
|
+
this._opts.log?.info('rounds', 'round moved', {
|
|
477
|
+
roundId: at.asked.id,
|
|
478
|
+
reason: 'provider moved',
|
|
479
|
+
host: shortId(host),
|
|
480
|
+
provider: provider.slice(0, 8)
|
|
481
|
+
});
|
|
363
482
|
await move(agentId);
|
|
364
483
|
}
|
|
365
484
|
continue;
|
|
366
485
|
}
|
|
367
486
|
const lost = was?.provider === provider && was.reachable && !at.host.reachable;
|
|
368
487
|
if (lost && output.round()?.state === 'taken') {
|
|
488
|
+
this._opts.log?.info('rounds', 'round reopened', {
|
|
489
|
+
roundId: at.asked.id,
|
|
490
|
+
reason: 'host lost',
|
|
491
|
+
host: provider.slice(0, 8)
|
|
492
|
+
});
|
|
369
493
|
await this._opts.engine.reopenModelRequest({ chunkId: at.asked.id }).catch(noop);
|
|
370
494
|
}
|
|
371
495
|
}
|
|
@@ -416,6 +540,9 @@ export class RoundOutput {
|
|
|
416
540
|
}
|
|
417
541
|
return round;
|
|
418
542
|
}
|
|
543
|
+
hold() {
|
|
544
|
+
this._emit = null;
|
|
545
|
+
}
|
|
419
546
|
reask(id, emit) {
|
|
420
547
|
this._id = id;
|
|
421
548
|
this._emit = emit;
|
|
@@ -463,16 +590,21 @@ export class RoundOutput {
|
|
|
463
590
|
this._takenAt = now ? at : this._takenAt;
|
|
464
591
|
}
|
|
465
592
|
_forward(chunk) {
|
|
593
|
+
const emit = this._emit;
|
|
466
594
|
const text = chunk.content?.text ?? chunk.thinking?.text;
|
|
467
595
|
const type = chunk.type;
|
|
468
596
|
const fresh = chunk.parentId === this._id && !this._emitted.has(chunk.id) && chunk.seq > this._resetAt;
|
|
469
597
|
const taken = !this._reopened && this._taker && b4a.equals(chunk.originDeviceId, this._taker);
|
|
470
|
-
if (!
|
|
598
|
+
if (!emit ||
|
|
599
|
+
!fresh ||
|
|
600
|
+
!taken ||
|
|
601
|
+
text === undefined ||
|
|
602
|
+
(type !== 'content' && type !== 'thinking')) {
|
|
471
603
|
return;
|
|
472
604
|
}
|
|
473
605
|
this._emitted.add(chunk.id);
|
|
474
606
|
this._hooks.forwarded({ type, text });
|
|
475
|
-
|
|
607
|
+
emit({ type, text });
|
|
476
608
|
}
|
|
477
609
|
}
|
|
478
610
|
export async function listRoundChunks(engine, chatId) {
|
|
@@ -609,17 +741,17 @@ function carried(rounds, promptPrefix, toolSet) {
|
|
|
609
741
|
}
|
|
610
742
|
return { system, tools: toolSet ? sets.resolve(toolSet) : undefined };
|
|
611
743
|
}
|
|
612
|
-
function
|
|
613
|
-
if (round?.state === 'done') {
|
|
614
|
-
return roundResult.parse(round.result);
|
|
615
|
-
}
|
|
744
|
+
function roundDone(round) {
|
|
616
745
|
if (round?.state === 'failed') {
|
|
617
746
|
throw new Error(round.reason || 'the model round failed');
|
|
618
747
|
}
|
|
619
748
|
if (round?.state === 'cancelled') {
|
|
620
749
|
throw new AbortedError();
|
|
621
750
|
}
|
|
622
|
-
return
|
|
751
|
+
return round?.state === 'done';
|
|
752
|
+
}
|
|
753
|
+
function roundEnd(round) {
|
|
754
|
+
return roundDone(round) ? roundResult.parse(round?.result) : undefined;
|
|
623
755
|
}
|
|
624
756
|
function lastSwitch(chunks, from) {
|
|
625
757
|
let last;
|
|
@@ -631,3 +763,6 @@ function lastSwitch(chunks, from) {
|
|
|
631
763
|
return last?.agentSwitch?.agentId;
|
|
632
764
|
}
|
|
633
765
|
function noop() { }
|
|
766
|
+
function shortId(key) {
|
|
767
|
+
return b4a.toHex(key).slice(0, 8);
|
|
768
|
+
}
|
|
@@ -13,6 +13,17 @@ export interface PdfSelection {
|
|
|
13
13
|
readonly shots: ReadonlyArray<PdfShot>;
|
|
14
14
|
readonly cutAt?: number;
|
|
15
15
|
}
|
|
16
|
+
interface PdfHit {
|
|
17
|
+
readonly metadataJson: string;
|
|
18
|
+
}
|
|
19
|
+
interface PdfTextHit extends PdfHit {
|
|
20
|
+
readonly content: string;
|
|
21
|
+
}
|
|
22
|
+
export interface PdfSelectionOptions {
|
|
23
|
+
readonly cited?: ReadonlySet<number>;
|
|
24
|
+
readonly textFirst?: boolean;
|
|
25
|
+
readonly readablePages?: ReadonlySet<number> | null;
|
|
26
|
+
}
|
|
16
27
|
export declare const PdfBoundsSchema: z.ZodObject<{
|
|
17
28
|
x: z.ZodNumber;
|
|
18
29
|
y: z.ZodNumber;
|
|
@@ -22,4 +33,8 @@ export declare const PdfBoundsSchema: z.ZodObject<{
|
|
|
22
33
|
export declare function pdfSelectionFileName(pdfFileName: string): string;
|
|
23
34
|
export declare function packPdfSelection(selection: PdfSelection): Uint8Array;
|
|
24
35
|
export declare function parsePdfSelection(bytes: Uint8Array): PdfSelection;
|
|
25
|
-
export declare function
|
|
36
|
+
export declare function asksForPdfVisuals(question: string): boolean;
|
|
37
|
+
export declare function readablePdfTextPages(hits: ReadonlyArray<PdfTextHit>): Map<string, Set<number> | null>;
|
|
38
|
+
export declare function citedPdfPages(hits: ReadonlyArray<PdfHit>): Map<string, Set<number> | null>;
|
|
39
|
+
export declare function selectPdfPages(doc: PdfDoc, room: PdfRoom, { cited, textFirst, readablePages }?: PdfSelectionOptions): Promise<PdfSelection>;
|
|
40
|
+
export {};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
|
+
import { parseRagMetadata } from 'rag-extract';
|
|
2
3
|
import * as z from 'zod';
|
|
3
4
|
import { frameTokens } from './video-budget.js';
|
|
4
5
|
export const PDF_PAGES_MIME = 'application/vnd.qvac.pdf-pages';
|
|
@@ -31,14 +32,74 @@ export function parsePdfSelection(bytes) {
|
|
|
31
32
|
function shotLabel(page, names = []) {
|
|
32
33
|
return names.length ? `page ${page + 1} (${names.join(', ')})` : `page ${page + 1}`;
|
|
33
34
|
}
|
|
34
|
-
|
|
35
|
+
const VISUAL_REQUEST = /\b(?:charts?|figures?|diagrams?|graphs?|plots?|maps?|images?|pictures?|photos?|illustrations?|visuals?|layouts?|handwriting|handwritten|scanned|screenshots?)\b|\b(?:this|that|the|attached)\s+scan\b/i;
|
|
36
|
+
const TABLE_REFERENCE = /\b(?:the|this|that|attached|above|below|following|all|any|each)\s+tables?\b|\btables?\s*(?:#?\s*\d+|[A-Z]\b|of|on|from|above|below)\b|\b(?:read|show|describe|explain|extract|summari[sz]e|analy[sz]e|find|inspect|compare)\s+(?:me\s+)?(?:(?:all|any|each)\s+)?(?:the\s+|this\s+|that\s+)?tables?\b/i;
|
|
37
|
+
const TABLE_MENTION = /\btables?\b/i;
|
|
38
|
+
const TABLE_OUTPUT = /\b(?:in|as|into)\s+(?:a\s+|the\s+)?table\b|\b(?:make|create|generate|compile|write|produce)\b[^.!?\n]{0,40}\b(?:a\s+|an\s+)?table\b/i;
|
|
39
|
+
export function asksForPdfVisuals(question) {
|
|
40
|
+
if (VISUAL_REQUEST.test(question))
|
|
41
|
+
return true;
|
|
42
|
+
const tableQuestion = question.replace(TABLE_OUTPUT, ' ');
|
|
43
|
+
if (TABLE_REFERENCE.test(tableQuestion)) {
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
return TABLE_MENTION.test(question) && !TABLE_OUTPUT.test(question);
|
|
47
|
+
}
|
|
48
|
+
export function readablePdfTextPages(hits) {
|
|
49
|
+
const readable = new Map();
|
|
50
|
+
for (const { content, metadataJson } of hits) {
|
|
51
|
+
if (typeof content !== 'string')
|
|
52
|
+
continue;
|
|
53
|
+
const metadata = parseRagMetadata(metadataJson);
|
|
54
|
+
const text = content.normalize('NFKC').replace(/\s+/gu, ' ').trim();
|
|
55
|
+
if (!metadata?.attachmentId || !hasReadableContent(text, metadata.tokenCount)) {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const pages = readable.get(metadata.attachmentId);
|
|
59
|
+
if (metadata.page === null || pages === null) {
|
|
60
|
+
readable.set(metadata.attachmentId, null);
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
readable.set(metadata.attachmentId, (pages ?? new Set()).add(metadata.page));
|
|
64
|
+
}
|
|
65
|
+
return readable;
|
|
66
|
+
}
|
|
67
|
+
function hasReadableContent(text, cachedChunkTokens) {
|
|
68
|
+
const words = text.match(/[\p{L}\p{N}]{2,}/gu) ?? [];
|
|
69
|
+
if (text.length < 24 ||
|
|
70
|
+
words.length < 4 ||
|
|
71
|
+
/^(?:files in this chat:|document outline(?:\b|$)|readable,\s+\d+\s+(?:pages?|lines?)(?:\b|$))/i.test(text)) {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
return text.length >= 80 || cachedChunkTokens >= 24;
|
|
75
|
+
}
|
|
76
|
+
export function citedPdfPages(hits) {
|
|
77
|
+
const cited = new Map();
|
|
78
|
+
for (const { metadataJson } of hits) {
|
|
79
|
+
const metadata = parseRagMetadata(metadataJson);
|
|
80
|
+
if (!metadata?.attachmentId) {
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const pages = cited.get(metadata.attachmentId);
|
|
84
|
+
if (metadata.page === null || pages === null) {
|
|
85
|
+
cited.set(metadata.attachmentId, null);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
cited.set(metadata.attachmentId, (pages ?? new Set()).add(metadata.page));
|
|
89
|
+
}
|
|
90
|
+
return cited;
|
|
91
|
+
}
|
|
92
|
+
export async function selectPdfPages(doc, room, { cited, textFirst = false, readablePages } = {}) {
|
|
35
93
|
const pageCount = doc.pageCount();
|
|
36
94
|
const shots = [];
|
|
37
95
|
let cutAt;
|
|
38
96
|
for (let page = 0; page < pageCount; page++) {
|
|
97
|
+
if (cited && !cited.has(page + 1)) {
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
39
100
|
let shot;
|
|
40
101
|
try {
|
|
41
|
-
shot = await pictured(doc, page);
|
|
102
|
+
shot = await pictured(doc, page, { textFirst, readablePages });
|
|
42
103
|
}
|
|
43
104
|
catch {
|
|
44
105
|
continue;
|
|
@@ -60,8 +121,11 @@ export async function selectPdfPages(doc, room) {
|
|
|
60
121
|
}
|
|
61
122
|
return cutAt === undefined ? { pageCount, shots } : { pageCount, shots, cutAt };
|
|
62
123
|
}
|
|
63
|
-
async function pictured(doc, page) {
|
|
124
|
+
async function pictured(doc, page, { textFirst, readablePages }) {
|
|
64
125
|
const flags = doc.pageFlags(page);
|
|
126
|
+
if (textFirst && (readablePages?.has(page + 1) || (readablePages === null && flags.hasText))) {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
65
129
|
if (flags.hasText && !flags.hasImage) {
|
|
66
130
|
return null;
|
|
67
131
|
}
|
|
@@ -1,17 +1,24 @@
|
|
|
1
1
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { ModelRoundRequest } from '@qvac/harness';
|
|
3
|
+
import type { MediaRoundPort } from '@qvac/harness/lib/contract/index.ts';
|
|
2
4
|
import type { LoggerInterface } from '@qvac/logging';
|
|
3
5
|
import type { AssistantChunk, AssistantChunkList } from '../../spec/hyperschema/types.d.ts';
|
|
4
6
|
import type { Capabilities, WatchStream } from '../api/index.ts';
|
|
7
|
+
import { type ScopedCapabilities } from '../mesh-scoped-engine.ts';
|
|
5
8
|
import type { Harness } from './harness.ts';
|
|
9
|
+
import type { MediaKind } from '../probe-rounds.ts';
|
|
6
10
|
import { type MediaSource } from './model-round.ts';
|
|
7
11
|
export interface RoundHostOptions {
|
|
8
12
|
readonly engine: Capabilities;
|
|
9
13
|
readonly serve: NonNullable<Harness['serveModelRound']>;
|
|
10
14
|
readonly deviceId: () => Buffer;
|
|
11
15
|
readonly media?: RoundMedia | null;
|
|
16
|
+
readonly load?: RoundLoad | null;
|
|
17
|
+
readonly generation?: RoundGeneration | null;
|
|
12
18
|
readonly built?: (meshId: string, chatId: string) => void;
|
|
13
19
|
readonly log?: LoggerInterface | null;
|
|
14
20
|
}
|
|
21
|
+
export type RoundLoad = (engine: ScopedCapabilities, agentId: string) => Promise<ModelRoundRequest['load']>;
|
|
15
22
|
export interface RoundMedia {
|
|
16
23
|
resolve(sources: ReadonlyArray<MediaSource>, served: {
|
|
17
24
|
readonly id: string;
|
|
@@ -20,6 +27,16 @@ export interface RoundMedia {
|
|
|
20
27
|
}): Promise<Array<string | null>>;
|
|
21
28
|
release(id: string): void;
|
|
22
29
|
}
|
|
30
|
+
export interface GenerationWait {
|
|
31
|
+
readonly engine: ScopedCapabilities;
|
|
32
|
+
readonly agentId: string;
|
|
33
|
+
readonly kind: MediaKind;
|
|
34
|
+
readonly signal: AbortSignal;
|
|
35
|
+
}
|
|
36
|
+
export interface RoundGeneration {
|
|
37
|
+
readonly serve: MediaRoundPort;
|
|
38
|
+
ready(wait: GenerationWait): Promise<boolean>;
|
|
39
|
+
}
|
|
23
40
|
export declare function servedCacheKey(round: AssistantChunk, ownerKey: string | null | undefined): string | undefined;
|
|
24
41
|
export declare class RoundHost {
|
|
25
42
|
private readonly _opts;
|
|
@@ -31,14 +48,19 @@ export declare class RoundHost {
|
|
|
31
48
|
left(chatId: string, agentId: string): string[];
|
|
32
49
|
holds(chatId: string): boolean;
|
|
33
50
|
stop(): void;
|
|
51
|
+
drop(meshId: string): void;
|
|
34
52
|
private _reconcile;
|
|
35
53
|
private _pick;
|
|
36
54
|
private _serve;
|
|
37
55
|
private _take;
|
|
38
56
|
private _provides;
|
|
57
|
+
private _load;
|
|
39
58
|
private _answer;
|
|
59
|
+
private _generate;
|
|
60
|
+
private _stored;
|
|
40
61
|
private _probe;
|
|
41
62
|
private _claimKey;
|
|
42
63
|
private _retake;
|
|
43
64
|
private _takeFor;
|
|
65
|
+
private _settle;
|
|
44
66
|
}
|