@qvac/core 1.0.0-rc.8 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +234 -3
- package/README.md +1 -0
- package/dist-lib/lib/api/attachments.d.ts +1 -1
- package/dist-lib/lib/api/attachments.js +11 -0
- package/dist-lib/lib/api/chunks.d.ts +4 -2
- package/dist-lib/lib/api/chunks.js +6 -3
- package/dist-lib/lib/api/devices.js +6 -18
- package/dist-lib/lib/api/execution.d.ts +6 -2
- package/dist-lib/lib/api/execution.js +29 -3
- package/dist-lib/lib/api/host-run-log.d.ts +7 -1
- package/dist-lib/lib/api/host-run-log.js +3 -1
- package/dist-lib/lib/api/index.d.ts +3 -1
- package/dist-lib/lib/api/index.js +17 -6
- package/dist-lib/lib/api/members.js +13 -29
- package/dist-lib/lib/api/mesh.d.ts +2 -1
- package/dist-lib/lib/api/projects.d.ts +1 -1
- package/dist-lib/lib/api/projects.js +9 -5
- package/dist-lib/lib/api/rag-label.d.ts +4 -0
- package/dist-lib/lib/api/rag-label.js +21 -0
- package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
- package/dist-lib/lib/api/rag-lexical.js +1058 -0
- package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
- package/dist-lib/lib/api/rag-manifest.js +46 -0
- package/dist-lib/lib/api/rag-outline.d.ts +36 -0
- package/dist-lib/lib/api/rag-outline.js +426 -0
- package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
- package/dist-lib/lib/api/rag-plugin.js +70 -0
- package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
- package/dist-lib/lib/api/rag-synthesis.js +102 -0
- package/dist-lib/lib/api/rag-targets.d.ts +23 -0
- package/dist-lib/lib/api/rag-targets.js +215 -0
- package/dist-lib/lib/api/rag-window.d.ts +5 -0
- package/dist-lib/lib/api/rag-window.js +46 -0
- package/dist-lib/lib/api/rag.d.ts +3 -2
- package/dist-lib/lib/api/rag.js +421 -181
- package/dist-lib/lib/api/served.js +8 -14
- package/dist-lib/lib/api/workspace.d.ts +8 -1
- package/dist-lib/lib/api/workspace.js +27 -13
- package/dist-lib/lib/assistant/harness.d.ts +9 -1
- package/dist-lib/lib/assistant/history.d.ts +11 -2
- package/dist-lib/lib/assistant/history.js +74 -4
- package/dist-lib/lib/assistant/media-round.d.ts +61 -0
- package/dist-lib/lib/assistant/media-round.js +111 -0
- package/dist-lib/lib/assistant/message.d.ts +7 -0
- package/dist-lib/lib/assistant/model-round.d.ts +12 -0
- package/dist-lib/lib/assistant/model-round.js +130 -17
- package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
- package/dist-lib/lib/assistant/pdf-pages.js +67 -3
- package/dist-lib/lib/assistant/round-host.d.ts +22 -0
- package/dist-lib/lib/assistant/round-host.js +142 -8
- package/dist-lib/lib/assistant/upload.d.ts +12 -0
- package/dist-lib/lib/assistant/upload.js +36 -0
- package/dist-lib/lib/assistant-client.d.ts +1 -0
- package/dist-lib/lib/assistant-client.js +40 -19
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +41 -1
- package/dist-lib/lib/assistant.d.ts +15 -0
- package/dist-lib/lib/assistant.js +474 -89
- package/dist-lib/lib/client.d.ts +1 -0
- package/dist-lib/lib/client.js +4 -1
- package/dist-lib/lib/compaction.js +2 -2
- package/dist-lib/lib/core/agent.js +12 -45
- package/dist-lib/lib/core/approval-rules.js +1 -8
- package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
- package/dist-lib/lib/core/archived-mesh.js +12 -8
- package/dist-lib/lib/core/chat-copy.d.ts +18 -3
- package/dist-lib/lib/core/chat-copy.js +74 -25
- package/dist-lib/lib/core/chat-fork.js +15 -6
- package/dist-lib/lib/core/chat-group.js +16 -24
- package/dist-lib/lib/core/chat-move.d.ts +9 -0
- package/dist-lib/lib/core/chat-move.js +94 -3
- package/dist-lib/lib/core/chat-order.js +12 -18
- package/dist-lib/lib/core/chat.js +38 -31
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +18 -25
- package/dist-lib/lib/core/collections.d.ts +2 -0
- package/dist-lib/lib/core/collections.js +2 -0
- package/dist-lib/lib/core/contract/activity.d.ts +4 -0
- package/dist-lib/lib/core/contract/activity.js +34 -4
- package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
- package/dist-lib/lib/core/contract/chat-group.js +21 -1
- package/dist-lib/lib/core/contract/chat.d.ts +9 -0
- package/dist-lib/lib/core/contract/chat.js +127 -5
- package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
- package/dist-lib/lib/core/contract/chunks.js +37 -6
- package/dist-lib/lib/core/contract/features.d.ts +21 -0
- package/dist-lib/lib/core/contract/features.js +22 -0
- package/dist-lib/lib/core/contract/index.js +2 -0
- package/dist-lib/lib/core/contract/indexing.js +4 -2
- package/dist-lib/lib/core/contract/knowledge.js +5 -4
- package/dist-lib/lib/core/contract/members.js +16 -10
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
- package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
- package/dist-lib/lib/core/contract/projects.d.ts +5 -0
- package/dist-lib/lib/core/contract/projects.js +13 -5
- package/dist-lib/lib/core/contract/roles.js +32 -4
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
- package/dist-lib/lib/core/contract/run-requests.js +35 -2
- package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
- package/dist-lib/lib/core/contract/tool-configs.js +3 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
- package/dist-lib/lib/core/contract/workspace.js +40 -7
- package/dist-lib/lib/core/contract/writers.js +2 -2
- package/dist-lib/lib/core/departures.d.ts +3 -1
- package/dist-lib/lib/core/departures.js +3 -0
- package/dist-lib/lib/core/devices.js +1 -8
- package/dist-lib/lib/core/files/attachments.d.ts +2 -0
- package/dist-lib/lib/core/files/attachments.js +19 -1
- package/dist-lib/lib/core/files/index.d.ts +20 -4
- package/dist-lib/lib/core/files/index.js +132 -21
- package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
- package/dist-lib/lib/core/files/processing/document.js +7 -0
- package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
- package/dist-lib/lib/core/files/processing/index.js +144 -12
- package/dist-lib/lib/core/files/processing/protocol.js +13 -34
- package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
- package/dist-lib/lib/core/files/rag/build.js +123 -19
- package/dist-lib/lib/core/files/rag/failure.js +2 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
- package/dist-lib/lib/core/files/rag/index.js +13 -1
- package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
- package/dist-lib/lib/core/files/rag/load.js +27 -6
- package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
- package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
- package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/produce.js +104 -16
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
- package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
- package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
- package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
- package/dist-lib/lib/core/files/records/knowledge.js +7 -40
- package/dist-lib/lib/core/files/records/reclaim.js +5 -10
- package/dist-lib/lib/core/files/upload.js +68 -41
- package/dist-lib/lib/core/fs-drives.js +14 -26
- package/dist-lib/lib/core/invite.d.ts +1 -0
- package/dist-lib/lib/core/invite.js +4 -2
- package/dist-lib/lib/core/local/files.d.ts +1 -1
- package/dist-lib/lib/core/local/files.js +5 -5
- package/dist-lib/lib/core/local/index.d.ts +13 -6
- package/dist-lib/lib/core/local/index.js +103 -69
- package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
- package/dist-lib/lib/core/local/run-logs.js +5 -5
- package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
- package/dist-lib/lib/core/mesh-processor.js +87 -0
- package/dist-lib/lib/core/mesh.d.ts +48 -5
- package/dist-lib/lib/core/mesh.js +318 -129
- package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
- package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
- package/dist-lib/lib/core/migrations/writer-links.js +32 -0
- package/dist-lib/lib/core/project-keys.d.ts +6 -0
- package/dist-lib/lib/core/project-keys.js +11 -0
- package/dist-lib/lib/core/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/public-mesh.js +10 -8
- package/dist-lib/lib/core/run-requests.d.ts +3 -0
- package/dist-lib/lib/core/run-requests.js +21 -1
- package/dist-lib/lib/core/skill-availability.d.ts +33 -0
- package/dist-lib/lib/core/skill-availability.js +67 -0
- package/dist-lib/lib/core/sync/background.d.ts +0 -1
- package/dist-lib/lib/core/sync/background.js +2 -1
- package/dist-lib/lib/core/sync/changed.d.ts +4 -2
- package/dist-lib/lib/core/sync/changed.js +1 -1
- package/dist-lib/lib/core/sync/index.js +52 -10
- package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
- package/dist-lib/lib/core/sync/view-updates.js +82 -91
- package/dist-lib/lib/core/sync-status.js +97 -41
- package/dist-lib/lib/core/tool-activity.js +6 -12
- package/dist-lib/lib/core/tool-configs.js +17 -29
- package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
- package/dist-lib/lib/core/upgrade-summary.js +67 -0
- package/dist-lib/lib/core/view-read.d.ts +4 -0
- package/dist-lib/lib/core/view-read.js +23 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
- package/dist-lib/lib/core/workspace-upgrade.js +27 -16
- package/dist-lib/lib/core.d.ts +39 -42
- package/dist-lib/lib/core.js +661 -639
- package/dist-lib/lib/errors.d.ts +20 -0
- package/dist-lib/lib/errors.js +20 -0
- package/dist-lib/lib/llm-model-names.d.ts +1 -1
- package/dist-lib/lib/llm-model-names.js +1 -1
- package/dist-lib/lib/mesh-scoped-engine.js +4 -0
- package/dist-lib/lib/model-context.js +1 -2
- package/dist-lib/lib/probe-rounds.d.ts +7 -2
- package/dist-lib/lib/probe-rounds.js +6 -1
- package/dist-lib/lib/run-transcript.d.ts +4 -1
- package/dist-lib/lib/run-transcript.js +15 -6
- package/dist-lib/schema/engine.js +83 -6
- package/dist-lib/schema/indexes.d.ts +3 -0
- package/dist-lib/schema/indexes.js +5 -0
- package/dist-lib/schema/local.js +11 -2
- package/dist-lib/schema/qvac.js +49 -5
- package/dist-lib/spec/bind.js +34 -0
- package/dist-lib/spec/calls.js +5 -0
- package/dist-lib/spec/capabilities.d.ts +17 -0
- package/dist-lib/spec/hrpc/hrpc.json +65 -0
- package/dist-lib/spec/hrpc/index.js +68 -5
- package/dist-lib/spec/hrpc/messages.js +260 -23
- package/dist-lib/spec/hyperschema/index.js +260 -23
- package/dist-lib/spec/hyperschema/schema.json +181 -0
- package/dist-lib/spec/hyperschema/types.d.ts +48 -0
- package/dist-lib/spec/local/hyperdb/messages.js +270 -224
- package/dist-lib/spec/local/hyperschema/index.js +73 -38
- package/dist-lib/spec/local/hyperschema/schema.json +31 -0
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
- package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
- package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
- package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
- package/package.json +75 -70
- package/tui/README.md +2 -1
- package/tui/app.mjs +28 -7
- package/tui/boot.mjs +5 -1
- package/tui/lib/attachment-files.mjs +82 -0
- package/tui/lib/reopen-meshes.mjs +15 -0
- package/tui/main.mjs +4 -43
- package/worker/assistant-sidecar.mjs +10 -3
- package/worker/stow-entry.mjs +3 -52
|
@@ -7,6 +7,7 @@ import { historyFromChunks, listHistoryChunks, replayFromChunks, runPromptView,
|
|
|
7
7
|
import { nameSenders, restoreSentContent } from './message.js';
|
|
8
8
|
import { PdfBoundsSchema } from './pdf-pages.js';
|
|
9
9
|
import { isProbeRound, PROBE_KINDS } from '../probe-rounds.js';
|
|
10
|
+
import { MODEL_MISSING, mediaOptions, mediaOutputFrom, putBlob } from './media-round.js';
|
|
10
11
|
const message = z.custom((value) => isRecord(value) && typeof value.role === 'string' && typeof value.content === 'string');
|
|
11
12
|
const record = () => z.custom((value) => isRecord(value));
|
|
12
13
|
export function wholeImageId(ref) {
|
|
@@ -32,7 +33,8 @@ const roundOptions = z.object({
|
|
|
32
33
|
responseFormat: record().optional(),
|
|
33
34
|
toolDialect: z.custom().optional(),
|
|
34
35
|
unforced: record().optional(),
|
|
35
|
-
media: z.array(z.object({ at: z.number(), refs: z.array(mediaRef) })).optional()
|
|
36
|
+
media: z.array(z.object({ at: z.number(), refs: z.array(mediaRef) })).optional(),
|
|
37
|
+
mediaRound: mediaOptions.optional()
|
|
36
38
|
});
|
|
37
39
|
const roundResult = z.custom((value) => isRecord(value) && typeof value.contentText === 'string' && isRecord(value.final));
|
|
38
40
|
export class Echoes {
|
|
@@ -55,7 +57,9 @@ export class Echoes {
|
|
|
55
57
|
}
|
|
56
58
|
export class OwnerRounds {
|
|
57
59
|
port;
|
|
60
|
+
media;
|
|
58
61
|
inputLength = 0;
|
|
62
|
+
roundBase = 0;
|
|
59
63
|
share = null;
|
|
60
64
|
_opts;
|
|
61
65
|
_echoes = new Echoes();
|
|
@@ -64,9 +68,11 @@ export class OwnerRounds {
|
|
|
64
68
|
_rounds = 0;
|
|
65
69
|
_retakes = 0;
|
|
66
70
|
_takers = new Map();
|
|
71
|
+
_generated = [];
|
|
67
72
|
constructor(opts) {
|
|
68
73
|
this._opts = opts;
|
|
69
74
|
this.port = (request, io) => this._round(request, io);
|
|
75
|
+
this.media = (request, io) => this._generate(request, io);
|
|
70
76
|
}
|
|
71
77
|
get rounds() {
|
|
72
78
|
return this._rounds;
|
|
@@ -151,7 +157,7 @@ export class OwnerRounds {
|
|
|
151
157
|
const { modelId: asked, generationParams } = options;
|
|
152
158
|
const call = { kind, modelId: asked, messages, generationParams, responseFormat };
|
|
153
159
|
try {
|
|
154
|
-
const { contentText } = await this._probed({ id, agentId }, signal);
|
|
160
|
+
const { contentText } = await this._probed({ id, agentId }, signal, (result) => roundResult.parse(result));
|
|
155
161
|
this._opts.inspect?.({ ...call, outcome: 'completed', result: contentText });
|
|
156
162
|
return contentText;
|
|
157
163
|
}
|
|
@@ -160,7 +166,56 @@ export class OwnerRounds {
|
|
|
160
166
|
throw err;
|
|
161
167
|
}
|
|
162
168
|
}
|
|
163
|
-
async
|
|
169
|
+
async _generate(request, io) {
|
|
170
|
+
if (io.signal.aborted)
|
|
171
|
+
throw new AbortedError();
|
|
172
|
+
const { engine, chatId, runId, deviceId, meshId, local } = this._opts;
|
|
173
|
+
const agentId = (await this._switchedTo()) ?? this._opts.agentId;
|
|
174
|
+
const { modelChoice, params } = request;
|
|
175
|
+
const refs = this._named(request.sources);
|
|
176
|
+
const sources = refs || !meshId
|
|
177
|
+
? undefined
|
|
178
|
+
: await Promise.all((request.images ?? []).map((bytes) => putBlob(engine, meshId, bytes)));
|
|
179
|
+
const options = {
|
|
180
|
+
kind: request.kind,
|
|
181
|
+
attempt: 0,
|
|
182
|
+
modelId: modelChoice ?? request.kind,
|
|
183
|
+
continuationNudge: '',
|
|
184
|
+
deliveredNudge: '',
|
|
185
|
+
mediaRound: { modelChoice, params, sources, refs }
|
|
186
|
+
};
|
|
187
|
+
const { id } = await engine.appendChunk({
|
|
188
|
+
chatId,
|
|
189
|
+
runId,
|
|
190
|
+
type: 'model-request',
|
|
191
|
+
originDeviceId: deviceId,
|
|
192
|
+
modelRequest: { round: 0, agents: [agentId], options, state: 'open' }
|
|
193
|
+
});
|
|
194
|
+
const settled = await this._probed({ id, agentId }, io.signal, (result) => result);
|
|
195
|
+
const { result, generated } = await mediaOutputFrom(engine, settled, io.signal);
|
|
196
|
+
if ('error' in result && result.code === MODEL_MISSING && local) {
|
|
197
|
+
return local(request, io);
|
|
198
|
+
}
|
|
199
|
+
if (generated) {
|
|
200
|
+
this._generated.push(generated);
|
|
201
|
+
}
|
|
202
|
+
return result;
|
|
203
|
+
}
|
|
204
|
+
generated(bytes) {
|
|
205
|
+
const at = this._generated.findIndex((output) => b4a.equals(output.bytes, bytes));
|
|
206
|
+
if (at === -1) {
|
|
207
|
+
return undefined;
|
|
208
|
+
}
|
|
209
|
+
return this._generated.splice(at, 1)[0].blobId;
|
|
210
|
+
}
|
|
211
|
+
_named(sources) {
|
|
212
|
+
const refs = (sources ?? []).flatMap(({ attachmentId, path, width, height }) => {
|
|
213
|
+
const id = attachmentId ?? (path ? wholeImageId(this._opts.media?.(path)) : undefined);
|
|
214
|
+
return id ? [{ attachmentId: id, width, height }] : [];
|
|
215
|
+
});
|
|
216
|
+
return refs.length && refs.length === sources?.length ? refs : undefined;
|
|
217
|
+
}
|
|
218
|
+
async _probed(asked, signal, parse) {
|
|
164
219
|
const { engine, chatId, from } = this._opts;
|
|
165
220
|
const watch = engine.chunksDeltaWatch({ chatId, gte: from || undefined });
|
|
166
221
|
const agents = engine.agentsWatch({});
|
|
@@ -173,9 +228,8 @@ export class OwnerRounds {
|
|
|
173
228
|
for await (const frame of watch) {
|
|
174
229
|
round = frame.chunks?.find((chunk) => chunk.id === asked.id)?.modelRequest ?? round;
|
|
175
230
|
ended = !!round && round.state !== 'open' && round.state !== 'taken';
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
return result;
|
|
231
|
+
if (roundDone(round)) {
|
|
232
|
+
return parse(round?.result);
|
|
179
233
|
}
|
|
180
234
|
}
|
|
181
235
|
throw new AbortedError();
|
|
@@ -185,22 +239,27 @@ export class OwnerRounds {
|
|
|
185
239
|
watch.destroy();
|
|
186
240
|
agents.destroy();
|
|
187
241
|
if (!ended) {
|
|
242
|
+
this._opts.log?.info('rounds', 'probe cancelled', {
|
|
243
|
+
roundId: asked.id,
|
|
244
|
+
reason: signal.aborted ? 'aborted' : 'watch ended'
|
|
245
|
+
});
|
|
188
246
|
await engine.cancelModelRequest({ chunkId: asked.id }).catch(noop);
|
|
189
247
|
}
|
|
190
248
|
}
|
|
191
249
|
}
|
|
192
250
|
async _dropOnHostLoss(agents, asked, round) {
|
|
193
|
-
let reachable = false;
|
|
194
251
|
for await (const _frame of agents) {
|
|
195
252
|
const agent = await this._opts.engine.getAgent({ agentId: asked.agentId }).catch(() => null);
|
|
196
253
|
if (!agent) {
|
|
197
254
|
continue;
|
|
198
255
|
}
|
|
199
|
-
const was = reachable;
|
|
200
|
-
reachable = !!agent.hostReachable;
|
|
201
256
|
const host = round()?.hostDeviceId;
|
|
202
257
|
const moved = !!host && !b4a.equals(agent.providerDeviceId, host);
|
|
203
|
-
if (moved ||
|
|
258
|
+
if (moved || !agent.hostReachable) {
|
|
259
|
+
this._opts.log?.info('rounds', 'probe cancelled', {
|
|
260
|
+
roundId: asked.id,
|
|
261
|
+
reason: moved ? 'provider moved' : 'host lost'
|
|
262
|
+
});
|
|
204
263
|
await this._opts.engine.cancelModelRequest({ chunkId: asked.id }).catch(noop);
|
|
205
264
|
return;
|
|
206
265
|
}
|
|
@@ -218,7 +277,7 @@ export class OwnerRounds {
|
|
|
218
277
|
type: 'model-request',
|
|
219
278
|
originDeviceId: deviceId,
|
|
220
279
|
modelRequest: {
|
|
221
|
-
round: sent.round,
|
|
280
|
+
round: sent.round + this.roundBase,
|
|
222
281
|
agents: [agentId],
|
|
223
282
|
cacheKey: sent.cacheKey,
|
|
224
283
|
promptPrefix: sent.promptPrefix,
|
|
@@ -227,6 +286,14 @@ export class OwnerRounds {
|
|
|
227
286
|
state: 'open'
|
|
228
287
|
}
|
|
229
288
|
});
|
|
289
|
+
this._opts.log?.info('rounds', 'round asked', {
|
|
290
|
+
runId,
|
|
291
|
+
roundId: id,
|
|
292
|
+
chatId,
|
|
293
|
+
mesh: this._opts.meshId?.slice(0, 8),
|
|
294
|
+
agentId,
|
|
295
|
+
round: sent.round + this.roundBase
|
|
296
|
+
});
|
|
230
297
|
return id;
|
|
231
298
|
}
|
|
232
299
|
async _modelOf(agentId, modelId) {
|
|
@@ -251,7 +318,10 @@ export class OwnerRounds {
|
|
|
251
318
|
runId,
|
|
252
319
|
type: 'context',
|
|
253
320
|
originDeviceId: deviceId,
|
|
254
|
-
context: {
|
|
321
|
+
context: {
|
|
322
|
+
turn: { sent: sent?.content, shots: this._pdfShots(sent) },
|
|
323
|
+
toolSetHash: request.toolSet ?? null
|
|
324
|
+
}
|
|
255
325
|
});
|
|
256
326
|
}
|
|
257
327
|
async _options(request, system) {
|
|
@@ -277,6 +347,17 @@ export class OwnerRounds {
|
|
|
277
347
|
}
|
|
278
348
|
return options;
|
|
279
349
|
}
|
|
350
|
+
_pdfShots(turn) {
|
|
351
|
+
return (turn?.attachments ?? []).flatMap(({ path }) => {
|
|
352
|
+
const ref = this._opts.media?.(path);
|
|
353
|
+
if (ref?.page === undefined || ref.frame !== undefined) {
|
|
354
|
+
return [];
|
|
355
|
+
}
|
|
356
|
+
return [
|
|
357
|
+
ref.clip ? { id: ref.id, page: ref.page, clip: ref.clip } : { id: ref.id, page: ref.page }
|
|
358
|
+
];
|
|
359
|
+
});
|
|
360
|
+
}
|
|
280
361
|
_namedImages(turn) {
|
|
281
362
|
const imageIds = (turn.attachments ?? []).flatMap(({ path }) => wholeImageId(this._opts.media?.(path)) ?? []);
|
|
282
363
|
return imageIds.length ? { ...turn, imageIds } : turn;
|
|
@@ -327,9 +408,27 @@ export class OwnerRounds {
|
|
|
327
408
|
await engine.cancelModelRequest({ chunkId: at.asked.id }).catch(noop);
|
|
328
409
|
};
|
|
329
410
|
void this._followHost(agents, { at, output, move }).catch(noop);
|
|
411
|
+
const since = Date.now();
|
|
412
|
+
let takenAt;
|
|
330
413
|
try {
|
|
331
414
|
for await (const frame of watch) {
|
|
332
415
|
const round = output.apply(frame);
|
|
416
|
+
if (round?.state === 'taken' && round.takenBy && round.takenAt !== takenAt) {
|
|
417
|
+
takenAt = round.takenAt;
|
|
418
|
+
this._opts.log?.debug('rounds', 'round taken', {
|
|
419
|
+
roundId: at.asked.id,
|
|
420
|
+
takenBy: shortId(round.takenBy)
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
if (!at.moving && round && round.state !== 'open' && round.state !== 'taken') {
|
|
424
|
+
this._opts.log?.info('rounds', 'round ended', {
|
|
425
|
+
roundId: at.asked.id,
|
|
426
|
+
state: round.state,
|
|
427
|
+
takenBy: round.takenBy ? shortId(round.takenBy) : undefined,
|
|
428
|
+
host: round.hostDeviceId ? shortId(round.hostDeviceId) : undefined,
|
|
429
|
+
ms: Date.now() - since
|
|
430
|
+
});
|
|
431
|
+
}
|
|
333
432
|
const result = at.moving ? undefined : roundEnd(round);
|
|
334
433
|
if (result) {
|
|
335
434
|
this._takers.set(sent.round, round?.takenBy ?? null);
|
|
@@ -374,12 +473,23 @@ export class OwnerRounds {
|
|
|
374
473
|
if (provider !== b4a.toHex(host)) {
|
|
375
474
|
if (at.host.moved !== provider) {
|
|
376
475
|
at.host.moved = provider;
|
|
476
|
+
this._opts.log?.info('rounds', 'round moved', {
|
|
477
|
+
roundId: at.asked.id,
|
|
478
|
+
reason: 'provider moved',
|
|
479
|
+
host: shortId(host),
|
|
480
|
+
provider: provider.slice(0, 8)
|
|
481
|
+
});
|
|
377
482
|
await move(agentId);
|
|
378
483
|
}
|
|
379
484
|
continue;
|
|
380
485
|
}
|
|
381
486
|
const lost = was?.provider === provider && was.reachable && !at.host.reachable;
|
|
382
487
|
if (lost && output.round()?.state === 'taken') {
|
|
488
|
+
this._opts.log?.info('rounds', 'round reopened', {
|
|
489
|
+
roundId: at.asked.id,
|
|
490
|
+
reason: 'host lost',
|
|
491
|
+
host: provider.slice(0, 8)
|
|
492
|
+
});
|
|
383
493
|
await this._opts.engine.reopenModelRequest({ chunkId: at.asked.id }).catch(noop);
|
|
384
494
|
}
|
|
385
495
|
}
|
|
@@ -631,17 +741,17 @@ function carried(rounds, promptPrefix, toolSet) {
|
|
|
631
741
|
}
|
|
632
742
|
return { system, tools: toolSet ? sets.resolve(toolSet) : undefined };
|
|
633
743
|
}
|
|
634
|
-
function
|
|
635
|
-
if (round?.state === 'done') {
|
|
636
|
-
return roundResult.parse(round.result);
|
|
637
|
-
}
|
|
744
|
+
function roundDone(round) {
|
|
638
745
|
if (round?.state === 'failed') {
|
|
639
746
|
throw new Error(round.reason || 'the model round failed');
|
|
640
747
|
}
|
|
641
748
|
if (round?.state === 'cancelled') {
|
|
642
749
|
throw new AbortedError();
|
|
643
750
|
}
|
|
644
|
-
return
|
|
751
|
+
return round?.state === 'done';
|
|
752
|
+
}
|
|
753
|
+
function roundEnd(round) {
|
|
754
|
+
return roundDone(round) ? roundResult.parse(round?.result) : undefined;
|
|
645
755
|
}
|
|
646
756
|
function lastSwitch(chunks, from) {
|
|
647
757
|
let last;
|
|
@@ -653,3 +763,6 @@ function lastSwitch(chunks, from) {
|
|
|
653
763
|
return last?.agentSwitch?.agentId;
|
|
654
764
|
}
|
|
655
765
|
function noop() { }
|
|
766
|
+
function shortId(key) {
|
|
767
|
+
return b4a.toHex(key).slice(0, 8);
|
|
768
|
+
}
|
|
@@ -13,6 +13,17 @@ export interface PdfSelection {
|
|
|
13
13
|
readonly shots: ReadonlyArray<PdfShot>;
|
|
14
14
|
readonly cutAt?: number;
|
|
15
15
|
}
|
|
16
|
+
interface PdfHit {
|
|
17
|
+
readonly metadataJson: string;
|
|
18
|
+
}
|
|
19
|
+
interface PdfTextHit extends PdfHit {
|
|
20
|
+
readonly content: string;
|
|
21
|
+
}
|
|
22
|
+
export interface PdfSelectionOptions {
|
|
23
|
+
readonly cited?: ReadonlySet<number>;
|
|
24
|
+
readonly textFirst?: boolean;
|
|
25
|
+
readonly readablePages?: ReadonlySet<number> | null;
|
|
26
|
+
}
|
|
16
27
|
export declare const PdfBoundsSchema: z.ZodObject<{
|
|
17
28
|
x: z.ZodNumber;
|
|
18
29
|
y: z.ZodNumber;
|
|
@@ -22,4 +33,8 @@ export declare const PdfBoundsSchema: z.ZodObject<{
|
|
|
22
33
|
export declare function pdfSelectionFileName(pdfFileName: string): string;
|
|
23
34
|
export declare function packPdfSelection(selection: PdfSelection): Uint8Array;
|
|
24
35
|
export declare function parsePdfSelection(bytes: Uint8Array): PdfSelection;
|
|
25
|
-
export declare function
|
|
36
|
+
export declare function asksForPdfVisuals(question: string): boolean;
|
|
37
|
+
export declare function readablePdfTextPages(hits: ReadonlyArray<PdfTextHit>): Map<string, Set<number> | null>;
|
|
38
|
+
export declare function citedPdfPages(hits: ReadonlyArray<PdfHit>): Map<string, Set<number> | null>;
|
|
39
|
+
export declare function selectPdfPages(doc: PdfDoc, room: PdfRoom, { cited, textFirst, readablePages }?: PdfSelectionOptions): Promise<PdfSelection>;
|
|
40
|
+
export {};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
|
+
import { parseRagMetadata } from 'rag-extract';
|
|
2
3
|
import * as z from 'zod';
|
|
3
4
|
import { frameTokens } from './video-budget.js';
|
|
4
5
|
export const PDF_PAGES_MIME = 'application/vnd.qvac.pdf-pages';
|
|
@@ -31,14 +32,74 @@ export function parsePdfSelection(bytes) {
|
|
|
31
32
|
function shotLabel(page, names = []) {
|
|
32
33
|
return names.length ? `page ${page + 1} (${names.join(', ')})` : `page ${page + 1}`;
|
|
33
34
|
}
|
|
34
|
-
|
|
35
|
+
const VISUAL_REQUEST = /\b(?:charts?|figures?|diagrams?|graphs?|plots?|maps?|images?|pictures?|photos?|illustrations?|visuals?|layouts?|handwriting|handwritten|scanned|screenshots?)\b|\b(?:this|that|the|attached)\s+scan\b/i;
|
|
36
|
+
const TABLE_REFERENCE = /\b(?:the|this|that|attached|above|below|following|all|any|each)\s+tables?\b|\btables?\s*(?:#?\s*\d+|[A-Z]\b|of|on|from|above|below)\b|\b(?:read|show|describe|explain|extract|summari[sz]e|analy[sz]e|find|inspect|compare)\s+(?:me\s+)?(?:(?:all|any|each)\s+)?(?:the\s+|this\s+|that\s+)?tables?\b/i;
|
|
37
|
+
const TABLE_MENTION = /\btables?\b/i;
|
|
38
|
+
const TABLE_OUTPUT = /\b(?:in|as|into)\s+(?:a\s+|the\s+)?table\b|\b(?:make|create|generate|compile|write|produce)\b[^.!?\n]{0,40}\b(?:a\s+|an\s+)?table\b/i;
|
|
39
|
+
export function asksForPdfVisuals(question) {
|
|
40
|
+
if (VISUAL_REQUEST.test(question))
|
|
41
|
+
return true;
|
|
42
|
+
const tableQuestion = question.replace(TABLE_OUTPUT, ' ');
|
|
43
|
+
if (TABLE_REFERENCE.test(tableQuestion)) {
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
return TABLE_MENTION.test(question) && !TABLE_OUTPUT.test(question);
|
|
47
|
+
}
|
|
48
|
+
export function readablePdfTextPages(hits) {
|
|
49
|
+
const readable = new Map();
|
|
50
|
+
for (const { content, metadataJson } of hits) {
|
|
51
|
+
if (typeof content !== 'string')
|
|
52
|
+
continue;
|
|
53
|
+
const metadata = parseRagMetadata(metadataJson);
|
|
54
|
+
const text = content.normalize('NFKC').replace(/\s+/gu, ' ').trim();
|
|
55
|
+
if (!metadata?.attachmentId || !hasReadableContent(text, metadata.tokenCount)) {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const pages = readable.get(metadata.attachmentId);
|
|
59
|
+
if (metadata.page === null || pages === null) {
|
|
60
|
+
readable.set(metadata.attachmentId, null);
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
readable.set(metadata.attachmentId, (pages ?? new Set()).add(metadata.page));
|
|
64
|
+
}
|
|
65
|
+
return readable;
|
|
66
|
+
}
|
|
67
|
+
function hasReadableContent(text, cachedChunkTokens) {
|
|
68
|
+
const words = text.match(/[\p{L}\p{N}]{2,}/gu) ?? [];
|
|
69
|
+
if (text.length < 24 ||
|
|
70
|
+
words.length < 4 ||
|
|
71
|
+
/^(?:files in this chat:|document outline(?:\b|$)|readable,\s+\d+\s+(?:pages?|lines?)(?:\b|$))/i.test(text)) {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
return text.length >= 80 || cachedChunkTokens >= 24;
|
|
75
|
+
}
|
|
76
|
+
export function citedPdfPages(hits) {
|
|
77
|
+
const cited = new Map();
|
|
78
|
+
for (const { metadataJson } of hits) {
|
|
79
|
+
const metadata = parseRagMetadata(metadataJson);
|
|
80
|
+
if (!metadata?.attachmentId) {
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const pages = cited.get(metadata.attachmentId);
|
|
84
|
+
if (metadata.page === null || pages === null) {
|
|
85
|
+
cited.set(metadata.attachmentId, null);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
cited.set(metadata.attachmentId, (pages ?? new Set()).add(metadata.page));
|
|
89
|
+
}
|
|
90
|
+
return cited;
|
|
91
|
+
}
|
|
92
|
+
export async function selectPdfPages(doc, room, { cited, textFirst = false, readablePages } = {}) {
|
|
35
93
|
const pageCount = doc.pageCount();
|
|
36
94
|
const shots = [];
|
|
37
95
|
let cutAt;
|
|
38
96
|
for (let page = 0; page < pageCount; page++) {
|
|
97
|
+
if (cited && !cited.has(page + 1)) {
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
39
100
|
let shot;
|
|
40
101
|
try {
|
|
41
|
-
shot = await pictured(doc, page);
|
|
102
|
+
shot = await pictured(doc, page, { textFirst, readablePages });
|
|
42
103
|
}
|
|
43
104
|
catch {
|
|
44
105
|
continue;
|
|
@@ -60,8 +121,11 @@ export async function selectPdfPages(doc, room) {
|
|
|
60
121
|
}
|
|
61
122
|
return cutAt === undefined ? { pageCount, shots } : { pageCount, shots, cutAt };
|
|
62
123
|
}
|
|
63
|
-
async function pictured(doc, page) {
|
|
124
|
+
async function pictured(doc, page, { textFirst, readablePages }) {
|
|
64
125
|
const flags = doc.pageFlags(page);
|
|
126
|
+
if (textFirst && (readablePages?.has(page + 1) || (readablePages === null && flags.hasText))) {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
65
129
|
if (flags.hasText && !flags.hasImage) {
|
|
66
130
|
return null;
|
|
67
131
|
}
|
|
@@ -1,17 +1,24 @@
|
|
|
1
1
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { ModelRoundRequest } from '@qvac/harness';
|
|
3
|
+
import type { MediaRoundPort } from '@qvac/harness/lib/contract/index.ts';
|
|
2
4
|
import type { LoggerInterface } from '@qvac/logging';
|
|
3
5
|
import type { AssistantChunk, AssistantChunkList } from '../../spec/hyperschema/types.d.ts';
|
|
4
6
|
import type { Capabilities, WatchStream } from '../api/index.ts';
|
|
7
|
+
import { type ScopedCapabilities } from '../mesh-scoped-engine.ts';
|
|
5
8
|
import type { Harness } from './harness.ts';
|
|
9
|
+
import type { MediaKind } from '../probe-rounds.ts';
|
|
6
10
|
import { type MediaSource } from './model-round.ts';
|
|
7
11
|
export interface RoundHostOptions {
|
|
8
12
|
readonly engine: Capabilities;
|
|
9
13
|
readonly serve: NonNullable<Harness['serveModelRound']>;
|
|
10
14
|
readonly deviceId: () => Buffer;
|
|
11
15
|
readonly media?: RoundMedia | null;
|
|
16
|
+
readonly load?: RoundLoad | null;
|
|
17
|
+
readonly generation?: RoundGeneration | null;
|
|
12
18
|
readonly built?: (meshId: string, chatId: string) => void;
|
|
13
19
|
readonly log?: LoggerInterface | null;
|
|
14
20
|
}
|
|
21
|
+
export type RoundLoad = (engine: ScopedCapabilities, agentId: string) => Promise<ModelRoundRequest['load']>;
|
|
15
22
|
export interface RoundMedia {
|
|
16
23
|
resolve(sources: ReadonlyArray<MediaSource>, served: {
|
|
17
24
|
readonly id: string;
|
|
@@ -20,6 +27,16 @@ export interface RoundMedia {
|
|
|
20
27
|
}): Promise<Array<string | null>>;
|
|
21
28
|
release(id: string): void;
|
|
22
29
|
}
|
|
30
|
+
export interface GenerationWait {
|
|
31
|
+
readonly engine: ScopedCapabilities;
|
|
32
|
+
readonly agentId: string;
|
|
33
|
+
readonly kind: MediaKind;
|
|
34
|
+
readonly signal: AbortSignal;
|
|
35
|
+
}
|
|
36
|
+
export interface RoundGeneration {
|
|
37
|
+
readonly serve: MediaRoundPort;
|
|
38
|
+
ready(wait: GenerationWait): Promise<boolean>;
|
|
39
|
+
}
|
|
23
40
|
export declare function servedCacheKey(round: AssistantChunk, ownerKey: string | null | undefined): string | undefined;
|
|
24
41
|
export declare class RoundHost {
|
|
25
42
|
private readonly _opts;
|
|
@@ -31,14 +48,19 @@ export declare class RoundHost {
|
|
|
31
48
|
left(chatId: string, agentId: string): string[];
|
|
32
49
|
holds(chatId: string): boolean;
|
|
33
50
|
stop(): void;
|
|
51
|
+
drop(meshId: string): void;
|
|
34
52
|
private _reconcile;
|
|
35
53
|
private _pick;
|
|
36
54
|
private _serve;
|
|
37
55
|
private _take;
|
|
38
56
|
private _provides;
|
|
57
|
+
private _load;
|
|
39
58
|
private _answer;
|
|
59
|
+
private _generate;
|
|
60
|
+
private _stored;
|
|
40
61
|
private _probe;
|
|
41
62
|
private _claimKey;
|
|
42
63
|
private _retake;
|
|
43
64
|
private _takeFor;
|
|
65
|
+
private _settle;
|
|
44
66
|
}
|