@qvac/core 1.0.0-rc.8 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (226) hide show
  1. package/CHANGELOG.md +243 -3
  2. package/README.md +1 -0
  3. package/dist-lib/lib/api/attachments.d.ts +1 -1
  4. package/dist-lib/lib/api/attachments.js +11 -0
  5. package/dist-lib/lib/api/chunks.d.ts +4 -2
  6. package/dist-lib/lib/api/chunks.js +6 -3
  7. package/dist-lib/lib/api/devices.js +6 -18
  8. package/dist-lib/lib/api/execution.d.ts +6 -2
  9. package/dist-lib/lib/api/execution.js +29 -3
  10. package/dist-lib/lib/api/host-run-log.d.ts +7 -1
  11. package/dist-lib/lib/api/host-run-log.js +3 -1
  12. package/dist-lib/lib/api/index.d.ts +3 -1
  13. package/dist-lib/lib/api/index.js +17 -6
  14. package/dist-lib/lib/api/members.js +13 -29
  15. package/dist-lib/lib/api/mesh.d.ts +2 -1
  16. package/dist-lib/lib/api/projects.d.ts +1 -1
  17. package/dist-lib/lib/api/projects.js +9 -5
  18. package/dist-lib/lib/api/rag-label.d.ts +4 -0
  19. package/dist-lib/lib/api/rag-label.js +21 -0
  20. package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
  21. package/dist-lib/lib/api/rag-lexical.js +1058 -0
  22. package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
  23. package/dist-lib/lib/api/rag-manifest.js +46 -0
  24. package/dist-lib/lib/api/rag-outline.d.ts +36 -0
  25. package/dist-lib/lib/api/rag-outline.js +426 -0
  26. package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
  27. package/dist-lib/lib/api/rag-plugin.js +70 -0
  28. package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
  29. package/dist-lib/lib/api/rag-synthesis.js +102 -0
  30. package/dist-lib/lib/api/rag-targets.d.ts +23 -0
  31. package/dist-lib/lib/api/rag-targets.js +215 -0
  32. package/dist-lib/lib/api/rag-window.d.ts +5 -0
  33. package/dist-lib/lib/api/rag-window.js +46 -0
  34. package/dist-lib/lib/api/rag.d.ts +3 -2
  35. package/dist-lib/lib/api/rag.js +421 -181
  36. package/dist-lib/lib/api/served.js +8 -14
  37. package/dist-lib/lib/api/workspace.d.ts +8 -1
  38. package/dist-lib/lib/api/workspace.js +27 -13
  39. package/dist-lib/lib/assistant/harness.d.ts +9 -1
  40. package/dist-lib/lib/assistant/history.d.ts +11 -2
  41. package/dist-lib/lib/assistant/history.js +74 -4
  42. package/dist-lib/lib/assistant/media-round.d.ts +61 -0
  43. package/dist-lib/lib/assistant/media-round.js +111 -0
  44. package/dist-lib/lib/assistant/message.d.ts +7 -0
  45. package/dist-lib/lib/assistant/model-round.d.ts +12 -0
  46. package/dist-lib/lib/assistant/model-round.js +130 -17
  47. package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
  48. package/dist-lib/lib/assistant/pdf-pages.js +67 -3
  49. package/dist-lib/lib/assistant/round-host.d.ts +22 -0
  50. package/dist-lib/lib/assistant/round-host.js +142 -8
  51. package/dist-lib/lib/assistant/upload.d.ts +12 -0
  52. package/dist-lib/lib/assistant/upload.js +36 -0
  53. package/dist-lib/lib/assistant-client.d.ts +1 -0
  54. package/dist-lib/lib/assistant-client.js +40 -19
  55. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  56. package/dist-lib/lib/assistant-tree.js +41 -1
  57. package/dist-lib/lib/assistant.d.ts +15 -0
  58. package/dist-lib/lib/assistant.js +474 -89
  59. package/dist-lib/lib/client.d.ts +1 -0
  60. package/dist-lib/lib/client.js +4 -1
  61. package/dist-lib/lib/compaction.js +2 -2
  62. package/dist-lib/lib/core/agent.js +12 -45
  63. package/dist-lib/lib/core/approval-rules.js +1 -8
  64. package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
  65. package/dist-lib/lib/core/archived-mesh.js +12 -8
  66. package/dist-lib/lib/core/chat-copy.d.ts +18 -3
  67. package/dist-lib/lib/core/chat-copy.js +74 -25
  68. package/dist-lib/lib/core/chat-fork.js +15 -6
  69. package/dist-lib/lib/core/chat-group.js +16 -24
  70. package/dist-lib/lib/core/chat-move.d.ts +9 -0
  71. package/dist-lib/lib/core/chat-move.js +94 -3
  72. package/dist-lib/lib/core/chat-order.js +12 -18
  73. package/dist-lib/lib/core/chat.js +38 -31
  74. package/dist-lib/lib/core/chunk.d.ts +2 -2
  75. package/dist-lib/lib/core/chunk.js +18 -25
  76. package/dist-lib/lib/core/collections.d.ts +2 -0
  77. package/dist-lib/lib/core/collections.js +2 -0
  78. package/dist-lib/lib/core/contract/activity.d.ts +4 -0
  79. package/dist-lib/lib/core/contract/activity.js +34 -4
  80. package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
  81. package/dist-lib/lib/core/contract/chat-group.js +21 -1
  82. package/dist-lib/lib/core/contract/chat.d.ts +9 -0
  83. package/dist-lib/lib/core/contract/chat.js +127 -5
  84. package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
  85. package/dist-lib/lib/core/contract/chunks.js +37 -6
  86. package/dist-lib/lib/core/contract/features.d.ts +21 -0
  87. package/dist-lib/lib/core/contract/features.js +22 -0
  88. package/dist-lib/lib/core/contract/index.js +2 -0
  89. package/dist-lib/lib/core/contract/indexing.js +4 -2
  90. package/dist-lib/lib/core/contract/knowledge.js +5 -4
  91. package/dist-lib/lib/core/contract/members.js +16 -10
  92. package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
  93. package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
  94. package/dist-lib/lib/core/contract/projects.d.ts +5 -0
  95. package/dist-lib/lib/core/contract/projects.js +13 -5
  96. package/dist-lib/lib/core/contract/roles.js +32 -4
  97. package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
  98. package/dist-lib/lib/core/contract/run-requests.js +35 -2
  99. package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
  100. package/dist-lib/lib/core/contract/tool-configs.js +3 -2
  101. package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
  102. package/dist-lib/lib/core/contract/workspace.js +40 -7
  103. package/dist-lib/lib/core/contract/writers.js +2 -2
  104. package/dist-lib/lib/core/departures.d.ts +3 -1
  105. package/dist-lib/lib/core/departures.js +3 -0
  106. package/dist-lib/lib/core/devices.js +1 -8
  107. package/dist-lib/lib/core/files/attachments.d.ts +2 -0
  108. package/dist-lib/lib/core/files/attachments.js +19 -1
  109. package/dist-lib/lib/core/files/index.d.ts +20 -4
  110. package/dist-lib/lib/core/files/index.js +132 -21
  111. package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
  112. package/dist-lib/lib/core/files/processing/document.js +7 -0
  113. package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
  114. package/dist-lib/lib/core/files/processing/index.js +144 -12
  115. package/dist-lib/lib/core/files/processing/protocol.js +13 -34
  116. package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
  117. package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
  118. package/dist-lib/lib/core/files/rag/build.js +123 -19
  119. package/dist-lib/lib/core/files/rag/failure.js +2 -1
  120. package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
  121. package/dist-lib/lib/core/files/rag/index.js +13 -1
  122. package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
  123. package/dist-lib/lib/core/files/rag/load.js +27 -6
  124. package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
  125. package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
  126. package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
  127. package/dist-lib/lib/core/files/rag/produce.js +104 -16
  128. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
  129. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
  130. package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
  131. package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
  132. package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
  133. package/dist-lib/lib/core/files/records/knowledge.js +7 -40
  134. package/dist-lib/lib/core/files/records/reclaim.js +5 -10
  135. package/dist-lib/lib/core/files/upload.js +68 -41
  136. package/dist-lib/lib/core/fs-drives.js +14 -26
  137. package/dist-lib/lib/core/invite.d.ts +1 -0
  138. package/dist-lib/lib/core/invite.js +4 -2
  139. package/dist-lib/lib/core/local/files.d.ts +1 -1
  140. package/dist-lib/lib/core/local/files.js +5 -5
  141. package/dist-lib/lib/core/local/index.d.ts +13 -6
  142. package/dist-lib/lib/core/local/index.js +103 -69
  143. package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
  144. package/dist-lib/lib/core/local/run-logs.js +5 -5
  145. package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
  146. package/dist-lib/lib/core/mesh-processor.js +87 -0
  147. package/dist-lib/lib/core/mesh.d.ts +48 -5
  148. package/dist-lib/lib/core/mesh.js +318 -129
  149. package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
  150. package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
  151. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
  152. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
  153. package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
  154. package/dist-lib/lib/core/migrations/writer-links.js +32 -0
  155. package/dist-lib/lib/core/project-keys.d.ts +6 -0
  156. package/dist-lib/lib/core/project-keys.js +11 -0
  157. package/dist-lib/lib/core/public-mesh.d.ts +3 -0
  158. package/dist-lib/lib/core/public-mesh.js +10 -8
  159. package/dist-lib/lib/core/run-requests.d.ts +3 -0
  160. package/dist-lib/lib/core/run-requests.js +21 -1
  161. package/dist-lib/lib/core/skill-availability.d.ts +33 -0
  162. package/dist-lib/lib/core/skill-availability.js +67 -0
  163. package/dist-lib/lib/core/sync/background.d.ts +0 -1
  164. package/dist-lib/lib/core/sync/background.js +2 -1
  165. package/dist-lib/lib/core/sync/changed.d.ts +4 -2
  166. package/dist-lib/lib/core/sync/changed.js +1 -1
  167. package/dist-lib/lib/core/sync/index.js +52 -10
  168. package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
  169. package/dist-lib/lib/core/sync/view-updates.js +82 -91
  170. package/dist-lib/lib/core/sync-status.js +97 -41
  171. package/dist-lib/lib/core/tool-activity.js +6 -12
  172. package/dist-lib/lib/core/tool-configs.js +17 -29
  173. package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
  174. package/dist-lib/lib/core/upgrade-summary.js +67 -0
  175. package/dist-lib/lib/core/view-read.d.ts +4 -0
  176. package/dist-lib/lib/core/view-read.js +23 -0
  177. package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
  178. package/dist-lib/lib/core/workspace-upgrade.js +27 -16
  179. package/dist-lib/lib/core.d.ts +39 -42
  180. package/dist-lib/lib/core.js +661 -639
  181. package/dist-lib/lib/errors.d.ts +20 -0
  182. package/dist-lib/lib/errors.js +20 -0
  183. package/dist-lib/lib/llm-model-names.d.ts +1 -1
  184. package/dist-lib/lib/llm-model-names.js +1 -1
  185. package/dist-lib/lib/mesh-scoped-engine.js +4 -0
  186. package/dist-lib/lib/model-context.js +1 -2
  187. package/dist-lib/lib/probe-rounds.d.ts +7 -2
  188. package/dist-lib/lib/probe-rounds.js +6 -1
  189. package/dist-lib/lib/run-transcript.d.ts +4 -1
  190. package/dist-lib/lib/run-transcript.js +15 -6
  191. package/dist-lib/schema/engine.js +83 -6
  192. package/dist-lib/schema/indexes.d.ts +3 -0
  193. package/dist-lib/schema/indexes.js +5 -0
  194. package/dist-lib/schema/local.js +11 -2
  195. package/dist-lib/schema/qvac.js +49 -5
  196. package/dist-lib/spec/bind.js +34 -0
  197. package/dist-lib/spec/calls.js +5 -0
  198. package/dist-lib/spec/capabilities.d.ts +17 -0
  199. package/dist-lib/spec/hrpc/hrpc.json +65 -0
  200. package/dist-lib/spec/hrpc/index.js +68 -5
  201. package/dist-lib/spec/hrpc/messages.js +260 -23
  202. package/dist-lib/spec/hyperschema/index.js +260 -23
  203. package/dist-lib/spec/hyperschema/schema.json +181 -0
  204. package/dist-lib/spec/hyperschema/types.d.ts +48 -0
  205. package/dist-lib/spec/local/hyperdb/messages.js +270 -224
  206. package/dist-lib/spec/local/hyperschema/index.js +73 -38
  207. package/dist-lib/spec/local/hyperschema/schema.json +31 -0
  208. package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
  209. package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
  210. package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
  211. package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
  212. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
  213. package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
  214. package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
  215. package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
  216. package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
  217. package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
  218. package/package.json +75 -70
  219. package/tui/README.md +2 -1
  220. package/tui/app.mjs +28 -7
  221. package/tui/boot.mjs +5 -1
  222. package/tui/lib/attachment-files.mjs +82 -0
  223. package/tui/lib/reopen-meshes.mjs +15 -0
  224. package/tui/main.mjs +4 -43
  225. package/worker/assistant-sidecar.mjs +10 -3
  226. package/worker/stow-entry.mjs +3 -52
@@ -7,6 +7,7 @@ import { historyFromChunks, listHistoryChunks, replayFromChunks, runPromptView,
7
7
  import { nameSenders, restoreSentContent } from './message.js';
8
8
  import { PdfBoundsSchema } from './pdf-pages.js';
9
9
  import { isProbeRound, PROBE_KINDS } from '../probe-rounds.js';
10
+ import { MODEL_MISSING, mediaOptions, mediaOutputFrom, putBlob } from './media-round.js';
10
11
  const message = z.custom((value) => isRecord(value) && typeof value.role === 'string' && typeof value.content === 'string');
11
12
  const record = () => z.custom((value) => isRecord(value));
12
13
  export function wholeImageId(ref) {
@@ -32,7 +33,8 @@ const roundOptions = z.object({
32
33
  responseFormat: record().optional(),
33
34
  toolDialect: z.custom().optional(),
34
35
  unforced: record().optional(),
35
- media: z.array(z.object({ at: z.number(), refs: z.array(mediaRef) })).optional()
36
+ media: z.array(z.object({ at: z.number(), refs: z.array(mediaRef) })).optional(),
37
+ mediaRound: mediaOptions.optional()
36
38
  });
37
39
  const roundResult = z.custom((value) => isRecord(value) && typeof value.contentText === 'string' && isRecord(value.final));
38
40
  export class Echoes {
@@ -55,7 +57,9 @@ export class Echoes {
55
57
  }
56
58
  export class OwnerRounds {
57
59
  port;
60
+ media;
58
61
  inputLength = 0;
62
+ roundBase = 0;
59
63
  share = null;
60
64
  _opts;
61
65
  _echoes = new Echoes();
@@ -64,9 +68,11 @@ export class OwnerRounds {
64
68
  _rounds = 0;
65
69
  _retakes = 0;
66
70
  _takers = new Map();
71
+ _generated = [];
67
72
  constructor(opts) {
68
73
  this._opts = opts;
69
74
  this.port = (request, io) => this._round(request, io);
75
+ this.media = (request, io) => this._generate(request, io);
70
76
  }
71
77
  get rounds() {
72
78
  return this._rounds;
@@ -151,7 +157,7 @@ export class OwnerRounds {
151
157
  const { modelId: asked, generationParams } = options;
152
158
  const call = { kind, modelId: asked, messages, generationParams, responseFormat };
153
159
  try {
154
- const { contentText } = await this._probed({ id, agentId }, signal);
160
+ const { contentText } = await this._probed({ id, agentId }, signal, (result) => roundResult.parse(result));
155
161
  this._opts.inspect?.({ ...call, outcome: 'completed', result: contentText });
156
162
  return contentText;
157
163
  }
@@ -160,7 +166,56 @@ export class OwnerRounds {
160
166
  throw err;
161
167
  }
162
168
  }
163
- async _probed(asked, signal) {
169
+ async _generate(request, io) {
170
+ if (io.signal.aborted)
171
+ throw new AbortedError();
172
+ const { engine, chatId, runId, deviceId, meshId, local } = this._opts;
173
+ const agentId = (await this._switchedTo()) ?? this._opts.agentId;
174
+ const { modelChoice, params } = request;
175
+ const refs = this._named(request.sources);
176
+ const sources = refs || !meshId
177
+ ? undefined
178
+ : await Promise.all((request.images ?? []).map((bytes) => putBlob(engine, meshId, bytes)));
179
+ const options = {
180
+ kind: request.kind,
181
+ attempt: 0,
182
+ modelId: modelChoice ?? request.kind,
183
+ continuationNudge: '',
184
+ deliveredNudge: '',
185
+ mediaRound: { modelChoice, params, sources, refs }
186
+ };
187
+ const { id } = await engine.appendChunk({
188
+ chatId,
189
+ runId,
190
+ type: 'model-request',
191
+ originDeviceId: deviceId,
192
+ modelRequest: { round: 0, agents: [agentId], options, state: 'open' }
193
+ });
194
+ const settled = await this._probed({ id, agentId }, io.signal, (result) => result);
195
+ const { result, generated } = await mediaOutputFrom(engine, settled, io.signal);
196
+ if ('error' in result && result.code === MODEL_MISSING && local) {
197
+ return local(request, io);
198
+ }
199
+ if (generated) {
200
+ this._generated.push(generated);
201
+ }
202
+ return result;
203
+ }
204
+ generated(bytes) {
205
+ const at = this._generated.findIndex((output) => b4a.equals(output.bytes, bytes));
206
+ if (at === -1) {
207
+ return undefined;
208
+ }
209
+ return this._generated.splice(at, 1)[0].blobId;
210
+ }
211
+ _named(sources) {
212
+ const refs = (sources ?? []).flatMap(({ attachmentId, path, width, height }) => {
213
+ const id = attachmentId ?? (path ? wholeImageId(this._opts.media?.(path)) : undefined);
214
+ return id ? [{ attachmentId: id, width, height }] : [];
215
+ });
216
+ return refs.length && refs.length === sources?.length ? refs : undefined;
217
+ }
218
+ async _probed(asked, signal, parse) {
164
219
  const { engine, chatId, from } = this._opts;
165
220
  const watch = engine.chunksDeltaWatch({ chatId, gte: from || undefined });
166
221
  const agents = engine.agentsWatch({});
@@ -173,9 +228,8 @@ export class OwnerRounds {
173
228
  for await (const frame of watch) {
174
229
  round = frame.chunks?.find((chunk) => chunk.id === asked.id)?.modelRequest ?? round;
175
230
  ended = !!round && round.state !== 'open' && round.state !== 'taken';
176
- const result = roundEnd(round);
177
- if (result) {
178
- return result;
231
+ if (roundDone(round)) {
232
+ return parse(round?.result);
179
233
  }
180
234
  }
181
235
  throw new AbortedError();
@@ -185,22 +239,27 @@ export class OwnerRounds {
185
239
  watch.destroy();
186
240
  agents.destroy();
187
241
  if (!ended) {
242
+ this._opts.log?.info('rounds', 'probe cancelled', {
243
+ roundId: asked.id,
244
+ reason: signal.aborted ? 'aborted' : 'watch ended'
245
+ });
188
246
  await engine.cancelModelRequest({ chunkId: asked.id }).catch(noop);
189
247
  }
190
248
  }
191
249
  }
192
250
  async _dropOnHostLoss(agents, asked, round) {
193
- let reachable = false;
194
251
  for await (const _frame of agents) {
195
252
  const agent = await this._opts.engine.getAgent({ agentId: asked.agentId }).catch(() => null);
196
253
  if (!agent) {
197
254
  continue;
198
255
  }
199
- const was = reachable;
200
- reachable = !!agent.hostReachable;
201
256
  const host = round()?.hostDeviceId;
202
257
  const moved = !!host && !b4a.equals(agent.providerDeviceId, host);
203
- if (moved || (was && !reachable)) {
258
+ if (moved || !agent.hostReachable) {
259
+ this._opts.log?.info('rounds', 'probe cancelled', {
260
+ roundId: asked.id,
261
+ reason: moved ? 'provider moved' : 'host lost'
262
+ });
204
263
  await this._opts.engine.cancelModelRequest({ chunkId: asked.id }).catch(noop);
205
264
  return;
206
265
  }
@@ -218,7 +277,7 @@ export class OwnerRounds {
218
277
  type: 'model-request',
219
278
  originDeviceId: deviceId,
220
279
  modelRequest: {
221
- round: sent.round,
280
+ round: sent.round + this.roundBase,
222
281
  agents: [agentId],
223
282
  cacheKey: sent.cacheKey,
224
283
  promptPrefix: sent.promptPrefix,
@@ -227,6 +286,14 @@ export class OwnerRounds {
227
286
  state: 'open'
228
287
  }
229
288
  });
289
+ this._opts.log?.info('rounds', 'round asked', {
290
+ runId,
291
+ roundId: id,
292
+ chatId,
293
+ mesh: this._opts.meshId?.slice(0, 8),
294
+ agentId,
295
+ round: sent.round + this.roundBase
296
+ });
230
297
  return id;
231
298
  }
232
299
  async _modelOf(agentId, modelId) {
@@ -251,7 +318,10 @@ export class OwnerRounds {
251
318
  runId,
252
319
  type: 'context',
253
320
  originDeviceId: deviceId,
254
- context: { turn: { sent: sent?.content }, toolSetHash: request.toolSet ?? null }
321
+ context: {
322
+ turn: { sent: sent?.content, shots: this._pdfShots(sent) },
323
+ toolSetHash: request.toolSet ?? null
324
+ }
255
325
  });
256
326
  }
257
327
  async _options(request, system) {
@@ -277,6 +347,17 @@ export class OwnerRounds {
277
347
  }
278
348
  return options;
279
349
  }
350
+ _pdfShots(turn) {
351
+ return (turn?.attachments ?? []).flatMap(({ path }) => {
352
+ const ref = this._opts.media?.(path);
353
+ if (ref?.page === undefined || ref.frame !== undefined) {
354
+ return [];
355
+ }
356
+ return [
357
+ ref.clip ? { id: ref.id, page: ref.page, clip: ref.clip } : { id: ref.id, page: ref.page }
358
+ ];
359
+ });
360
+ }
280
361
  _namedImages(turn) {
281
362
  const imageIds = (turn.attachments ?? []).flatMap(({ path }) => wholeImageId(this._opts.media?.(path)) ?? []);
282
363
  return imageIds.length ? { ...turn, imageIds } : turn;
@@ -327,9 +408,27 @@ export class OwnerRounds {
327
408
  await engine.cancelModelRequest({ chunkId: at.asked.id }).catch(noop);
328
409
  };
329
410
  void this._followHost(agents, { at, output, move }).catch(noop);
411
+ const since = Date.now();
412
+ let takenAt;
330
413
  try {
331
414
  for await (const frame of watch) {
332
415
  const round = output.apply(frame);
416
+ if (round?.state === 'taken' && round.takenBy && round.takenAt !== takenAt) {
417
+ takenAt = round.takenAt;
418
+ this._opts.log?.debug('rounds', 'round taken', {
419
+ roundId: at.asked.id,
420
+ takenBy: shortId(round.takenBy)
421
+ });
422
+ }
423
+ if (!at.moving && round && round.state !== 'open' && round.state !== 'taken') {
424
+ this._opts.log?.info('rounds', 'round ended', {
425
+ roundId: at.asked.id,
426
+ state: round.state,
427
+ takenBy: round.takenBy ? shortId(round.takenBy) : undefined,
428
+ host: round.hostDeviceId ? shortId(round.hostDeviceId) : undefined,
429
+ ms: Date.now() - since
430
+ });
431
+ }
333
432
  const result = at.moving ? undefined : roundEnd(round);
334
433
  if (result) {
335
434
  this._takers.set(sent.round, round?.takenBy ?? null);
@@ -374,12 +473,23 @@ export class OwnerRounds {
374
473
  if (provider !== b4a.toHex(host)) {
375
474
  if (at.host.moved !== provider) {
376
475
  at.host.moved = provider;
476
+ this._opts.log?.info('rounds', 'round moved', {
477
+ roundId: at.asked.id,
478
+ reason: 'provider moved',
479
+ host: shortId(host),
480
+ provider: provider.slice(0, 8)
481
+ });
377
482
  await move(agentId);
378
483
  }
379
484
  continue;
380
485
  }
381
486
  const lost = was?.provider === provider && was.reachable && !at.host.reachable;
382
487
  if (lost && output.round()?.state === 'taken') {
488
+ this._opts.log?.info('rounds', 'round reopened', {
489
+ roundId: at.asked.id,
490
+ reason: 'host lost',
491
+ host: provider.slice(0, 8)
492
+ });
383
493
  await this._opts.engine.reopenModelRequest({ chunkId: at.asked.id }).catch(noop);
384
494
  }
385
495
  }
@@ -631,17 +741,17 @@ function carried(rounds, promptPrefix, toolSet) {
631
741
  }
632
742
  return { system, tools: toolSet ? sets.resolve(toolSet) : undefined };
633
743
  }
634
- function roundEnd(round) {
635
- if (round?.state === 'done') {
636
- return roundResult.parse(round.result);
637
- }
744
+ function roundDone(round) {
638
745
  if (round?.state === 'failed') {
639
746
  throw new Error(round.reason || 'the model round failed');
640
747
  }
641
748
  if (round?.state === 'cancelled') {
642
749
  throw new AbortedError();
643
750
  }
644
- return undefined;
751
+ return round?.state === 'done';
752
+ }
753
+ function roundEnd(round) {
754
+ return roundDone(round) ? roundResult.parse(round?.result) : undefined;
645
755
  }
646
756
  function lastSwitch(chunks, from) {
647
757
  let last;
@@ -653,3 +763,6 @@ function lastSwitch(chunks, from) {
653
763
  return last?.agentSwitch?.agentId;
654
764
  }
655
765
  function noop() { }
766
+ function shortId(key) {
767
+ return b4a.toHex(key).slice(0, 8);
768
+ }
@@ -13,6 +13,17 @@ export interface PdfSelection {
13
13
  readonly shots: ReadonlyArray<PdfShot>;
14
14
  readonly cutAt?: number;
15
15
  }
16
+ interface PdfHit {
17
+ readonly metadataJson: string;
18
+ }
19
+ interface PdfTextHit extends PdfHit {
20
+ readonly content: string;
21
+ }
22
+ export interface PdfSelectionOptions {
23
+ readonly cited?: ReadonlySet<number>;
24
+ readonly textFirst?: boolean;
25
+ readonly readablePages?: ReadonlySet<number> | null;
26
+ }
16
27
  export declare const PdfBoundsSchema: z.ZodObject<{
17
28
  x: z.ZodNumber;
18
29
  y: z.ZodNumber;
@@ -22,4 +33,8 @@ export declare const PdfBoundsSchema: z.ZodObject<{
22
33
  export declare function pdfSelectionFileName(pdfFileName: string): string;
23
34
  export declare function packPdfSelection(selection: PdfSelection): Uint8Array;
24
35
  export declare function parsePdfSelection(bytes: Uint8Array): PdfSelection;
25
- export declare function selectPdfPages(doc: PdfDoc, room: PdfRoom): Promise<PdfSelection>;
36
+ export declare function asksForPdfVisuals(question: string): boolean;
37
+ export declare function readablePdfTextPages(hits: ReadonlyArray<PdfTextHit>): Map<string, Set<number> | null>;
38
+ export declare function citedPdfPages(hits: ReadonlyArray<PdfHit>): Map<string, Set<number> | null>;
39
+ export declare function selectPdfPages(doc: PdfDoc, room: PdfRoom, { cited, textFirst, readablePages }?: PdfSelectionOptions): Promise<PdfSelection>;
40
+ export {};
@@ -1,4 +1,5 @@
1
1
  import b4a from 'b4a';
2
+ import { parseRagMetadata } from 'rag-extract';
2
3
  import * as z from 'zod';
3
4
  import { frameTokens } from './video-budget.js';
4
5
  export const PDF_PAGES_MIME = 'application/vnd.qvac.pdf-pages';
@@ -31,14 +32,74 @@ export function parsePdfSelection(bytes) {
31
32
  function shotLabel(page, names = []) {
32
33
  return names.length ? `page ${page + 1} (${names.join(', ')})` : `page ${page + 1}`;
33
34
  }
34
- export async function selectPdfPages(doc, room) {
35
+ const VISUAL_REQUEST = /\b(?:charts?|figures?|diagrams?|graphs?|plots?|maps?|images?|pictures?|photos?|illustrations?|visuals?|layouts?|handwriting|handwritten|scanned|screenshots?)\b|\b(?:this|that|the|attached)\s+scan\b/i;
36
+ const TABLE_REFERENCE = /\b(?:the|this|that|attached|above|below|following|all|any|each)\s+tables?\b|\btables?\s*(?:#?\s*\d+|[A-Z]\b|of|on|from|above|below)\b|\b(?:read|show|describe|explain|extract|summari[sz]e|analy[sz]e|find|inspect|compare)\s+(?:me\s+)?(?:(?:all|any|each)\s+)?(?:the\s+|this\s+|that\s+)?tables?\b/i;
37
+ const TABLE_MENTION = /\btables?\b/i;
38
+ const TABLE_OUTPUT = /\b(?:in|as|into)\s+(?:a\s+|the\s+)?table\b|\b(?:make|create|generate|compile|write|produce)\b[^.!?\n]{0,40}\b(?:a\s+|an\s+)?table\b/i;
39
+ export function asksForPdfVisuals(question) {
40
+ if (VISUAL_REQUEST.test(question))
41
+ return true;
42
+ const tableQuestion = question.replace(TABLE_OUTPUT, ' ');
43
+ if (TABLE_REFERENCE.test(tableQuestion)) {
44
+ return true;
45
+ }
46
+ return TABLE_MENTION.test(question) && !TABLE_OUTPUT.test(question);
47
+ }
48
+ export function readablePdfTextPages(hits) {
49
+ const readable = new Map();
50
+ for (const { content, metadataJson } of hits) {
51
+ if (typeof content !== 'string')
52
+ continue;
53
+ const metadata = parseRagMetadata(metadataJson);
54
+ const text = content.normalize('NFKC').replace(/\s+/gu, ' ').trim();
55
+ if (!metadata?.attachmentId || !hasReadableContent(text, metadata.tokenCount)) {
56
+ continue;
57
+ }
58
+ const pages = readable.get(metadata.attachmentId);
59
+ if (metadata.page === null || pages === null) {
60
+ readable.set(metadata.attachmentId, null);
61
+ continue;
62
+ }
63
+ readable.set(metadata.attachmentId, (pages ?? new Set()).add(metadata.page));
64
+ }
65
+ return readable;
66
+ }
67
+ function hasReadableContent(text, cachedChunkTokens) {
68
+ const words = text.match(/[\p{L}\p{N}]{2,}/gu) ?? [];
69
+ if (text.length < 24 ||
70
+ words.length < 4 ||
71
+ /^(?:files in this chat:|document outline(?:\b|$)|readable,\s+\d+\s+(?:pages?|lines?)(?:\b|$))/i.test(text)) {
72
+ return false;
73
+ }
74
+ return text.length >= 80 || cachedChunkTokens >= 24;
75
+ }
76
+ export function citedPdfPages(hits) {
77
+ const cited = new Map();
78
+ for (const { metadataJson } of hits) {
79
+ const metadata = parseRagMetadata(metadataJson);
80
+ if (!metadata?.attachmentId) {
81
+ continue;
82
+ }
83
+ const pages = cited.get(metadata.attachmentId);
84
+ if (metadata.page === null || pages === null) {
85
+ cited.set(metadata.attachmentId, null);
86
+ continue;
87
+ }
88
+ cited.set(metadata.attachmentId, (pages ?? new Set()).add(metadata.page));
89
+ }
90
+ return cited;
91
+ }
92
+ export async function selectPdfPages(doc, room, { cited, textFirst = false, readablePages } = {}) {
35
93
  const pageCount = doc.pageCount();
36
94
  const shots = [];
37
95
  let cutAt;
38
96
  for (let page = 0; page < pageCount; page++) {
97
+ if (cited && !cited.has(page + 1)) {
98
+ continue;
99
+ }
39
100
  let shot;
40
101
  try {
41
- shot = await pictured(doc, page);
102
+ shot = await pictured(doc, page, { textFirst, readablePages });
42
103
  }
43
104
  catch {
44
105
  continue;
@@ -60,8 +121,11 @@ export async function selectPdfPages(doc, room) {
60
121
  }
61
122
  return cutAt === undefined ? { pageCount, shots } : { pageCount, shots, cutAt };
62
123
  }
63
- async function pictured(doc, page) {
124
+ async function pictured(doc, page, { textFirst, readablePages }) {
64
125
  const flags = doc.pageFlags(page);
126
+ if (textFirst && (readablePages?.has(page + 1) || (readablePages === null && flags.hasText))) {
127
+ return null;
128
+ }
65
129
  if (flags.hasText && !flags.hasImage) {
66
130
  return null;
67
131
  }
@@ -1,17 +1,24 @@
1
1
  import { type AbortSignal } from 'bare-abort-controller';
2
+ import type { ModelRoundRequest } from '@qvac/harness';
3
+ import type { MediaRoundPort } from '@qvac/harness/lib/contract/index.ts';
2
4
  import type { LoggerInterface } from '@qvac/logging';
3
5
  import type { AssistantChunk, AssistantChunkList } from '../../spec/hyperschema/types.d.ts';
4
6
  import type { Capabilities, WatchStream } from '../api/index.ts';
7
+ import { type ScopedCapabilities } from '../mesh-scoped-engine.ts';
5
8
  import type { Harness } from './harness.ts';
9
+ import type { MediaKind } from '../probe-rounds.ts';
6
10
  import { type MediaSource } from './model-round.ts';
7
11
  export interface RoundHostOptions {
8
12
  readonly engine: Capabilities;
9
13
  readonly serve: NonNullable<Harness['serveModelRound']>;
10
14
  readonly deviceId: () => Buffer;
11
15
  readonly media?: RoundMedia | null;
16
+ readonly load?: RoundLoad | null;
17
+ readonly generation?: RoundGeneration | null;
12
18
  readonly built?: (meshId: string, chatId: string) => void;
13
19
  readonly log?: LoggerInterface | null;
14
20
  }
21
+ export type RoundLoad = (engine: ScopedCapabilities, agentId: string) => Promise<ModelRoundRequest['load']>;
15
22
  export interface RoundMedia {
16
23
  resolve(sources: ReadonlyArray<MediaSource>, served: {
17
24
  readonly id: string;
@@ -20,6 +27,16 @@ export interface RoundMedia {
20
27
  }): Promise<Array<string | null>>;
21
28
  release(id: string): void;
22
29
  }
30
+ export interface GenerationWait {
31
+ readonly engine: ScopedCapabilities;
32
+ readonly agentId: string;
33
+ readonly kind: MediaKind;
34
+ readonly signal: AbortSignal;
35
+ }
36
+ export interface RoundGeneration {
37
+ readonly serve: MediaRoundPort;
38
+ ready(wait: GenerationWait): Promise<boolean>;
39
+ }
23
40
  export declare function servedCacheKey(round: AssistantChunk, ownerKey: string | null | undefined): string | undefined;
24
41
  export declare class RoundHost {
25
42
  private readonly _opts;
@@ -31,14 +48,19 @@ export declare class RoundHost {
31
48
  left(chatId: string, agentId: string): string[];
32
49
  holds(chatId: string): boolean;
33
50
  stop(): void;
51
+ drop(meshId: string): void;
34
52
  private _reconcile;
35
53
  private _pick;
36
54
  private _serve;
37
55
  private _take;
38
56
  private _provides;
57
+ private _load;
39
58
  private _answer;
59
+ private _generate;
60
+ private _stored;
40
61
  private _probe;
41
62
  private _claimKey;
42
63
  private _retake;
43
64
  private _takeFor;
65
+ private _settle;
44
66
  }