@qvac/core 1.0.0-rc.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. package/CHANGELOG.md +241 -3
  2. package/README.md +1 -0
  3. package/dist-lib/lib/api/attachments.d.ts +1 -1
  4. package/dist-lib/lib/api/attachments.js +11 -0
  5. package/dist-lib/lib/api/chunks.d.ts +4 -2
  6. package/dist-lib/lib/api/chunks.js +6 -3
  7. package/dist-lib/lib/api/device-routes.d.ts +6 -1
  8. package/dist-lib/lib/api/device-routes.js +20 -1
  9. package/dist-lib/lib/api/devices.js +6 -18
  10. package/dist-lib/lib/api/execution.d.ts +6 -2
  11. package/dist-lib/lib/api/execution.js +29 -3
  12. package/dist-lib/lib/api/host-run-log.d.ts +7 -1
  13. package/dist-lib/lib/api/host-run-log.js +3 -1
  14. package/dist-lib/lib/api/index.d.ts +3 -1
  15. package/dist-lib/lib/api/index.js +17 -6
  16. package/dist-lib/lib/api/members.js +13 -29
  17. package/dist-lib/lib/api/mesh.d.ts +2 -1
  18. package/dist-lib/lib/api/projects.d.ts +1 -1
  19. package/dist-lib/lib/api/projects.js +9 -5
  20. package/dist-lib/lib/api/rag-label.d.ts +4 -0
  21. package/dist-lib/lib/api/rag-label.js +21 -0
  22. package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
  23. package/dist-lib/lib/api/rag-lexical.js +1058 -0
  24. package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
  25. package/dist-lib/lib/api/rag-manifest.js +46 -0
  26. package/dist-lib/lib/api/rag-outline.d.ts +36 -0
  27. package/dist-lib/lib/api/rag-outline.js +426 -0
  28. package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
  29. package/dist-lib/lib/api/rag-plugin.js +70 -0
  30. package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
  31. package/dist-lib/lib/api/rag-synthesis.js +102 -0
  32. package/dist-lib/lib/api/rag-targets.d.ts +23 -0
  33. package/dist-lib/lib/api/rag-targets.js +215 -0
  34. package/dist-lib/lib/api/rag-window.d.ts +5 -0
  35. package/dist-lib/lib/api/rag-window.js +46 -0
  36. package/dist-lib/lib/api/rag.d.ts +3 -2
  37. package/dist-lib/lib/api/rag.js +421 -184
  38. package/dist-lib/lib/api/served.js +8 -14
  39. package/dist-lib/lib/api/workspace.d.ts +8 -1
  40. package/dist-lib/lib/api/workspace.js +27 -13
  41. package/dist-lib/lib/assistant/harness.d.ts +9 -1
  42. package/dist-lib/lib/assistant/history.d.ts +11 -2
  43. package/dist-lib/lib/assistant/history.js +74 -4
  44. package/dist-lib/lib/assistant/media-round.d.ts +61 -0
  45. package/dist-lib/lib/assistant/media-round.js +111 -0
  46. package/dist-lib/lib/assistant/message.d.ts +7 -0
  47. package/dist-lib/lib/assistant/model-round.d.ts +19 -1
  48. package/dist-lib/lib/assistant/model-round.js +163 -28
  49. package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
  50. package/dist-lib/lib/assistant/pdf-pages.js +67 -3
  51. package/dist-lib/lib/assistant/round-host.d.ts +22 -0
  52. package/dist-lib/lib/assistant/round-host.js +142 -8
  53. package/dist-lib/lib/assistant/upload.d.ts +12 -0
  54. package/dist-lib/lib/assistant/upload.js +36 -0
  55. package/dist-lib/lib/assistant-client.d.ts +1 -0
  56. package/dist-lib/lib/assistant-client.js +40 -19
  57. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  58. package/dist-lib/lib/assistant-tree.js +48 -1
  59. package/dist-lib/lib/assistant.d.ts +18 -1
  60. package/dist-lib/lib/assistant.js +484 -95
  61. package/dist-lib/lib/client.d.ts +1 -0
  62. package/dist-lib/lib/client.js +4 -1
  63. package/dist-lib/lib/compaction.js +2 -2
  64. package/dist-lib/lib/core/agent.js +12 -45
  65. package/dist-lib/lib/core/approval-rules.js +1 -8
  66. package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
  67. package/dist-lib/lib/core/archived-mesh.js +12 -8
  68. package/dist-lib/lib/core/chat-copy.d.ts +18 -3
  69. package/dist-lib/lib/core/chat-copy.js +74 -25
  70. package/dist-lib/lib/core/chat-fork.js +15 -6
  71. package/dist-lib/lib/core/chat-group.js +16 -24
  72. package/dist-lib/lib/core/chat-move.d.ts +9 -0
  73. package/dist-lib/lib/core/chat-move.js +94 -3
  74. package/dist-lib/lib/core/chat-order.js +12 -18
  75. package/dist-lib/lib/core/chat.js +38 -31
  76. package/dist-lib/lib/core/chunk.d.ts +2 -2
  77. package/dist-lib/lib/core/chunk.js +18 -25
  78. package/dist-lib/lib/core/collections.d.ts +2 -0
  79. package/dist-lib/lib/core/collections.js +2 -0
  80. package/dist-lib/lib/core/contract/activity.d.ts +4 -0
  81. package/dist-lib/lib/core/contract/activity.js +34 -4
  82. package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
  83. package/dist-lib/lib/core/contract/chat-group.js +21 -1
  84. package/dist-lib/lib/core/contract/chat.d.ts +9 -0
  85. package/dist-lib/lib/core/contract/chat.js +127 -5
  86. package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
  87. package/dist-lib/lib/core/contract/chunks.js +37 -6
  88. package/dist-lib/lib/core/contract/features.d.ts +21 -0
  89. package/dist-lib/lib/core/contract/features.js +22 -0
  90. package/dist-lib/lib/core/contract/index.js +2 -0
  91. package/dist-lib/lib/core/contract/indexing.js +4 -2
  92. package/dist-lib/lib/core/contract/knowledge.js +5 -4
  93. package/dist-lib/lib/core/contract/members.js +16 -10
  94. package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
  95. package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
  96. package/dist-lib/lib/core/contract/projects.d.ts +5 -0
  97. package/dist-lib/lib/core/contract/projects.js +13 -5
  98. package/dist-lib/lib/core/contract/roles.js +32 -4
  99. package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
  100. package/dist-lib/lib/core/contract/run-requests.js +35 -2
  101. package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
  102. package/dist-lib/lib/core/contract/tool-configs.js +3 -2
  103. package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
  104. package/dist-lib/lib/core/contract/workspace.js +40 -7
  105. package/dist-lib/lib/core/contract/writers.js +2 -2
  106. package/dist-lib/lib/core/departures.d.ts +3 -1
  107. package/dist-lib/lib/core/departures.js +3 -0
  108. package/dist-lib/lib/core/devices.js +1 -8
  109. package/dist-lib/lib/core/files/attachments.d.ts +2 -0
  110. package/dist-lib/lib/core/files/attachments.js +19 -1
  111. package/dist-lib/lib/core/files/index.d.ts +20 -4
  112. package/dist-lib/lib/core/files/index.js +132 -21
  113. package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
  114. package/dist-lib/lib/core/files/processing/document.js +7 -0
  115. package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
  116. package/dist-lib/lib/core/files/processing/index.js +144 -12
  117. package/dist-lib/lib/core/files/processing/protocol.js +13 -34
  118. package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
  119. package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
  120. package/dist-lib/lib/core/files/rag/build.js +123 -19
  121. package/dist-lib/lib/core/files/rag/failure.js +2 -1
  122. package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
  123. package/dist-lib/lib/core/files/rag/index.js +13 -1
  124. package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
  125. package/dist-lib/lib/core/files/rag/load.js +27 -6
  126. package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
  127. package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
  128. package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
  129. package/dist-lib/lib/core/files/rag/produce.js +104 -16
  130. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
  131. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
  132. package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
  133. package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
  134. package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
  135. package/dist-lib/lib/core/files/records/knowledge.js +7 -40
  136. package/dist-lib/lib/core/files/records/reclaim.js +5 -10
  137. package/dist-lib/lib/core/files/upload.js +68 -41
  138. package/dist-lib/lib/core/fs-drives.js +14 -26
  139. package/dist-lib/lib/core/invite.d.ts +1 -0
  140. package/dist-lib/lib/core/invite.js +4 -2
  141. package/dist-lib/lib/core/local/files.d.ts +1 -1
  142. package/dist-lib/lib/core/local/files.js +5 -5
  143. package/dist-lib/lib/core/local/index.d.ts +13 -6
  144. package/dist-lib/lib/core/local/index.js +103 -69
  145. package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
  146. package/dist-lib/lib/core/local/run-logs.js +5 -5
  147. package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
  148. package/dist-lib/lib/core/mesh-processor.js +87 -0
  149. package/dist-lib/lib/core/mesh.d.ts +48 -5
  150. package/dist-lib/lib/core/mesh.js +318 -129
  151. package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
  152. package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
  153. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
  154. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
  155. package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
  156. package/dist-lib/lib/core/migrations/writer-links.js +32 -0
  157. package/dist-lib/lib/core/project-keys.d.ts +6 -0
  158. package/dist-lib/lib/core/project-keys.js +11 -0
  159. package/dist-lib/lib/core/public-mesh.d.ts +3 -0
  160. package/dist-lib/lib/core/public-mesh.js +10 -8
  161. package/dist-lib/lib/core/run-requests.d.ts +3 -0
  162. package/dist-lib/lib/core/run-requests.js +21 -1
  163. package/dist-lib/lib/core/skill-availability.d.ts +33 -0
  164. package/dist-lib/lib/core/skill-availability.js +67 -0
  165. package/dist-lib/lib/core/sync/background.d.ts +0 -1
  166. package/dist-lib/lib/core/sync/background.js +2 -1
  167. package/dist-lib/lib/core/sync/changed.d.ts +4 -2
  168. package/dist-lib/lib/core/sync/changed.js +1 -1
  169. package/dist-lib/lib/core/sync/index.js +52 -10
  170. package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
  171. package/dist-lib/lib/core/sync/view-updates.js +82 -91
  172. package/dist-lib/lib/core/sync-status.js +97 -41
  173. package/dist-lib/lib/core/tool-activity.js +6 -12
  174. package/dist-lib/lib/core/tool-configs.js +17 -29
  175. package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
  176. package/dist-lib/lib/core/upgrade-summary.js +67 -0
  177. package/dist-lib/lib/core/view-read.d.ts +4 -0
  178. package/dist-lib/lib/core/view-read.js +23 -0
  179. package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
  180. package/dist-lib/lib/core/workspace-upgrade.js +27 -16
  181. package/dist-lib/lib/core.d.ts +45 -42
  182. package/dist-lib/lib/core.js +675 -640
  183. package/dist-lib/lib/errors.d.ts +24 -0
  184. package/dist-lib/lib/errors.js +24 -0
  185. package/dist-lib/lib/llm-model-names.d.ts +1 -1
  186. package/dist-lib/lib/llm-model-names.js +1 -1
  187. package/dist-lib/lib/mesh-scoped-engine.js +4 -0
  188. package/dist-lib/lib/model-context.js +1 -2
  189. package/dist-lib/lib/model-download.d.ts +6 -1
  190. package/dist-lib/lib/model-download.js +16 -3
  191. package/dist-lib/lib/oauth/resolve.js +29 -8
  192. package/dist-lib/lib/oauth/token-exchange-core.js +11 -0
  193. package/dist-lib/lib/probe-rounds.d.ts +7 -2
  194. package/dist-lib/lib/probe-rounds.js +6 -1
  195. package/dist-lib/lib/run-transcript.d.ts +4 -1
  196. package/dist-lib/lib/run-transcript.js +15 -6
  197. package/dist-lib/schema/engine.js +83 -6
  198. package/dist-lib/schema/indexes.d.ts +3 -0
  199. package/dist-lib/schema/indexes.js +5 -0
  200. package/dist-lib/schema/local.js +11 -2
  201. package/dist-lib/schema/qvac.js +49 -5
  202. package/dist-lib/spec/bind.js +34 -0
  203. package/dist-lib/spec/calls.js +5 -0
  204. package/dist-lib/spec/capabilities.d.ts +17 -0
  205. package/dist-lib/spec/hrpc/hrpc.json +65 -0
  206. package/dist-lib/spec/hrpc/index.js +68 -5
  207. package/dist-lib/spec/hrpc/messages.js +260 -23
  208. package/dist-lib/spec/hyperschema/index.js +260 -23
  209. package/dist-lib/spec/hyperschema/schema.json +181 -0
  210. package/dist-lib/spec/hyperschema/types.d.ts +48 -0
  211. package/dist-lib/spec/local/hyperdb/messages.js +270 -224
  212. package/dist-lib/spec/local/hyperschema/index.js +73 -38
  213. package/dist-lib/spec/local/hyperschema/schema.json +31 -0
  214. package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
  215. package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
  216. package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
  217. package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
  218. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
  219. package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
  220. package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
  221. package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
  222. package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
  223. package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
  224. package/package.json +82 -71
  225. package/tui/README.md +52 -0
  226. package/tui/app.mjs +69 -12
  227. package/tui/boot.mjs +27 -6
  228. package/tui/cli.mjs +93 -0
  229. package/tui/index.mjs +2 -87
  230. package/tui/lib/attachment-files.mjs +82 -0
  231. package/tui/lib/reopen-meshes.mjs +15 -0
  232. package/tui/lib/updater.mjs +86 -0
  233. package/tui/main.mjs +29 -57
  234. package/worker/assistant-sidecar.mjs +10 -3
  235. package/worker/stow-entry.mjs +3 -52
  236. package/dist-lib/lib/core/chat-project.d.ts +0 -2
  237. package/dist-lib/lib/core/chat-project.js +0 -15
@@ -7,6 +7,7 @@ import { historyFromChunks, listHistoryChunks, replayFromChunks, runPromptView,
7
7
  import { nameSenders, restoreSentContent } from './message.js';
8
8
  import { PdfBoundsSchema } from './pdf-pages.js';
9
9
  import { isProbeRound, PROBE_KINDS } from '../probe-rounds.js';
10
+ import { MODEL_MISSING, mediaOptions, mediaOutputFrom, putBlob } from './media-round.js';
10
11
  const message = z.custom((value) => isRecord(value) && typeof value.role === 'string' && typeof value.content === 'string');
11
12
  const record = () => z.custom((value) => isRecord(value));
12
13
  export function wholeImageId(ref) {
@@ -32,23 +33,46 @@ const roundOptions = z.object({
32
33
  responseFormat: record().optional(),
33
34
  toolDialect: z.custom().optional(),
34
35
  unforced: record().optional(),
35
- media: z.array(z.object({ at: z.number(), refs: z.array(mediaRef) })).optional()
36
+ media: z.array(z.object({ at: z.number(), refs: z.array(mediaRef) })).optional(),
37
+ mediaRound: mediaOptions.optional()
36
38
  });
37
39
  const roundResult = z.custom((value) => isRecord(value) && typeof value.contentText === 'string' && isRecord(value.final));
40
+ export class Echoes {
41
+ _pending = { content: '', thinking: '' };
42
+ forwarded({ type, text }) {
43
+ this._pending[type] += text;
44
+ }
45
+ own(type, text) {
46
+ const pending = this._pending[type];
47
+ if (pending.startsWith(text)) {
48
+ this._pending[type] = pending.slice(text.length);
49
+ return '';
50
+ }
51
+ if (pending && text.startsWith(pending)) {
52
+ this._pending[type] = '';
53
+ return text.slice(pending.length);
54
+ }
55
+ return text;
56
+ }
57
+ }
38
58
  export class OwnerRounds {
39
59
  port;
60
+ media;
40
61
  inputLength = 0;
62
+ roundBase = 0;
41
63
  share = null;
42
64
  _opts;
43
- _echoes = [];
65
+ _echoes = new Echoes();
44
66
  _known = null;
45
67
  _contextSent = false;
46
68
  _rounds = 0;
47
69
  _retakes = 0;
48
70
  _takers = new Map();
71
+ _generated = [];
49
72
  constructor(opts) {
50
73
  this._opts = opts;
51
74
  this.port = (request, io) => this._round(request, io);
75
+ this.media = (request, io) => this._generate(request, io);
52
76
  }
53
77
  get rounds() {
54
78
  return this._rounds;
@@ -65,13 +89,8 @@ export class OwnerRounds {
65
89
  this._retakes--;
66
90
  return true;
67
91
  }
68
- echoed(type, text) {
69
- const at = this._echoes.findIndex((echo) => echo.type === type && echo.text === text);
70
- if (at === -1) {
71
- return false;
72
- }
73
- this._echoes.splice(at, 1);
74
- return true;
92
+ unechoed(type, text) {
93
+ return this._echoes.own(type, text);
75
94
  }
76
95
  async retire() {
77
96
  const { engine, chatId, runId } = this._opts;
@@ -138,7 +157,7 @@ export class OwnerRounds {
138
157
  const { modelId: asked, generationParams } = options;
139
158
  const call = { kind, modelId: asked, messages, generationParams, responseFormat };
140
159
  try {
141
- const { contentText } = await this._probed({ id, agentId }, signal);
160
+ const { contentText } = await this._probed({ id, agentId }, signal, (result) => roundResult.parse(result));
142
161
  this._opts.inspect?.({ ...call, outcome: 'completed', result: contentText });
143
162
  return contentText;
144
163
  }
@@ -147,7 +166,56 @@ export class OwnerRounds {
147
166
  throw err;
148
167
  }
149
168
  }
150
- async _probed(asked, signal) {
169
+ async _generate(request, io) {
170
+ if (io.signal.aborted)
171
+ throw new AbortedError();
172
+ const { engine, chatId, runId, deviceId, meshId, local } = this._opts;
173
+ const agentId = (await this._switchedTo()) ?? this._opts.agentId;
174
+ const { modelChoice, params } = request;
175
+ const refs = this._named(request.sources);
176
+ const sources = refs || !meshId
177
+ ? undefined
178
+ : await Promise.all((request.images ?? []).map((bytes) => putBlob(engine, meshId, bytes)));
179
+ const options = {
180
+ kind: request.kind,
181
+ attempt: 0,
182
+ modelId: modelChoice ?? request.kind,
183
+ continuationNudge: '',
184
+ deliveredNudge: '',
185
+ mediaRound: { modelChoice, params, sources, refs }
186
+ };
187
+ const { id } = await engine.appendChunk({
188
+ chatId,
189
+ runId,
190
+ type: 'model-request',
191
+ originDeviceId: deviceId,
192
+ modelRequest: { round: 0, agents: [agentId], options, state: 'open' }
193
+ });
194
+ const settled = await this._probed({ id, agentId }, io.signal, (result) => result);
195
+ const { result, generated } = await mediaOutputFrom(engine, settled, io.signal);
196
+ if ('error' in result && result.code === MODEL_MISSING && local) {
197
+ return local(request, io);
198
+ }
199
+ if (generated) {
200
+ this._generated.push(generated);
201
+ }
202
+ return result;
203
+ }
204
+ generated(bytes) {
205
+ const at = this._generated.findIndex((output) => b4a.equals(output.bytes, bytes));
206
+ if (at === -1) {
207
+ return undefined;
208
+ }
209
+ return this._generated.splice(at, 1)[0].blobId;
210
+ }
211
+ _named(sources) {
212
+ const refs = (sources ?? []).flatMap(({ attachmentId, path, width, height }) => {
213
+ const id = attachmentId ?? (path ? wholeImageId(this._opts.media?.(path)) : undefined);
214
+ return id ? [{ attachmentId: id, width, height }] : [];
215
+ });
216
+ return refs.length && refs.length === sources?.length ? refs : undefined;
217
+ }
218
+ async _probed(asked, signal, parse) {
151
219
  const { engine, chatId, from } = this._opts;
152
220
  const watch = engine.chunksDeltaWatch({ chatId, gte: from || undefined });
153
221
  const agents = engine.agentsWatch({});
@@ -160,9 +228,8 @@ export class OwnerRounds {
160
228
  for await (const frame of watch) {
161
229
  round = frame.chunks?.find((chunk) => chunk.id === asked.id)?.modelRequest ?? round;
162
230
  ended = !!round && round.state !== 'open' && round.state !== 'taken';
163
- const result = roundEnd(round);
164
- if (result) {
165
- return result;
231
+ if (roundDone(round)) {
232
+ return parse(round?.result);
166
233
  }
167
234
  }
168
235
  throw new AbortedError();
@@ -172,22 +239,27 @@ export class OwnerRounds {
172
239
  watch.destroy();
173
240
  agents.destroy();
174
241
  if (!ended) {
242
+ this._opts.log?.info('rounds', 'probe cancelled', {
243
+ roundId: asked.id,
244
+ reason: signal.aborted ? 'aborted' : 'watch ended'
245
+ });
175
246
  await engine.cancelModelRequest({ chunkId: asked.id }).catch(noop);
176
247
  }
177
248
  }
178
249
  }
179
250
  async _dropOnHostLoss(agents, asked, round) {
180
- let reachable = false;
181
251
  for await (const _frame of agents) {
182
252
  const agent = await this._opts.engine.getAgent({ agentId: asked.agentId }).catch(() => null);
183
253
  if (!agent) {
184
254
  continue;
185
255
  }
186
- const was = reachable;
187
- reachable = !!agent.hostReachable;
188
256
  const host = round()?.hostDeviceId;
189
257
  const moved = !!host && !b4a.equals(agent.providerDeviceId, host);
190
- if (moved || (was && !reachable)) {
258
+ if (moved || !agent.hostReachable) {
259
+ this._opts.log?.info('rounds', 'probe cancelled', {
260
+ roundId: asked.id,
261
+ reason: moved ? 'provider moved' : 'host lost'
262
+ });
191
263
  await this._opts.engine.cancelModelRequest({ chunkId: asked.id }).catch(noop);
192
264
  return;
193
265
  }
@@ -205,7 +277,7 @@ export class OwnerRounds {
205
277
  type: 'model-request',
206
278
  originDeviceId: deviceId,
207
279
  modelRequest: {
208
- round: sent.round,
280
+ round: sent.round + this.roundBase,
209
281
  agents: [agentId],
210
282
  cacheKey: sent.cacheKey,
211
283
  promptPrefix: sent.promptPrefix,
@@ -214,6 +286,14 @@ export class OwnerRounds {
214
286
  state: 'open'
215
287
  }
216
288
  });
289
+ this._opts.log?.info('rounds', 'round asked', {
290
+ runId,
291
+ roundId: id,
292
+ chatId,
293
+ mesh: this._opts.meshId?.slice(0, 8),
294
+ agentId,
295
+ round: sent.round + this.roundBase
296
+ });
217
297
  return id;
218
298
  }
219
299
  async _modelOf(agentId, modelId) {
@@ -238,7 +318,10 @@ export class OwnerRounds {
238
318
  runId,
239
319
  type: 'context',
240
320
  originDeviceId: deviceId,
241
- context: { turn: { sent: sent?.content }, toolSetHash: request.toolSet ?? null }
321
+ context: {
322
+ turn: { sent: sent?.content, shots: this._pdfShots(sent) },
323
+ toolSetHash: request.toolSet ?? null
324
+ }
242
325
  });
243
326
  }
244
327
  async _options(request, system) {
@@ -264,6 +347,17 @@ export class OwnerRounds {
264
347
  }
265
348
  return options;
266
349
  }
350
+ _pdfShots(turn) {
351
+ return (turn?.attachments ?? []).flatMap(({ path }) => {
352
+ const ref = this._opts.media?.(path);
353
+ if (ref?.page === undefined || ref.frame !== undefined) {
354
+ return [];
355
+ }
356
+ return [
357
+ ref.clip ? { id: ref.id, page: ref.page, clip: ref.clip } : { id: ref.id, page: ref.page }
358
+ ];
359
+ });
360
+ }
267
361
  _namedImages(turn) {
268
362
  const imageIds = (turn.attachments ?? []).flatMap(({ path }) => wholeImageId(this._opts.media?.(path)) ?? []);
269
363
  return imageIds.length ? { ...turn, imageIds } : turn;
@@ -305,17 +399,36 @@ export class OwnerRounds {
305
399
  io.signal.addEventListener('abort', abort);
306
400
  const at = { asked: first, moving: null, host: null };
307
401
  const output = new RoundOutput(first.id, io, {
308
- forwarded: (echo) => this._echoes.push(echo),
402
+ forwarded: (echo) => this._echoes.forwarded(echo),
309
403
  retook: () => this._retakes++
310
404
  });
311
405
  const move = async (agentId) => {
312
406
  at.moving = { agentId, after: output.newest(), emit: io.retake() };
407
+ output.hold();
313
408
  await engine.cancelModelRequest({ chunkId: at.asked.id }).catch(noop);
314
409
  };
315
410
  void this._followHost(agents, { at, output, move }).catch(noop);
411
+ const since = Date.now();
412
+ let takenAt;
316
413
  try {
317
414
  for await (const frame of watch) {
318
415
  const round = output.apply(frame);
416
+ if (round?.state === 'taken' && round.takenBy && round.takenAt !== takenAt) {
417
+ takenAt = round.takenAt;
418
+ this._opts.log?.debug('rounds', 'round taken', {
419
+ roundId: at.asked.id,
420
+ takenBy: shortId(round.takenBy)
421
+ });
422
+ }
423
+ if (!at.moving && round && round.state !== 'open' && round.state !== 'taken') {
424
+ this._opts.log?.info('rounds', 'round ended', {
425
+ roundId: at.asked.id,
426
+ state: round.state,
427
+ takenBy: round.takenBy ? shortId(round.takenBy) : undefined,
428
+ host: round.hostDeviceId ? shortId(round.hostDeviceId) : undefined,
429
+ ms: Date.now() - since
430
+ });
431
+ }
319
432
  const result = at.moving ? undefined : roundEnd(round);
320
433
  if (result) {
321
434
  this._takers.set(sent.round, round?.takenBy ?? null);
@@ -360,12 +473,23 @@ export class OwnerRounds {
360
473
  if (provider !== b4a.toHex(host)) {
361
474
  if (at.host.moved !== provider) {
362
475
  at.host.moved = provider;
476
+ this._opts.log?.info('rounds', 'round moved', {
477
+ roundId: at.asked.id,
478
+ reason: 'provider moved',
479
+ host: shortId(host),
480
+ provider: provider.slice(0, 8)
481
+ });
363
482
  await move(agentId);
364
483
  }
365
484
  continue;
366
485
  }
367
486
  const lost = was?.provider === provider && was.reachable && !at.host.reachable;
368
487
  if (lost && output.round()?.state === 'taken') {
488
+ this._opts.log?.info('rounds', 'round reopened', {
489
+ roundId: at.asked.id,
490
+ reason: 'host lost',
491
+ host: provider.slice(0, 8)
492
+ });
369
493
  await this._opts.engine.reopenModelRequest({ chunkId: at.asked.id }).catch(noop);
370
494
  }
371
495
  }
@@ -416,6 +540,9 @@ export class RoundOutput {
416
540
  }
417
541
  return round;
418
542
  }
543
+ hold() {
544
+ this._emit = null;
545
+ }
419
546
  reask(id, emit) {
420
547
  this._id = id;
421
548
  this._emit = emit;
@@ -463,16 +590,21 @@ export class RoundOutput {
463
590
  this._takenAt = now ? at : this._takenAt;
464
591
  }
465
592
  _forward(chunk) {
593
+ const emit = this._emit;
466
594
  const text = chunk.content?.text ?? chunk.thinking?.text;
467
595
  const type = chunk.type;
468
596
  const fresh = chunk.parentId === this._id && !this._emitted.has(chunk.id) && chunk.seq > this._resetAt;
469
597
  const taken = !this._reopened && this._taker && b4a.equals(chunk.originDeviceId, this._taker);
470
- if (!fresh || !taken || text === undefined || (type !== 'content' && type !== 'thinking')) {
598
+ if (!emit ||
599
+ !fresh ||
600
+ !taken ||
601
+ text === undefined ||
602
+ (type !== 'content' && type !== 'thinking')) {
471
603
  return;
472
604
  }
473
605
  this._emitted.add(chunk.id);
474
606
  this._hooks.forwarded({ type, text });
475
- this._emit({ type, text });
607
+ emit({ type, text });
476
608
  }
477
609
  }
478
610
  export async function listRoundChunks(engine, chatId) {
@@ -609,17 +741,17 @@ function carried(rounds, promptPrefix, toolSet) {
609
741
  }
610
742
  return { system, tools: toolSet ? sets.resolve(toolSet) : undefined };
611
743
  }
612
- function roundEnd(round) {
613
- if (round?.state === 'done') {
614
- return roundResult.parse(round.result);
615
- }
744
+ function roundDone(round) {
616
745
  if (round?.state === 'failed') {
617
746
  throw new Error(round.reason || 'the model round failed');
618
747
  }
619
748
  if (round?.state === 'cancelled') {
620
749
  throw new AbortedError();
621
750
  }
622
- return undefined;
751
+ return round?.state === 'done';
752
+ }
753
+ function roundEnd(round) {
754
+ return roundDone(round) ? roundResult.parse(round?.result) : undefined;
623
755
  }
624
756
  function lastSwitch(chunks, from) {
625
757
  let last;
@@ -631,3 +763,6 @@ function lastSwitch(chunks, from) {
631
763
  return last?.agentSwitch?.agentId;
632
764
  }
633
765
  function noop() { }
766
+ function shortId(key) {
767
+ return b4a.toHex(key).slice(0, 8);
768
+ }
@@ -13,6 +13,17 @@ export interface PdfSelection {
13
13
  readonly shots: ReadonlyArray<PdfShot>;
14
14
  readonly cutAt?: number;
15
15
  }
16
+ interface PdfHit {
17
+ readonly metadataJson: string;
18
+ }
19
+ interface PdfTextHit extends PdfHit {
20
+ readonly content: string;
21
+ }
22
+ export interface PdfSelectionOptions {
23
+ readonly cited?: ReadonlySet<number>;
24
+ readonly textFirst?: boolean;
25
+ readonly readablePages?: ReadonlySet<number> | null;
26
+ }
16
27
  export declare const PdfBoundsSchema: z.ZodObject<{
17
28
  x: z.ZodNumber;
18
29
  y: z.ZodNumber;
@@ -22,4 +33,8 @@ export declare const PdfBoundsSchema: z.ZodObject<{
22
33
  export declare function pdfSelectionFileName(pdfFileName: string): string;
23
34
  export declare function packPdfSelection(selection: PdfSelection): Uint8Array;
24
35
  export declare function parsePdfSelection(bytes: Uint8Array): PdfSelection;
25
- export declare function selectPdfPages(doc: PdfDoc, room: PdfRoom): Promise<PdfSelection>;
36
+ export declare function asksForPdfVisuals(question: string): boolean;
37
+ export declare function readablePdfTextPages(hits: ReadonlyArray<PdfTextHit>): Map<string, Set<number> | null>;
38
+ export declare function citedPdfPages(hits: ReadonlyArray<PdfHit>): Map<string, Set<number> | null>;
39
+ export declare function selectPdfPages(doc: PdfDoc, room: PdfRoom, { cited, textFirst, readablePages }?: PdfSelectionOptions): Promise<PdfSelection>;
40
+ export {};
@@ -1,4 +1,5 @@
1
1
  import b4a from 'b4a';
2
+ import { parseRagMetadata } from 'rag-extract';
2
3
  import * as z from 'zod';
3
4
  import { frameTokens } from './video-budget.js';
4
5
  export const PDF_PAGES_MIME = 'application/vnd.qvac.pdf-pages';
@@ -31,14 +32,74 @@ export function parsePdfSelection(bytes) {
31
32
  function shotLabel(page, names = []) {
32
33
  return names.length ? `page ${page + 1} (${names.join(', ')})` : `page ${page + 1}`;
33
34
  }
34
- export async function selectPdfPages(doc, room) {
35
+ const VISUAL_REQUEST = /\b(?:charts?|figures?|diagrams?|graphs?|plots?|maps?|images?|pictures?|photos?|illustrations?|visuals?|layouts?|handwriting|handwritten|scanned|screenshots?)\b|\b(?:this|that|the|attached)\s+scan\b/i;
36
+ const TABLE_REFERENCE = /\b(?:the|this|that|attached|above|below|following|all|any|each)\s+tables?\b|\btables?\s*(?:#?\s*\d+|[A-Z]\b|of|on|from|above|below)\b|\b(?:read|show|describe|explain|extract|summari[sz]e|analy[sz]e|find|inspect|compare)\s+(?:me\s+)?(?:(?:all|any|each)\s+)?(?:the\s+|this\s+|that\s+)?tables?\b/i;
37
+ const TABLE_MENTION = /\btables?\b/i;
38
+ const TABLE_OUTPUT = /\b(?:in|as|into)\s+(?:a\s+|the\s+)?table\b|\b(?:make|create|generate|compile|write|produce)\b[^.!?\n]{0,40}\b(?:a\s+|an\s+)?table\b/i;
39
+ export function asksForPdfVisuals(question) {
40
+ if (VISUAL_REQUEST.test(question))
41
+ return true;
42
+ const tableQuestion = question.replace(TABLE_OUTPUT, ' ');
43
+ if (TABLE_REFERENCE.test(tableQuestion)) {
44
+ return true;
45
+ }
46
+ return TABLE_MENTION.test(question) && !TABLE_OUTPUT.test(question);
47
+ }
48
+ export function readablePdfTextPages(hits) {
49
+ const readable = new Map();
50
+ for (const { content, metadataJson } of hits) {
51
+ if (typeof content !== 'string')
52
+ continue;
53
+ const metadata = parseRagMetadata(metadataJson);
54
+ const text = content.normalize('NFKC').replace(/\s+/gu, ' ').trim();
55
+ if (!metadata?.attachmentId || !hasReadableContent(text, metadata.tokenCount)) {
56
+ continue;
57
+ }
58
+ const pages = readable.get(metadata.attachmentId);
59
+ if (metadata.page === null || pages === null) {
60
+ readable.set(metadata.attachmentId, null);
61
+ continue;
62
+ }
63
+ readable.set(metadata.attachmentId, (pages ?? new Set()).add(metadata.page));
64
+ }
65
+ return readable;
66
+ }
67
+ function hasReadableContent(text, cachedChunkTokens) {
68
+ const words = text.match(/[\p{L}\p{N}]{2,}/gu) ?? [];
69
+ if (text.length < 24 ||
70
+ words.length < 4 ||
71
+ /^(?:files in this chat:|document outline(?:\b|$)|readable,\s+\d+\s+(?:pages?|lines?)(?:\b|$))/i.test(text)) {
72
+ return false;
73
+ }
74
+ return text.length >= 80 || cachedChunkTokens >= 24;
75
+ }
76
+ export function citedPdfPages(hits) {
77
+ const cited = new Map();
78
+ for (const { metadataJson } of hits) {
79
+ const metadata = parseRagMetadata(metadataJson);
80
+ if (!metadata?.attachmentId) {
81
+ continue;
82
+ }
83
+ const pages = cited.get(metadata.attachmentId);
84
+ if (metadata.page === null || pages === null) {
85
+ cited.set(metadata.attachmentId, null);
86
+ continue;
87
+ }
88
+ cited.set(metadata.attachmentId, (pages ?? new Set()).add(metadata.page));
89
+ }
90
+ return cited;
91
+ }
92
+ export async function selectPdfPages(doc, room, { cited, textFirst = false, readablePages } = {}) {
35
93
  const pageCount = doc.pageCount();
36
94
  const shots = [];
37
95
  let cutAt;
38
96
  for (let page = 0; page < pageCount; page++) {
97
+ if (cited && !cited.has(page + 1)) {
98
+ continue;
99
+ }
39
100
  let shot;
40
101
  try {
41
- shot = await pictured(doc, page);
102
+ shot = await pictured(doc, page, { textFirst, readablePages });
42
103
  }
43
104
  catch {
44
105
  continue;
@@ -60,8 +121,11 @@ export async function selectPdfPages(doc, room) {
60
121
  }
61
122
  return cutAt === undefined ? { pageCount, shots } : { pageCount, shots, cutAt };
62
123
  }
63
- async function pictured(doc, page) {
124
+ async function pictured(doc, page, { textFirst, readablePages }) {
64
125
  const flags = doc.pageFlags(page);
126
+ if (textFirst && (readablePages?.has(page + 1) || (readablePages === null && flags.hasText))) {
127
+ return null;
128
+ }
65
129
  if (flags.hasText && !flags.hasImage) {
66
130
  return null;
67
131
  }
@@ -1,17 +1,24 @@
1
1
  import { type AbortSignal } from 'bare-abort-controller';
2
+ import type { ModelRoundRequest } from '@qvac/harness';
3
+ import type { MediaRoundPort } from '@qvac/harness/lib/contract/index.ts';
2
4
  import type { LoggerInterface } from '@qvac/logging';
3
5
  import type { AssistantChunk, AssistantChunkList } from '../../spec/hyperschema/types.d.ts';
4
6
  import type { Capabilities, WatchStream } from '../api/index.ts';
7
+ import { type ScopedCapabilities } from '../mesh-scoped-engine.ts';
5
8
  import type { Harness } from './harness.ts';
9
+ import type { MediaKind } from '../probe-rounds.ts';
6
10
  import { type MediaSource } from './model-round.ts';
7
11
  export interface RoundHostOptions {
8
12
  readonly engine: Capabilities;
9
13
  readonly serve: NonNullable<Harness['serveModelRound']>;
10
14
  readonly deviceId: () => Buffer;
11
15
  readonly media?: RoundMedia | null;
16
+ readonly load?: RoundLoad | null;
17
+ readonly generation?: RoundGeneration | null;
12
18
  readonly built?: (meshId: string, chatId: string) => void;
13
19
  readonly log?: LoggerInterface | null;
14
20
  }
21
+ export type RoundLoad = (engine: ScopedCapabilities, agentId: string) => Promise<ModelRoundRequest['load']>;
15
22
  export interface RoundMedia {
16
23
  resolve(sources: ReadonlyArray<MediaSource>, served: {
17
24
  readonly id: string;
@@ -20,6 +27,16 @@ export interface RoundMedia {
20
27
  }): Promise<Array<string | null>>;
21
28
  release(id: string): void;
22
29
  }
30
+ export interface GenerationWait {
31
+ readonly engine: ScopedCapabilities;
32
+ readonly agentId: string;
33
+ readonly kind: MediaKind;
34
+ readonly signal: AbortSignal;
35
+ }
36
+ export interface RoundGeneration {
37
+ readonly serve: MediaRoundPort;
38
+ ready(wait: GenerationWait): Promise<boolean>;
39
+ }
23
40
  export declare function servedCacheKey(round: AssistantChunk, ownerKey: string | null | undefined): string | undefined;
24
41
  export declare class RoundHost {
25
42
  private readonly _opts;
@@ -31,14 +48,19 @@ export declare class RoundHost {
31
48
  left(chatId: string, agentId: string): string[];
32
49
  holds(chatId: string): boolean;
33
50
  stop(): void;
51
+ drop(meshId: string): void;
34
52
  private _reconcile;
35
53
  private _pick;
36
54
  private _serve;
37
55
  private _take;
38
56
  private _provides;
57
+ private _load;
39
58
  private _answer;
59
+ private _generate;
60
+ private _stored;
40
61
  private _probe;
41
62
  private _claimKey;
42
63
  private _retake;
43
64
  private _takeFor;
65
+ private _settle;
44
66
  }