@qvac/core 1.0.0-rc.2 → 1.0.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/CHANGELOG.md +158 -2
  2. package/dist-lib/index.d.ts +1 -0
  3. package/dist-lib/index.js +1 -0
  4. package/dist-lib/lib/api/chunk-delta.d.ts +7 -2
  5. package/dist-lib/lib/api/chunk-delta.js +89 -7
  6. package/dist-lib/lib/api/chunks.js +14 -8
  7. package/dist-lib/lib/api/config.d.ts +4 -1
  8. package/dist-lib/lib/api/config.js +45 -4
  9. package/dist-lib/lib/api/custom-models.js +3 -5
  10. package/dist-lib/lib/api/devices.js +3 -1
  11. package/dist-lib/lib/api/files.d.ts +1 -1
  12. package/dist-lib/lib/api/files.js +65 -8
  13. package/dist-lib/lib/api/instrument.js +12 -4
  14. package/dist-lib/lib/api/members.js +7 -13
  15. package/dist-lib/lib/api/mesh.d.ts +1 -3
  16. package/dist-lib/lib/api/mesh.js +2 -2
  17. package/dist-lib/lib/api/notifications.js +8 -4
  18. package/dist-lib/lib/api/projects.js +5 -1
  19. package/dist-lib/lib/api/rag.js +32 -19
  20. package/dist-lib/lib/api/served.js +2 -1
  21. package/dist-lib/lib/api/workspace.js +0 -1
  22. package/dist-lib/lib/assistant/harness.d.ts +8 -2
  23. package/dist-lib/lib/assistant/harness.js +17 -4
  24. package/dist-lib/lib/assistant/history.d.ts +1 -1
  25. package/dist-lib/lib/assistant/history.js +19 -13
  26. package/dist-lib/lib/assistant/message.d.ts +6 -1
  27. package/dist-lib/lib/assistant/message.js +22 -2
  28. package/dist-lib/lib/assistant/model-round.d.ts +5 -0
  29. package/dist-lib/lib/assistant/model-round.js +46 -18
  30. package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
  31. package/dist-lib/lib/assistant/pdf-budget.js +10 -11
  32. package/dist-lib/lib/assistant/pdf-pages.d.ts +25 -0
  33. package/dist-lib/lib/assistant/pdf-pages.js +75 -0
  34. package/dist-lib/lib/assistant/round-host.js +5 -3
  35. package/dist-lib/lib/assistant-client.d.ts +2 -1
  36. package/dist-lib/lib/assistant-client.js +24 -6
  37. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  38. package/dist-lib/lib/assistant-tree.js +8 -5
  39. package/dist-lib/lib/assistant.d.ts +20 -7
  40. package/dist-lib/lib/assistant.js +468 -240
  41. package/dist-lib/lib/client.js +10 -1
  42. package/dist-lib/lib/compaction.d.ts +3 -1
  43. package/dist-lib/lib/compaction.js +18 -4
  44. package/dist-lib/lib/constants.d.ts +3 -0
  45. package/dist-lib/lib/constants.js +3 -0
  46. package/dist-lib/lib/core/chat-activity.js +13 -16
  47. package/dist-lib/lib/core/chat-copy.js +1 -1
  48. package/dist-lib/lib/core/chat-fork.js +2 -1
  49. package/dist-lib/lib/core/chat-move.d.ts +4 -0
  50. package/dist-lib/lib/core/chat-move.js +42 -11
  51. package/dist-lib/lib/core/chat.d.ts +0 -1
  52. package/dist-lib/lib/core/chat.js +1 -1
  53. package/dist-lib/lib/core/chunk.d.ts +14 -3
  54. package/dist-lib/lib/core/chunk.js +112 -22
  55. package/dist-lib/lib/core/collections.d.ts +12 -1
  56. package/dist-lib/lib/core/collections.js +12 -1
  57. package/dist-lib/lib/core/config-value.d.ts +6 -0
  58. package/dist-lib/lib/core/config-value.js +14 -0
  59. package/dist-lib/lib/core/contract/activity.d.ts +3 -2
  60. package/dist-lib/lib/core/contract/activity.js +39 -21
  61. package/dist-lib/lib/core/contract/agent-join.js +15 -19
  62. package/dist-lib/lib/core/contract/agents.js +8 -7
  63. package/dist-lib/lib/core/contract/chat.d.ts +3 -0
  64. package/dist-lib/lib/core/contract/chat.js +12 -1
  65. package/dist-lib/lib/core/contract/chunks.js +16 -6
  66. package/dist-lib/lib/core/contract/devices.js +6 -4
  67. package/dist-lib/lib/core/contract/index.js +2 -1
  68. package/dist-lib/lib/core/contract/invites.js +12 -31
  69. package/dist-lib/lib/core/contract/joined-meshes.js +2 -1
  70. package/dist-lib/lib/core/contract/mesh-rules.js +4 -0
  71. package/dist-lib/lib/core/contract/model-requests.js +7 -6
  72. package/dist-lib/lib/core/contract/project-device-keys.js +15 -6
  73. package/dist-lib/lib/core/contract/project-host-keys.js +5 -4
  74. package/dist-lib/lib/core/contract/projects.js +12 -13
  75. package/dist-lib/lib/core/contract/public-seeds.js +5 -5
  76. package/dist-lib/lib/core/contract/roles.js +2 -1
  77. package/dist-lib/lib/core/contract/run-requests.d.ts +1 -1
  78. package/dist-lib/lib/core/contract/run-requests.js +19 -13
  79. package/dist-lib/lib/core/contract/shipped-rows.d.ts +2 -0
  80. package/dist-lib/lib/core/contract/shipped-rows.js +22 -0
  81. package/dist-lib/lib/core/contract/workspace.js +4 -10
  82. package/dist-lib/lib/core/contract/writers.js +20 -6
  83. package/dist-lib/lib/core/devices.d.ts +2 -1
  84. package/dist-lib/lib/core/devices.js +4 -2
  85. package/dist-lib/lib/core/document-attachment.d.ts +6 -3
  86. package/dist-lib/lib/core/document-attachment.js +10 -5
  87. package/dist-lib/lib/core/files/attachments.js +3 -3
  88. package/dist-lib/lib/core/files/index.d.ts +8 -5
  89. package/dist-lib/lib/core/files/index.js +109 -18
  90. package/dist-lib/lib/core/files/processing/index.d.ts +1 -1
  91. package/dist-lib/lib/core/files/processing/index.js +8 -2
  92. package/dist-lib/lib/core/files/processing/source.d.ts +1 -0
  93. package/dist-lib/lib/core/files/processing/source.js +3 -0
  94. package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
  95. package/dist-lib/lib/core/files/processing/state.js +7 -0
  96. package/dist-lib/lib/core/files/rag/build.d.ts +1 -0
  97. package/dist-lib/lib/core/files/rag/build.js +14 -1
  98. package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
  99. package/dist-lib/lib/core/files/rag/index.js +1 -1
  100. package/dist-lib/lib/core/files/rag/produce.js +1 -0
  101. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +2 -0
  102. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +7 -1
  103. package/dist-lib/lib/core/first-event.d.ts +6 -0
  104. package/dist-lib/lib/core/first-event.js +13 -0
  105. package/dist-lib/lib/core/invite.d.ts +2 -2
  106. package/dist-lib/lib/core/invite.js +5 -4
  107. package/dist-lib/lib/core/local/index.d.ts +10 -1
  108. package/dist-lib/lib/core/local/index.js +105 -10
  109. package/dist-lib/lib/core/mesh.d.ts +15 -4
  110. package/dist-lib/lib/core/mesh.js +128 -57
  111. package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
  112. package/dist-lib/lib/core/migrations/chat-chunks.js +47 -0
  113. package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
  114. package/dist-lib/lib/core/migrations/file-index.js +6 -2
  115. package/dist-lib/lib/core/migrations/index.d.ts +2 -1
  116. package/dist-lib/lib/core/migrations/index.js +66 -9
  117. package/dist-lib/lib/core/notification.js +96 -23
  118. package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
  119. package/dist-lib/lib/core/pdf/figure.js +64 -0
  120. package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
  121. package/dist-lib/lib/core/pdf/rasterize.js +72 -22
  122. package/dist-lib/lib/core/peer-duplex.js +57 -5
  123. package/dist-lib/lib/core/public-mesh.js +8 -2
  124. package/dist-lib/lib/core/run-requests.js +2 -3
  125. package/dist-lib/lib/core/senders.d.ts +7 -6
  126. package/dist-lib/lib/core/senders.js +23 -30
  127. package/dist-lib/lib/core/shared-agents.d.ts +1 -0
  128. package/dist-lib/lib/core/shared-agents.js +47 -19
  129. package/dist-lib/lib/core/sync/background.d.ts +3 -15
  130. package/dist-lib/lib/core/sync/background.js +25 -102
  131. package/dist-lib/lib/core/sync/devices.js +6 -3
  132. package/dist-lib/lib/core/sync/index.js +12 -2
  133. package/dist-lib/lib/core/sync-status.js +3 -3
  134. package/dist-lib/lib/core.d.ts +59 -10
  135. package/dist-lib/lib/core.js +706 -174
  136. package/dist-lib/lib/credential-pusher.d.ts +11 -7
  137. package/dist-lib/lib/credential-pusher.js +40 -8
  138. package/dist-lib/lib/device-tree.d.ts +18 -0
  139. package/dist-lib/lib/device-tree.js +32 -0
  140. package/dist-lib/lib/doctor/redact.js +1 -1
  141. package/dist-lib/lib/errors.d.ts +4 -0
  142. package/dist-lib/lib/errors.js +4 -0
  143. package/dist-lib/lib/fs-drive-bridge.d.ts +7 -0
  144. package/dist-lib/lib/fs-drive-bridge.js +308 -30
  145. package/dist-lib/lib/integrations/setup-routes.js +7 -5
  146. package/dist-lib/lib/knowledge-search.d.ts +1 -1
  147. package/dist-lib/lib/knowledge-search.js +5 -46
  148. package/dist-lib/lib/kv-cache-retention.d.ts +35 -0
  149. package/dist-lib/lib/kv-cache-retention.js +98 -0
  150. package/dist-lib/lib/latest-frame.d.ts +10 -0
  151. package/dist-lib/lib/latest-frame.js +48 -0
  152. package/dist-lib/lib/model-download.d.ts +6 -1
  153. package/dist-lib/lib/model-download.js +32 -17
  154. package/dist-lib/lib/oauth/resolve.js +13 -5
  155. package/dist-lib/lib/reconnecting-watch.js +19 -11
  156. package/dist-lib/lib/run-notices.d.ts +2 -0
  157. package/dist-lib/lib/run-notices.js +5 -0
  158. package/dist-lib/lib/run-replay.d.ts +13 -1
  159. package/dist-lib/lib/run-replay.js +58 -21
  160. package/dist-lib/schema/build.js +66 -57
  161. package/dist-lib/schema/content.js +2 -1
  162. package/dist-lib/schema/engine.js +37 -5
  163. package/dist-lib/schema/field-order.js +7 -1
  164. package/dist-lib/schema/indexes.d.ts +4 -0
  165. package/dist-lib/schema/indexes.js +18 -0
  166. package/dist-lib/schema/local.js +9 -2
  167. package/dist-lib/schema/qvac.js +112 -4
  168. package/dist-lib/spec/bind.js +3 -0
  169. package/dist-lib/spec/calls.js +1 -0
  170. package/dist-lib/spec/capabilities.d.ts +2 -0
  171. package/dist-lib/spec/hrpc/hrpc.json +13 -0
  172. package/dist-lib/spec/hrpc/index.js +15 -3
  173. package/dist-lib/spec/hrpc/messages.js +1475 -1392
  174. package/dist-lib/spec/hyperschema/index.js +1475 -1392
  175. package/dist-lib/spec/hyperschema/schema.json +86 -1
  176. package/dist-lib/spec/hyperschema/types.d.ts +21 -1
  177. package/dist-lib/spec/local/hyperdb/messages.js +110 -54
  178. package/dist-lib/spec/local/hyperschema/index.js +56 -28
  179. package/dist-lib/spec/local/hyperschema/schema.json +55 -20
  180. package/dist-lib/spec/local/hyperschema/types.d.ts +8 -1
  181. package/dist-lib/spec/mesh/hyperdb/db.json +255 -51
  182. package/dist-lib/spec/mesh/hyperdb/index.js +1063 -404
  183. package/dist-lib/spec/mesh/hyperdb/messages.js +1647 -1214
  184. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +62 -56
  185. package/dist-lib/spec/mesh/hyperdispatch/index.js +156 -139
  186. package/dist-lib/spec/mesh/hyperdispatch/messages.js +787 -629
  187. package/dist-lib/spec/mesh/hyperschema/index.js +787 -629
  188. package/dist-lib/spec/mesh/hyperschema/schema.json +283 -174
  189. package/dist-lib/spec/mesh/hyperschema/types.d.ts +29 -1
  190. package/package.json +23 -17
  191. package/worker/assistant-sidecar.mjs +3 -0
  192. package/dist-lib/lib/core/files/processing/settle.d.ts +0 -13
  193. package/dist-lib/lib/core/files/processing/settle.js +0 -86
  194. package/dist-lib/lib/core/migrations/device-indexes.d.ts +0 -2
  195. package/dist-lib/lib/core/migrations/device-indexes.js +0 -7
@@ -79,7 +79,8 @@ export const ROUTE_ROLES = {
79
79
  '@qvac/put-agent-user': WORKSPACE,
80
80
  '@qvac/agent-join': CONTENT,
81
81
  '@qvac/take-run-request': CONTENT,
82
- '@qvac/put-skill-secret': ['account']
82
+ '@qvac/put-skill-secret': ['account'],
83
+ '@qvac/reindex-chat-chunks': CONTENT
83
84
  };
84
85
  export function inRole(role, route) {
85
86
  return ROUTE_ROLES[route]?.includes(role) ?? false;
@@ -2,7 +2,7 @@ import type HyperDB from 'hyperdb';
2
2
  import type { Router } from '../../../spec/mesh/hyperdispatch/index.js';
3
3
  import type { QvacAppendChunkOperation, QvacChunk } from '../../../spec/mesh/hyperschema/types.d.ts';
4
4
  import type { RouterContext } from './types.ts';
5
- export declare const TERMINAL_RUN_STATES: Set<string>;
5
+ export declare const TERMINAL_RUN_STATES: ReadonlySet<string>;
6
6
  export declare function runHasTerminalStatus(tx: HyperDB, chatId: string, runId: string): Promise<boolean>;
7
7
  export declare function runnerRefusal(op: QvacAppendChunkOperation, { tx, node }: RouterContext): Promise<string | null>;
8
8
  export declare function runTaker(tx: HyperDB, request: QvacChunk, writerKey: Buffer): Promise<Buffer | null>;
@@ -1,21 +1,27 @@
1
1
  import b4a from 'b4a';
2
2
  import { liveDeviceForWriter, trackActivity } from './activity.js';
3
+ import { putChunk } from './chat.js';
3
4
  import { ownerSigner } from './model-requests.js';
4
- import { chunkTypeOrdinal } from '../../../schema/indexes.js';
5
- import { CHUNKS, CHUNKS_BY_TYPE, DEVICES } from '../collections.js';
6
- export const TERMINAL_RUN_STATES = new Set(['completed', 'failed', 'canceled']);
5
+ import { WORKSPACES_RULES, rulesVersion } from './mesh-rules.js';
6
+ import { chunkTypeOrdinal, terminalRunStates } from '../../../schema/indexes.js';
7
+ import { CHUNKS, CHUNKS_BY_TYPE, DEVICES, TERMINAL_RUN_STATUSES } from '../collections.js';
8
+ export const TERMINAL_RUN_STATES = new Set(terminalRunStates);
7
9
  export async function runHasTerminalStatus(tx, chatId, runId) {
8
- const type = chunkTypeOrdinal('run-status');
9
- const statuses = tx.find(CHUNKS_BY_TYPE, {
10
- gte: { chatId, type },
11
- lte: { chatId, type }
12
- });
13
- for await (const chunk of statuses) {
14
- if (chunk.runId === runId && TERMINAL_RUN_STATES.has(chunk.runStatus?.state ?? '')) {
15
- return true;
10
+ if ((await rulesVersion(tx)) < WORKSPACES_RULES) {
11
+ const type = chunkTypeOrdinal('run-status');
12
+ const statuses = tx.find(CHUNKS_BY_TYPE, {
13
+ gte: { chatId, type },
14
+ lte: { chatId, type }
15
+ });
16
+ for await (const chunk of statuses) {
17
+ if (chunk.runId === runId && TERMINAL_RUN_STATES.has(chunk.runStatus?.state ?? '')) {
18
+ return true;
19
+ }
16
20
  }
21
+ return false;
17
22
  }
18
- return false;
23
+ const range = { gte: { chatId, runId }, lte: { chatId, runId } };
24
+ return (await tx.findOne(TERMINAL_RUN_STATUSES, range)) !== null;
19
25
  }
20
26
  export async function runnerRefusal(op, { tx, node }) {
21
27
  if (!op.runnerDeviceId) {
@@ -69,7 +75,7 @@ export function registerRunRequestRoutes(router) {
69
75
  runTakenAt: op.at,
70
76
  updatedAt: Math.max(request.updatedAt ?? 0, op.at)
71
77
  };
72
- await tx.insert(CHUNKS, taken);
78
+ await putChunk(tx, taken);
73
79
  return taken;
74
80
  }));
75
81
  }
@@ -0,0 +1,2 @@
1
+ import type HyperDB from 'hyperdb';
2
+ export declare function reindexShippedRows(tx: HyperDB): Promise<void>;
@@ -0,0 +1,22 @@
1
+ import { chunkTypeOrdinal } from '../../../schema/indexes.js';
2
+ import { CHUNK_COUNTERS, CHUNKS, CHUNKS_BY_TYPE, DEVICES } from '../collections.js';
3
+ export async function reindexShippedRows(tx) {
4
+ const devices = await tx.find(DEVICES).toArray();
5
+ for (const device of devices) {
6
+ await tx.insert(DEVICES, { ...device, version: undefined });
7
+ }
8
+ const statuses = [];
9
+ for await (const chunk of runStatuses(tx)) {
10
+ statuses.push(chunk);
11
+ }
12
+ for (const chunk of statuses) {
13
+ await tx.insert(CHUNKS, { ...chunk, version: undefined });
14
+ }
15
+ }
16
+ async function* runStatuses(db) {
17
+ const type = chunkTypeOrdinal('run-status');
18
+ for await (const { chatId } of db.find(CHUNK_COUNTERS)) {
19
+ const range = { gte: { chatId, type }, lte: { chatId, type } };
20
+ yield* db.find(CHUNKS_BY_TYPE, range);
21
+ }
22
+ }
@@ -1,4 +1,4 @@
1
- import { liveDeviceForWriter, trackActivity } from './activity.js';
1
+ import { liveDeviceForWriter, liveDevices, liveDevicesOf, trackActivity } from './activity.js';
2
2
  import { meshKind } from './mesh-kind.js';
3
3
  import { WORKSPACES_RULES, bornUnderRules, rulesVersion } from './mesh-rules.js';
4
4
  import b4a from 'b4a';
@@ -52,8 +52,8 @@ function ownerOf(device) {
52
52
  }
53
53
  export async function liveMemberIds(tx) {
54
54
  const live = new Set();
55
- for await (const device of tx.find(DEVICES)) {
56
- const owner = device.revoked ? null : ownerOf(device);
55
+ for await (const device of liveDevices(tx)) {
56
+ const owner = ownerOf(device);
57
57
  if (owner) {
58
58
  live.add(b4a.toHex(owner));
59
59
  }
@@ -100,13 +100,7 @@ export async function adminsLastDevice(tx, device) {
100
100
  if (device.revoked || !owner || (await memberRole(tx, owner)) !== 'admin') {
101
101
  return false;
102
102
  }
103
- for await (const other of tx.find(DEVICES)) {
104
- const theirs = ownerOf(other);
105
- if (!other.revoked && theirs && b4a.equals(theirs, owner) && !b4a.equals(other.id, device.id)) {
106
- return false;
107
- }
108
- }
109
- return true;
103
+ return (await liveDevicesOf(tx, owner)).every((other) => b4a.equals(other.id, device.id));
110
104
  }
111
105
  export async function leavesNoAdmin(context, memberId) {
112
106
  if (!(await isWorkspace(context.tx))) {
@@ -1,5 +1,5 @@
1
1
  import crypto from 'hypercore-crypto';
2
- import { devicesByWriterKey, liveDeviceForWriter, markWriterRevoked, trackActivity } from './activity.js';
2
+ import { devicesByWriterKey, devicesOfMember, liveDeviceForWriter, markWriterRevoked, trackActivity } from './activity.js';
3
3
  import { recordMemberRemoval } from './grant-order.js';
4
4
  import { verifyMemberWriterClaim, verifyWriterClaim } from '../device-attestation.js';
5
5
  import { meshKind, sharedKind } from './mesh-kind.js';
@@ -14,6 +14,7 @@ import { dropHostRows, hostedDevice, releaseHostedDevice } from './agent-join.js
14
14
  import { actsFor, dropRole, kicksAdminsLastDevice, leavesNoAdmin, uncountMember } from './workspace.js';
15
15
  import { verifyDeviceProof } from '../identity.js';
16
16
  import { sealed } from './seal.js';
17
+ import { WORKSPACES_RULES, rulesVersion } from './mesh-rules.js';
17
18
  import b4a from 'b4a';
18
19
  import { DEVICES, MEMBERS, MESH_INVITE_GRANTS, PROJECT_MEMBERS } from '../collections.js';
19
20
  async function boundToAnother(tx, writerKey, id) {
@@ -186,6 +187,19 @@ async function revokeWriter(payload, context) {
186
187
  }
187
188
  return revoked;
188
189
  }
190
+ async function ownLeave({ writerKey, deviceId, revokedAt }, { tx, node }) {
191
+ if (!b4a.equals(node.key, writerKey) || !deviceId) {
192
+ return false;
193
+ }
194
+ if ((await rulesVersion(tx)) < WORKSPACES_RULES) {
195
+ return false;
196
+ }
197
+ const device = await tx.get(DEVICES, { id: deviceId });
198
+ if (!device?.writerKey || !b4a.equals(device.writerKey, writerKey)) {
199
+ return false;
200
+ }
201
+ return device.joinedAt <= (revokedAt ?? 0);
202
+ }
189
203
  export function registerWriterRoutes(router) {
190
204
  router.add('@qvac/add-writer', trackActivity(async (payload, context) => {
191
205
  if (await meshKind(context.tx)) {
@@ -219,6 +233,10 @@ export function registerWriterRoutes(router) {
219
233
  return payload.device;
220
234
  }));
221
235
  router.add('@qvac/remove-writer', trackActivity(async (payload, context) => {
236
+ if (context.node.optimistic && !(await ownLeave(payload, context))) {
237
+ context.log?.warn('contract', 'remove-writer dropped: its author is not a writer here');
238
+ return;
239
+ }
222
240
  const self = b4a.equals(context.node.key, payload.writerKey);
223
241
  if (!self && !(await hasStanding(context))) {
224
242
  context.log?.warn('contract', 'remove-writer dropped: the author speaks for no member here');
@@ -256,11 +274,7 @@ export function registerWriterRoutes(router) {
256
274
  await context.tx.delete(PROJECT_MEMBERS, { memberId });
257
275
  const removals = [];
258
276
  const devices = [];
259
- for await (const device of context.tx.find(DEVICES)) {
260
- const claims = device.memberId ?? device.claimedMemberId;
261
- if (!claims || !b4a.equals(claims, memberId)) {
262
- continue;
263
- }
277
+ for (const device of await devicesOfMember(context.tx, memberId)) {
264
278
  devices.push(device.id);
265
279
  if (!device.revoked && device.writerKey) {
266
280
  removals.push({ writerKey: device.writerKey, deviceId: device.id, revokedAt });
@@ -1,4 +1,4 @@
1
- import type { QvacToolReportEntry } from '../../spec/mesh/hyperschema/types.d.ts';
1
+ import type { QvacCustomModelReportEntry, QvacToolReportEntry } from '../../spec/mesh/hyperschema/types.d.ts';
2
2
  import type { AppBuildMetadata } from './app-build.ts';
3
3
  import type { Mesh } from './mesh.ts';
4
4
  export interface SaveDeviceCapabilitiesInput {
@@ -9,6 +9,7 @@ export interface SaveDeviceCapabilitiesInput {
9
9
  cachedModelNames: string[];
10
10
  readySkillNames?: string[] | null;
11
11
  toolCatalog?: QvacToolReportEntry[] | null;
12
+ customModels?: QvacCustomModelReportEntry[] | null;
12
13
  }
13
14
  export declare function saveDeviceCapabilities(store: Mesh, input: SaveDeviceCapabilitiesInput, isAccount: (mesh: Mesh) => boolean): Promise<void>;
14
15
  export declare function retractInventory(store: Mesh, deviceId: Buffer, isAccount: (mesh: Mesh) => boolean): Promise<void>;
@@ -32,7 +32,8 @@ export async function retractInventory(store, deviceId, isAccount) {
32
32
  row.installedSkillNames,
33
33
  row.installedModelNames,
34
34
  row.cachedModelNames,
35
- row.toolCatalog
35
+ row.toolCatalog,
36
+ row.customModels
36
37
  ].some((list) => list?.length) ||
37
38
  (row.readySkillNames?.length ?? 0) > (trimmed.readySkillNames?.length ?? 0);
38
39
  if (leaked) {
@@ -48,7 +49,8 @@ async function hostedOnly(store, input) {
48
49
  installedModelNames: [],
49
50
  cachedModelNames: [],
50
51
  readySkillNames: readinessForAgents(input.readySkillNames ?? [], hosted),
51
- toolCatalog: []
52
+ toolCatalog: [],
53
+ customModels: []
52
54
  };
53
55
  }
54
56
  export async function advertiseDeviceVersion(store, deviceId, app) {
@@ -25,11 +25,14 @@ export declare function documentUnreadReason({ knowledgeOn, failedFiles, attache
25
25
  export declare function documentUnreadNotice(reason: DocumentUnreadReason, plural: boolean): string;
26
26
  export interface PdfPagesCutState {
27
27
  readonly fileName: string;
28
- readonly lastPage: number;
28
+ readonly cutAt: number;
29
29
  readonly pageCount: number;
30
+ readonly shown: number;
31
+ readonly wholeText: boolean;
30
32
  }
31
- export declare function pdfPagesCutNotice({ fileName, lastPage, pageCount }: PdfPagesCutState): string;
32
- export declare function pdfPagesSkippedNotice(fileName: string): string;
33
+ export declare function pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText }: PdfPagesCutState): string;
34
+ export declare const PDF_PAGES_SOURCE_LINE = "Page images from the PDFs in this chat are a source too: when an image shows what the excerpts do not, answer from the image.";
35
+ export declare function pdfPagesShownNotice(fileName: string, labels: ReadonlyArray<string>): string;
33
36
  export interface AnnotatedMessage {
34
37
  content: string;
35
38
  documents?: DocumentAttachment[];
@@ -33,11 +33,16 @@ export function documentUnreadNotice(reason, plural) {
33
33
  }[reason];
34
34
  return `[The attached document${plural ? 's' : ''} could not be read for this reply: ${why}.]`;
35
35
  }
36
- export function pdfPagesCutNotice({ fileName, lastPage, pageCount }) {
37
- return `[Only pages 1–${lastPage} of "${fileName}" (${pageCount} pages) were shown to you as images; pages ${lastPage + 1}–${pageCount} were not, and no tool can show them. If asked about a later page you have no excerpt for, tell the user only the first ${lastPage} pages could be read. Do not guess its content.]`;
38
- }
39
- export function pdfPagesSkippedNotice(fileName) {
40
- return `[No page of "${fileName}" was shown to you as an image: the files before it in this message used up the page images, and no tool can show them. If asked about it and you have no excerpt, tell the user it could not be read. Do not guess its content.]`;
36
+ export function pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText }) {
37
+ const missing = shown
38
+ ? `Some pictures in "${fileName}" (${pageCount} pages), the first on page ${cutAt + 1}, were not shown to you as images`
39
+ : `No picture in "${fileName}" (${pageCount} pages) was shown to you as an image`;
40
+ const text = wholeText ? '; its text is all above' : '';
41
+ return `[${missing}: they did not fit in the context window, and no tool can show them${text}. If asked about a picture you cannot see, say it could not be shown. Do not guess its content.]`;
42
+ }
43
+ export const PDF_PAGES_SOURCE_LINE = 'Page images from the PDFs in this chat are a source too: when an image shows what the excerpts do not, answer from the image.';
44
+ export function pdfPagesShownNotice(fileName, labels) {
45
+ return `[The images from "${fileName}" are, in order: ${labels.join('; ')}.]`;
41
46
  }
42
47
  export function annotateDocumentAttachments(messages) {
43
48
  for (const turn of messages.filter((message) => message.documents?.length)) {
@@ -1,6 +1,6 @@
1
1
  import { CoreError } from '../../errors.js';
2
2
  import { linkAttachment } from '../attachment-link.js';
3
- import { getChunk, updateChunk } from '../chunk.js';
3
+ import { getChunk, readChunk, updateChunk } from '../chunk.js';
4
4
  import { uploadFile } from './upload.js';
5
5
  import b4a from 'b4a';
6
6
  export const MAX_ATTACHMENT_BYTES = 100 * 1024 * 1024;
@@ -39,7 +39,7 @@ export function attachmentFiles(mesh, { maxBytes = MAX_ATTACHMENT_BYTES, deviceI
39
39
  }
40
40
  const attachment = chunk.attachment;
41
41
  if (attachment.deletedAt) {
42
- return chunk;
42
+ return readChunk(chunk);
43
43
  }
44
44
  const updated = await updateChunk(mesh, {
45
45
  id: chunkId,
@@ -48,7 +48,7 @@ export function attachmentFiles(mesh, { maxBytes = MAX_ATTACHMENT_BYTES, deviceI
48
48
  if (!updated.attachment?.deletedAt) {
49
49
  throw new CoreError('ATTACHMENT_DELETE_DROPPED');
50
50
  }
51
- return updated;
51
+ return readChunk(updated);
52
52
  }
53
53
  };
54
54
  }
@@ -1,6 +1,6 @@
1
1
  import type { AbortSignal } from 'bare-abort-controller';
2
2
  import EventEmitter from 'bare-events';
3
- import type { AssistantFileStatusList, AssistantFileReadRequest, AssistantFileReadFrame, AssistantFilePreviewResponse, AssistantIndexingStatusList, AssistantKnowledgeBaseList } from '../../../spec/hyperschema/types.d.ts';
3
+ import type { AssistantFileStatus, AssistantFileStatusList, AssistantFileReadRequest, AssistantFileReadFrame, AssistantFilePreviewResponse, AssistantIndexingStatusList, AssistantKnowledgeBaseList } from '../../../spec/hyperschema/types.d.ts';
4
4
  import type { QvacChunk, QvacKnowledge, QvacKnowledgeArtifact } from '../../../spec/mesh/hyperschema/types.d.ts';
5
5
  import { attachmentFiles } from './attachments.ts';
6
6
  import { addKnowledgeArtifact, type FileRecord } from './records/index.ts';
@@ -11,8 +11,7 @@ import { BlobStore } from './store/index.ts';
11
11
  import { type IndexingDevice, type IndexingProgress, type IndexingEmbedder, type RagExtractors, type VectorIndex } from './rag/index.ts';
12
12
  import type { Mesh } from '../mesh.ts';
13
13
  import type { ChatFileKey } from './records/source.ts';
14
- import type { SettleOptions } from './processing/settle.ts';
15
- export { fullTextBudgetChars, fullTextHits, joinedIndex, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, type VectorHit, type RagDocumentRef } from './rag/index.ts';
14
+ export { fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, type VectorHit, type RagDocumentRef } from './rag/index.ts';
16
15
  export { isKnowledgeRoutedAttachment } from './records/index.ts';
17
16
  export { assertRemovableAt } from './records/index.ts';
18
17
  export { assertArtifactOutput } from './records/index.ts';
@@ -115,7 +114,11 @@ export declare class Files extends EventEmitter {
115
114
  load(id: string, state: LocalWork, options?: FileLoadOptions): void;
116
115
  clearLoaded(processorId?: string): void;
117
116
  hydratedArtifact(id: string): string | null;
118
- settle(ids: string[], options?: SettleOptions): Promise<void>;
117
+ ready(chatId: string, ids: string[]): Promise<AssistantFileStatusList>;
118
+ indexingHere(chatId: string): Promise<void>;
119
+ indexesHere(file: AssistantFileStatus): boolean;
120
+ private _until;
121
+ private _changes;
119
122
  reconnect(): void;
120
123
  report: (update: Parameters<IndexingProgress>[0], processorId?: string) => void;
121
124
  progress(chatId: string): import("./processing/state.ts").ChatProgressEntry[];
@@ -131,7 +134,7 @@ export declare class Files extends EventEmitter {
131
134
  private _processorStatus;
132
135
  close(): Promise<void>;
133
136
  }
134
- export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
137
+ export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
135
138
  export type { IndexingDevice, IndexingEmbedder, RagExtractors, RagSource, VectorIndex } from './rag/index.ts';
136
139
  export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.ts';
137
140
  export { CoreBlobServer, PENDING_BLOB_REFERENCE, isPendingBlobReference, parseBlobReferenceCoreKey } from './store/index.ts';
@@ -1,14 +1,16 @@
1
1
  import b4a from 'b4a';
2
2
  import EventEmitter from 'bare-events';
3
+ import { Readable } from 'streamx';
3
4
  import { attachmentFiles } from './attachments.js';
4
5
  import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, reclaimPurgedFiles, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId, knowledgeCapBytes, usedKnowledgeBytes } from './records/index.js';
5
6
  import { knowledgeFiles } from './knowledge.js';
6
7
  import { randomId } from '../id.js';
7
8
  import { findChunksOfType, getChunk, listChunks } from '../chunk.js';
8
- import { settleFiles, isClaimLive, listIndexingByChat, listIndexingByKnowledge, FileState, FileProcessing } from './processing/index.js';
9
+ import { isClaimLive, listIndexingByChat, listIndexingByKnowledge, FileState, FileProcessing } from './processing/index.js';
9
10
  import { DeadlineScheduler } from '../../deadline.js';
10
11
  import { liveDeviceForWriter } from '../contract/activity.js';
11
12
  import { runDefinition } from '../shared-agents.js';
13
+ import { toolsetEnabled } from '@qvac/harness/lib/tools/toolsets.ts';
12
14
  import { BlobStore, parseBlobReference } from './store/index.js';
13
15
  import { AbortedError } from '@qvac/harness/race-abort';
14
16
  import { RagProcessor } from './rag/index.js';
@@ -17,7 +19,7 @@ import { parseFailure } from './rag/failure.js';
17
19
  import { CoreError } from '../../errors.js';
18
20
  import { errorMessage } from '../../guards.js';
19
21
  import { ACTIVE_KNOWLEDGE_BASE, ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS, INDEXING_STATUS_BY_CHAT, INDEXING_STATUS_BY_KNOWLEDGE, KNOWLEDGE, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE } from '../collections.js';
20
- export { fullTextBudgetChars, fullTextHits, joinedIndex, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
22
+ export { fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
21
23
  export { isKnowledgeRoutedAttachment } from './records/index.js';
22
24
  export { assertRemovableAt } from './records/index.js';
23
25
  export { assertArtifactOutput } from './records/index.js';
@@ -395,18 +397,81 @@ export class Files extends EventEmitter {
395
397
  hydratedArtifact(id) {
396
398
  return this.state.get(id)?.loaded?.artifactId ?? null;
397
399
  }
398
- async settle(ids, options = {}) {
399
- if (!this._processing)
400
- return;
401
- await settleFiles(this, {
402
- ...options,
403
- ids,
404
- processing: this._processing,
405
- chatIds: async () => {
406
- const rows = await Promise.all(ids.map((id) => getKnowledge(this._mesh, id)));
407
- return [...new Set(rows.flatMap((row) => (row?.chatId ? [row.chatId] : [])))];
400
+ ready(chatId, ids) {
401
+ const wanted = new Set(ids);
402
+ this._processing?.prioritize(ids);
403
+ return this._until(chatId, (all) => {
404
+ const files = all.filter((file) => wanted.has(file.fileId));
405
+ return files.every(settledFile) ? files : null;
406
+ });
407
+ }
408
+ async indexingHere(chatId) {
409
+ await this._until(chatId, (files) => files.some((file) => this.indexesHere(file)) ? null : files);
410
+ }
411
+ indexesHere(file) {
412
+ const rag = file.processors.find((entry) => entry.processorId === 'rag');
413
+ if (!this._processing?.active || settledFile(file) || rag?.state === 'not-applicable') {
414
+ return false;
415
+ }
416
+ if (this._processing.awaitsBytes(file.fileId)) {
417
+ return false;
418
+ }
419
+ if (rag?.claimedBy) {
420
+ return b4a.equals(rag.claimedBy, this._mesh.localWriterKey);
421
+ }
422
+ return this._processing.pending(file.fileId);
423
+ }
424
+ async _until(chatId, settle) {
425
+ let all = [];
426
+ for await (const _ of this._changes()) {
427
+ all = (await this.status(chatId)).files;
428
+ const files = settle(all);
429
+ if (files || this.closed) {
430
+ return { files: files ?? all };
431
+ }
432
+ }
433
+ return { files: all };
434
+ }
435
+ _changes() {
436
+ let due = true;
437
+ let pending = null;
438
+ const deliver = () => {
439
+ const cb = pending;
440
+ if (!due || !cb) {
441
+ return;
442
+ }
443
+ due = false;
444
+ pending = null;
445
+ changes.push(true);
446
+ cb(null);
447
+ };
448
+ const onUpdate = () => {
449
+ due = true;
450
+ deliver();
451
+ };
452
+ const onClose = () => {
453
+ changes.push(true);
454
+ changes.push(null);
455
+ pending?.(null);
456
+ };
457
+ const changes = new Readable({
458
+ read(cb) {
459
+ pending = cb;
460
+ deliver();
461
+ },
462
+ predestroy() {
463
+ pending?.(null);
464
+ pending = null;
465
+ },
466
+ destroy: (cb) => {
467
+ this.unwatch(onUpdate);
468
+ this.off('close', onClose);
469
+ cb(null);
408
470
  }
409
471
  });
472
+ this.watch(onUpdate);
473
+ this.once('close', onClose);
474
+ return changes;
410
475
  }
411
476
  reconnect() {
412
477
  for (const { work } of this._processors.values()) {
@@ -529,9 +594,11 @@ export class Files extends EventEmitter {
529
594
  return { files: [] };
530
595
  }
531
596
  const agent = await runDefinition(this._mesh, await snap.get(AGENTS, { id: chat.agentId }));
597
+ const knowledgeOn = toolsetEnabled('knowledge', agent?.disabledToolsets);
532
598
  const knowledge = await snap.find(ACTIVE_KNOWLEDGE_BY_CHAT, range).toArray();
533
599
  const chunks = await findChunksOfType(snap, { chatId, type: 'attachment' });
534
600
  const statuses = await snap.find(INDEXING_STATUS_BY_CHAT, range).toArray();
601
+ const statusesByKnowledge = groupBy(statuses, (row) => row.knowledgeId);
535
602
  const sources = new Map(knowledge.map((row) => [row.id, row]));
536
603
  const ragIds = new Set(sources.keys());
537
604
  for (const chunk of chunks) {
@@ -565,12 +632,12 @@ export class Files extends EventEmitter {
565
632
  lte: { knowledgeId: id }
566
633
  })
567
634
  .toArray();
568
- const applicable = ragIds.has(id);
635
+ const applicable = knowledgeOn && ragIds.has(id);
569
636
  const local = this.state.get(id);
570
637
  const output = rows.find((row) => !isProcessorOutput(row.embedderVersion) &&
571
638
  (!this._model || row.embedderVersion.startsWith(`${this._model}|`)));
572
639
  const published = !!output;
573
- const attempts = statuses.filter((row) => row.knowledgeId === id && !isProcessorArtifactId(row.id));
640
+ const attempts = (statusesByKnowledge.get(id) ?? []).filter((row) => !isProcessorArtifactId(row.id));
574
641
  const claim = attempts.find((row) => isClaimLive(row, Date.now()));
575
642
  const failures = attempts.some((row) => row.status === 'ready')
576
643
  ? []
@@ -585,6 +652,7 @@ export class Files extends EventEmitter {
585
652
  processor.outputId = local?.loaded?.artifactId ?? output?.id;
586
653
  if (!applicable) {
587
654
  processor.state = 'not-applicable';
655
+ processor.reason = ragIds.has(id) ? 'knowledge off' : undefined;
588
656
  }
589
657
  else if (local?.loaded) {
590
658
  processor.state = 'ready';
@@ -625,7 +693,12 @@ export class Files extends EventEmitter {
625
693
  : await this._store.available(source.blobId);
626
694
  const processors = [
627
695
  processor,
628
- ...(await this._extraProcessorStatuses(id, { chunks, knowledge, rows, statuses }))
696
+ ...(await this._extraProcessorStatuses(id, {
697
+ chunks,
698
+ knowledge,
699
+ rows,
700
+ statusesByKnowledge
701
+ }))
629
702
  ];
630
703
  const uploadProgress = this._uploads.get(id)?.status.uploadProgress;
631
704
  result.set(id, {
@@ -655,7 +728,7 @@ export class Files extends EventEmitter {
655
728
  processor.agentId = agent.id;
656
729
  }
657
730
  }
658
- async _extraProcessorStatuses(id, { chunks, knowledge, rows, statuses }) {
731
+ async _extraProcessorStatuses(id, { chunks, knowledge, rows, statusesByKnowledge }) {
659
732
  const processors = [];
660
733
  for (const { processor: descriptor } of this._processors.values()) {
661
734
  if (descriptor.id === 'rag')
@@ -667,7 +740,10 @@ export class Files extends EventEmitter {
667
740
  const input = record ? { ...record, kind: descriptor.input } : null;
668
741
  processors.push(!input || !descriptor.applies(input)
669
742
  ? { processorId: descriptor.id, state: 'not-applicable', published: false }
670
- : await this._processorStatus(descriptor, input, { rows, attempts: statuses }));
743
+ : await this._processorStatus(descriptor, input, {
744
+ rows,
745
+ attempts: statusesByKnowledge.get(id) ?? []
746
+ }));
671
747
  }
672
748
  return processors;
673
749
  }
@@ -719,6 +795,9 @@ export class Files extends EventEmitter {
719
795
  await this._store.close();
720
796
  }
721
797
  }
798
+ function settledFile(file) {
799
+ return file.readyForChat || file.state === 'failed';
800
+ }
722
801
  function yieldsToVerdict(work) {
723
802
  return work.state === 'pending' || (work.state === 'failed' && work.retryAt === undefined);
724
803
  }
@@ -731,6 +810,18 @@ function fileState(processor, opts) {
731
810
  }
732
811
  return { state: 'pending', reason: 'source unavailable on this device' };
733
812
  }
734
- export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.js';
813
+ function groupBy(items, key) {
814
+ const groups = new Map();
815
+ for (const item of items) {
816
+ const id = key(item);
817
+ let group = groups.get(id);
818
+ if (!group) {
819
+ groups.set(id, (group = []));
820
+ }
821
+ group.push(item);
822
+ }
823
+ return groups;
824
+ }
825
+ export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.js';
735
826
  export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.js';
736
827
  export { CoreBlobServer, PENDING_BLOB_REFERENCE, isPendingBlobReference, parseBlobReferenceCoreKey } from './store/index.js';
@@ -67,6 +67,7 @@ export declare class FileProcessing {
67
67
  private _enqueue;
68
68
  get active(): boolean;
69
69
  pending(id: string): boolean;
70
+ awaitsBytes(id: string): boolean;
70
71
  prioritize(ids: string[]): void;
71
72
  private _shutdown;
72
73
  private _stop;
@@ -85,4 +86,3 @@ export type { FileDocument } from './document.ts';
85
86
  export { isClaimLive, listIndexingByChat, listIndexingByKnowledge } from './protocol.ts';
86
87
  export { FileState } from './state.ts';
87
88
  export type { IndexingProgressEntry, LocalWork } from './state.ts';
88
- export { settleFiles } from './settle.ts';
@@ -130,12 +130,19 @@ export class FileProcessing {
130
130
  this._source.pending((key) => key === id) ||
131
131
  this._output.pending((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id)));
132
132
  }
133
+ awaitsBytes(id) {
134
+ return (this._source.awaitsBytes((key) => key === id) ||
135
+ this._output.awaitsBytes((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id));
136
+ }
133
137
  prioritize(ids) {
134
138
  for (const lane of this._lanes) {
135
139
  const queued = new Map(lane.queue);
136
140
  lane.queue.clear();
137
141
  for (const id of ids) {
138
- lane.queue.set(id, queued.get(id) ?? []);
142
+ const wakes = queued.get(id);
143
+ if (wakes) {
144
+ lane.queue.set(id, wakes);
145
+ }
139
146
  }
140
147
  for (const [id, wakes] of queued) {
141
148
  if (!lane.queue.has(id)) {
@@ -349,4 +356,3 @@ export class FileProcessing {
349
356
  }
350
357
  export { isClaimLive, listIndexingByChat, listIndexingByKnowledge } from './protocol.js';
351
358
  export { FileState } from './state.js';
352
- export { settleFiles } from './settle.js';
@@ -35,5 +35,6 @@ export declare class IndexingSource {
35
35
  read(id: string, blobRef: string, signal?: AbortSignal): Promise<SourceRead>;
36
36
  cancel(matches?: (id: string) => boolean): Promise<void>;
37
37
  pending(matches: (id: string) => boolean): boolean;
38
+ awaitsBytes(matches: (id: string) => boolean): boolean;
38
39
  private _fetch;
39
40
  }
@@ -70,6 +70,9 @@ export class IndexingSource {
70
70
  }
71
71
  return false;
72
72
  }
73
+ awaitsBytes(matches) {
74
+ return this.pending(matches) || this.retry.missing(matches);
75
+ }
73
76
  _fetch(id, reference) {
74
77
  const download = this._download;
75
78
  if (!download) {
@@ -45,6 +45,7 @@ export declare class FileRetries {
45
45
  blockedUntil(id: string, now: number): number | null;
46
46
  miss(id: string, now: number, retryMs: number): number;
47
47
  markMissing(id: string): void;
48
+ missing(matches: (id: string) => boolean): boolean;
48
49
  markFound(id: string): void;
49
50
  clear(id?: string): void;
50
51
  reconnect(onRetry?: (id: string, deadline: number) => void): number | null;
@@ -15,6 +15,13 @@ export class FileRetries {
15
15
  markMissing(id) {
16
16
  this._missing.add(id);
17
17
  }
18
+ missing(matches) {
19
+ for (const id of this._missing) {
20
+ if (matches(id))
21
+ return true;
22
+ }
23
+ return false;
24
+ }
18
25
  markFound(id) {
19
26
  this._missing.delete(id);
20
27
  }
@@ -20,6 +20,7 @@ export interface ArtifactBuildInput {
20
20
  bytes: Buffer;
21
21
  signal: AbortSignal;
22
22
  onProgress?: BuildProgress;
23
+ onExtracting?: (pages: number, totalPages: number) => Promise<void>;
23
24
  onExtracted?: () => void | Promise<void>;
24
25
  extractors?: RagExtractors;
25
26
  }