@qvac/core 1.0.0-rc.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. package/CHANGELOG.md +241 -3
  2. package/README.md +1 -0
  3. package/dist-lib/lib/api/attachments.d.ts +1 -1
  4. package/dist-lib/lib/api/attachments.js +11 -0
  5. package/dist-lib/lib/api/chunks.d.ts +4 -2
  6. package/dist-lib/lib/api/chunks.js +6 -3
  7. package/dist-lib/lib/api/device-routes.d.ts +6 -1
  8. package/dist-lib/lib/api/device-routes.js +20 -1
  9. package/dist-lib/lib/api/devices.js +6 -18
  10. package/dist-lib/lib/api/execution.d.ts +6 -2
  11. package/dist-lib/lib/api/execution.js +29 -3
  12. package/dist-lib/lib/api/host-run-log.d.ts +7 -1
  13. package/dist-lib/lib/api/host-run-log.js +3 -1
  14. package/dist-lib/lib/api/index.d.ts +3 -1
  15. package/dist-lib/lib/api/index.js +17 -6
  16. package/dist-lib/lib/api/members.js +13 -29
  17. package/dist-lib/lib/api/mesh.d.ts +2 -1
  18. package/dist-lib/lib/api/projects.d.ts +1 -1
  19. package/dist-lib/lib/api/projects.js +9 -5
  20. package/dist-lib/lib/api/rag-label.d.ts +4 -0
  21. package/dist-lib/lib/api/rag-label.js +21 -0
  22. package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
  23. package/dist-lib/lib/api/rag-lexical.js +1058 -0
  24. package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
  25. package/dist-lib/lib/api/rag-manifest.js +46 -0
  26. package/dist-lib/lib/api/rag-outline.d.ts +36 -0
  27. package/dist-lib/lib/api/rag-outline.js +426 -0
  28. package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
  29. package/dist-lib/lib/api/rag-plugin.js +70 -0
  30. package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
  31. package/dist-lib/lib/api/rag-synthesis.js +102 -0
  32. package/dist-lib/lib/api/rag-targets.d.ts +23 -0
  33. package/dist-lib/lib/api/rag-targets.js +215 -0
  34. package/dist-lib/lib/api/rag-window.d.ts +5 -0
  35. package/dist-lib/lib/api/rag-window.js +46 -0
  36. package/dist-lib/lib/api/rag.d.ts +3 -2
  37. package/dist-lib/lib/api/rag.js +421 -184
  38. package/dist-lib/lib/api/served.js +8 -14
  39. package/dist-lib/lib/api/workspace.d.ts +8 -1
  40. package/dist-lib/lib/api/workspace.js +27 -13
  41. package/dist-lib/lib/assistant/harness.d.ts +9 -1
  42. package/dist-lib/lib/assistant/history.d.ts +11 -2
  43. package/dist-lib/lib/assistant/history.js +74 -4
  44. package/dist-lib/lib/assistant/media-round.d.ts +61 -0
  45. package/dist-lib/lib/assistant/media-round.js +111 -0
  46. package/dist-lib/lib/assistant/message.d.ts +7 -0
  47. package/dist-lib/lib/assistant/model-round.d.ts +19 -1
  48. package/dist-lib/lib/assistant/model-round.js +163 -28
  49. package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
  50. package/dist-lib/lib/assistant/pdf-pages.js +67 -3
  51. package/dist-lib/lib/assistant/round-host.d.ts +22 -0
  52. package/dist-lib/lib/assistant/round-host.js +142 -8
  53. package/dist-lib/lib/assistant/upload.d.ts +12 -0
  54. package/dist-lib/lib/assistant/upload.js +36 -0
  55. package/dist-lib/lib/assistant-client.d.ts +1 -0
  56. package/dist-lib/lib/assistant-client.js +40 -19
  57. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  58. package/dist-lib/lib/assistant-tree.js +48 -1
  59. package/dist-lib/lib/assistant.d.ts +18 -1
  60. package/dist-lib/lib/assistant.js +484 -95
  61. package/dist-lib/lib/client.d.ts +1 -0
  62. package/dist-lib/lib/client.js +4 -1
  63. package/dist-lib/lib/compaction.js +2 -2
  64. package/dist-lib/lib/core/agent.js +12 -45
  65. package/dist-lib/lib/core/approval-rules.js +1 -8
  66. package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
  67. package/dist-lib/lib/core/archived-mesh.js +12 -8
  68. package/dist-lib/lib/core/chat-copy.d.ts +18 -3
  69. package/dist-lib/lib/core/chat-copy.js +74 -25
  70. package/dist-lib/lib/core/chat-fork.js +15 -6
  71. package/dist-lib/lib/core/chat-group.js +16 -24
  72. package/dist-lib/lib/core/chat-move.d.ts +9 -0
  73. package/dist-lib/lib/core/chat-move.js +94 -3
  74. package/dist-lib/lib/core/chat-order.js +12 -18
  75. package/dist-lib/lib/core/chat.js +38 -31
  76. package/dist-lib/lib/core/chunk.d.ts +2 -2
  77. package/dist-lib/lib/core/chunk.js +18 -25
  78. package/dist-lib/lib/core/collections.d.ts +2 -0
  79. package/dist-lib/lib/core/collections.js +2 -0
  80. package/dist-lib/lib/core/contract/activity.d.ts +4 -0
  81. package/dist-lib/lib/core/contract/activity.js +34 -4
  82. package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
  83. package/dist-lib/lib/core/contract/chat-group.js +21 -1
  84. package/dist-lib/lib/core/contract/chat.d.ts +9 -0
  85. package/dist-lib/lib/core/contract/chat.js +127 -5
  86. package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
  87. package/dist-lib/lib/core/contract/chunks.js +37 -6
  88. package/dist-lib/lib/core/contract/features.d.ts +21 -0
  89. package/dist-lib/lib/core/contract/features.js +22 -0
  90. package/dist-lib/lib/core/contract/index.js +2 -0
  91. package/dist-lib/lib/core/contract/indexing.js +4 -2
  92. package/dist-lib/lib/core/contract/knowledge.js +5 -4
  93. package/dist-lib/lib/core/contract/members.js +16 -10
  94. package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
  95. package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
  96. package/dist-lib/lib/core/contract/projects.d.ts +5 -0
  97. package/dist-lib/lib/core/contract/projects.js +13 -5
  98. package/dist-lib/lib/core/contract/roles.js +32 -4
  99. package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
  100. package/dist-lib/lib/core/contract/run-requests.js +35 -2
  101. package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
  102. package/dist-lib/lib/core/contract/tool-configs.js +3 -2
  103. package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
  104. package/dist-lib/lib/core/contract/workspace.js +40 -7
  105. package/dist-lib/lib/core/contract/writers.js +2 -2
  106. package/dist-lib/lib/core/departures.d.ts +3 -1
  107. package/dist-lib/lib/core/departures.js +3 -0
  108. package/dist-lib/lib/core/devices.js +1 -8
  109. package/dist-lib/lib/core/files/attachments.d.ts +2 -0
  110. package/dist-lib/lib/core/files/attachments.js +19 -1
  111. package/dist-lib/lib/core/files/index.d.ts +20 -4
  112. package/dist-lib/lib/core/files/index.js +132 -21
  113. package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
  114. package/dist-lib/lib/core/files/processing/document.js +7 -0
  115. package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
  116. package/dist-lib/lib/core/files/processing/index.js +144 -12
  117. package/dist-lib/lib/core/files/processing/protocol.js +13 -34
  118. package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
  119. package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
  120. package/dist-lib/lib/core/files/rag/build.js +123 -19
  121. package/dist-lib/lib/core/files/rag/failure.js +2 -1
  122. package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
  123. package/dist-lib/lib/core/files/rag/index.js +13 -1
  124. package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
  125. package/dist-lib/lib/core/files/rag/load.js +27 -6
  126. package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
  127. package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
  128. package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
  129. package/dist-lib/lib/core/files/rag/produce.js +104 -16
  130. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
  131. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
  132. package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
  133. package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
  134. package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
  135. package/dist-lib/lib/core/files/records/knowledge.js +7 -40
  136. package/dist-lib/lib/core/files/records/reclaim.js +5 -10
  137. package/dist-lib/lib/core/files/upload.js +68 -41
  138. package/dist-lib/lib/core/fs-drives.js +14 -26
  139. package/dist-lib/lib/core/invite.d.ts +1 -0
  140. package/dist-lib/lib/core/invite.js +4 -2
  141. package/dist-lib/lib/core/local/files.d.ts +1 -1
  142. package/dist-lib/lib/core/local/files.js +5 -5
  143. package/dist-lib/lib/core/local/index.d.ts +13 -6
  144. package/dist-lib/lib/core/local/index.js +103 -69
  145. package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
  146. package/dist-lib/lib/core/local/run-logs.js +5 -5
  147. package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
  148. package/dist-lib/lib/core/mesh-processor.js +87 -0
  149. package/dist-lib/lib/core/mesh.d.ts +48 -5
  150. package/dist-lib/lib/core/mesh.js +318 -129
  151. package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
  152. package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
  153. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
  154. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
  155. package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
  156. package/dist-lib/lib/core/migrations/writer-links.js +32 -0
  157. package/dist-lib/lib/core/project-keys.d.ts +6 -0
  158. package/dist-lib/lib/core/project-keys.js +11 -0
  159. package/dist-lib/lib/core/public-mesh.d.ts +3 -0
  160. package/dist-lib/lib/core/public-mesh.js +10 -8
  161. package/dist-lib/lib/core/run-requests.d.ts +3 -0
  162. package/dist-lib/lib/core/run-requests.js +21 -1
  163. package/dist-lib/lib/core/skill-availability.d.ts +33 -0
  164. package/dist-lib/lib/core/skill-availability.js +67 -0
  165. package/dist-lib/lib/core/sync/background.d.ts +0 -1
  166. package/dist-lib/lib/core/sync/background.js +2 -1
  167. package/dist-lib/lib/core/sync/changed.d.ts +4 -2
  168. package/dist-lib/lib/core/sync/changed.js +1 -1
  169. package/dist-lib/lib/core/sync/index.js +52 -10
  170. package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
  171. package/dist-lib/lib/core/sync/view-updates.js +82 -91
  172. package/dist-lib/lib/core/sync-status.js +97 -41
  173. package/dist-lib/lib/core/tool-activity.js +6 -12
  174. package/dist-lib/lib/core/tool-configs.js +17 -29
  175. package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
  176. package/dist-lib/lib/core/upgrade-summary.js +67 -0
  177. package/dist-lib/lib/core/view-read.d.ts +4 -0
  178. package/dist-lib/lib/core/view-read.js +23 -0
  179. package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
  180. package/dist-lib/lib/core/workspace-upgrade.js +27 -16
  181. package/dist-lib/lib/core.d.ts +45 -42
  182. package/dist-lib/lib/core.js +675 -640
  183. package/dist-lib/lib/errors.d.ts +24 -0
  184. package/dist-lib/lib/errors.js +24 -0
  185. package/dist-lib/lib/llm-model-names.d.ts +1 -1
  186. package/dist-lib/lib/llm-model-names.js +1 -1
  187. package/dist-lib/lib/mesh-scoped-engine.js +4 -0
  188. package/dist-lib/lib/model-context.js +1 -2
  189. package/dist-lib/lib/model-download.d.ts +6 -1
  190. package/dist-lib/lib/model-download.js +16 -3
  191. package/dist-lib/lib/oauth/resolve.js +29 -8
  192. package/dist-lib/lib/oauth/token-exchange-core.js +11 -0
  193. package/dist-lib/lib/probe-rounds.d.ts +7 -2
  194. package/dist-lib/lib/probe-rounds.js +6 -1
  195. package/dist-lib/lib/run-transcript.d.ts +4 -1
  196. package/dist-lib/lib/run-transcript.js +15 -6
  197. package/dist-lib/schema/engine.js +83 -6
  198. package/dist-lib/schema/indexes.d.ts +3 -0
  199. package/dist-lib/schema/indexes.js +5 -0
  200. package/dist-lib/schema/local.js +11 -2
  201. package/dist-lib/schema/qvac.js +49 -5
  202. package/dist-lib/spec/bind.js +34 -0
  203. package/dist-lib/spec/calls.js +5 -0
  204. package/dist-lib/spec/capabilities.d.ts +17 -0
  205. package/dist-lib/spec/hrpc/hrpc.json +65 -0
  206. package/dist-lib/spec/hrpc/index.js +68 -5
  207. package/dist-lib/spec/hrpc/messages.js +260 -23
  208. package/dist-lib/spec/hyperschema/index.js +260 -23
  209. package/dist-lib/spec/hyperschema/schema.json +181 -0
  210. package/dist-lib/spec/hyperschema/types.d.ts +48 -0
  211. package/dist-lib/spec/local/hyperdb/messages.js +270 -224
  212. package/dist-lib/spec/local/hyperschema/index.js +73 -38
  213. package/dist-lib/spec/local/hyperschema/schema.json +31 -0
  214. package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
  215. package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
  216. package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
  217. package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
  218. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
  219. package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
  220. package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
  221. package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
  222. package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
  223. package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
  224. package/package.json +82 -71
  225. package/tui/README.md +52 -0
  226. package/tui/app.mjs +69 -12
  227. package/tui/boot.mjs +27 -6
  228. package/tui/cli.mjs +93 -0
  229. package/tui/index.mjs +2 -87
  230. package/tui/lib/attachment-files.mjs +82 -0
  231. package/tui/lib/reopen-meshes.mjs +15 -0
  232. package/tui/lib/updater.mjs +86 -0
  233. package/tui/main.mjs +29 -57
  234. package/worker/assistant-sidecar.mjs +10 -3
  235. package/worker/stow-entry.mjs +3 -52
  236. package/dist-lib/lib/core/chat-project.d.ts +0 -2
  237. package/dist-lib/lib/core/chat-project.js +0 -15
@@ -2,10 +2,16 @@ import { toolsetEnabled } from '@qvac/harness/lib/tools/toolsets.ts';
2
2
  import { getAgent } from '../core/agent.js';
3
3
  import { runDefinition } from '../core/shared-agents.js';
4
4
  import { getChat } from '../core/chat.js';
5
- import { chatProject } from '../core/chat-project.js';
6
- import { isKnowledgeRoutedAttachment, fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from '../core/files/index.js';
7
- import { heldMesh, requireMesh } from './current-mesh.js';
5
+ import { isKnowledgeRoutedAttachment, fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, referencedDocuments, PluginVectorIndex } from '../core/files/index.js';
6
+ import { requireMesh } from './current-mesh.js';
7
+ import { searchPlugin } from './rag-plugin.js';
8
+ import { rankLexical } from './rag-lexical.js';
9
+ import { manifestBlock, readable, UNREAD } from './rag-manifest.js';
10
+ import { classifyRequest, outlineBlock } from './rag-synthesis.js';
11
+ import { DEFAULT_CHUNK_BUDGET_TOKENS } from 'rag-extract';
8
12
  import { errorMessage } from '../guards.js';
13
+ import { parseFailure } from '../core/files/rag/failure.js';
14
+ import { listIndexingByKnowledge } from '../core/files/processing/index.js';
9
15
  import b4a from 'b4a';
10
16
  export const MAX_RAG_SEARCH_LIMIT = 20;
11
17
  const DEFAULT_RAG_SEARCH_LIMIT = 5;
@@ -14,218 +20,341 @@ function clampLimit(limit) {
14
20
  return DEFAULT_RAG_SEARCH_LIMIT;
15
21
  return Math.min(Math.max(1, Math.floor(limit)), MAX_RAG_SEARCH_LIMIT);
16
22
  }
17
- export function createRagApi(pool, { index, embedder, effectiveCtxSize = null }) {
18
- return {
19
- ragSearch: async ({ chatId, query, mode, limit, attachmentIds, meshId, immediate }) => {
20
- let pendingFiles = [];
21
- let projectFilesNotReady = 0;
22
- let chatFilesIndexingHere = 0;
23
- const respond = (overrides) => {
24
- const result = response({
25
- pendingFiles,
26
- projectFilesNotReady,
27
- chatFilesIndexingHere,
28
- ...overrides
29
- });
30
- return { ...result, chatFilesNotReady: result.pendingFiles?.length ?? 0 };
31
- };
32
- const mesh = await requireMesh(pool, meshId);
33
- const chat = await getChat(mesh, chatId);
34
- const toolMode = mode === 'tool';
35
- if (!chat) {
36
- return respond({ enabled: false });
23
+ export function createRagApi(pool, { index, embedder, effectiveCtxSize = null, extractors }) {
24
+ const ragSearch = async ({ chatId, query, mode, limit, attachmentIds, meshId, immediate, previousQuery, excerptsOnly }, { vectors = false } = {}) => {
25
+ let pendingFiles = [];
26
+ let projectFilesNotReady = 0;
27
+ let chatFilesIndexingHere = 0;
28
+ let manifest;
29
+ const respond = (overrides) => {
30
+ const result = response({
31
+ pendingFiles,
32
+ projectFilesNotReady,
33
+ chatFilesIndexingHere,
34
+ ...overrides,
35
+ outline: manifest ?? overrides.outline
36
+ });
37
+ return { ...result, chatFilesNotReady: result.pendingFiles?.length ?? 0 };
38
+ };
39
+ const mesh = await requireMesh(pool, meshId);
40
+ const chat = await getChat(mesh, chatId);
41
+ const toolMode = mode === 'tool';
42
+ if (!chat) {
43
+ return respond({ enabled: false });
44
+ }
45
+ const request = classifyRequest(query);
46
+ const agent = await runDefinition(mesh, await getAgent(mesh, chat.agentId));
47
+ const scopeIds = new Set(toolMode ? [] : (attachmentIds ?? []));
48
+ const off = new Set(chat.knowledgeSourcesOff ?? []);
49
+ const kept = (id) => !off.has('chat') || scopeIds.has(id);
50
+ const chatKnowledge = async () => (await mesh.files.listKnowledge(chatId)).filter((row) => kept(row.id));
51
+ const chatFiles = async () => (await mesh.files.status(chatId)).files.filter((file) => kept(file.fileId));
52
+ const indexingHere = (files) => files.filter((file) => scopeIds.has(file.fileId) &&
53
+ (vectors ? mesh.files.indexesHere(file) : mesh.files.extractingHere(file))).length;
54
+ if (!immediate && scopeIds.size > 0) {
55
+ const ids = [...scopeIds];
56
+ await (vectors ? mesh.files.indexingHere(chatId, ids) : mesh.files.textReadyHere(chatId, ids));
57
+ }
58
+ else if (immediate && vectors && scopeIds.size > 0) {
59
+ await mesh.files.requestVectors([...scopeIds]);
60
+ }
61
+ const chatRows = await chatKnowledge();
62
+ if (toolMode && chatRows.length > 0) {
63
+ const ids = chatRows.map((row) => row.id);
64
+ if (immediate) {
65
+ await mesh.files.requestVectors(ids);
37
66
  }
38
- const agent = await runDefinition(mesh, await getAgent(mesh, chat.agentId));
39
- const scopeIds = new Set(toolMode ? [] : (attachmentIds ?? []));
40
- const off = new Set(chat.knowledgeSourcesOff ?? []);
41
- const kept = (id) => !off.has('chat') || scopeIds.has(id);
42
- const chatKnowledge = async () => (await mesh.files.listKnowledge(chatId)).filter((row) => kept(row.id));
43
- const chatFiles = async () => (await mesh.files.status(chatId)).files.filter((file) => kept(file.fileId));
44
- const indexingHere = (files) => files.filter((file) => scopeIds.has(file.fileId) && mesh.files.indexesHere(file)).length;
45
- if (!immediate && scopeIds.size > 0) {
46
- await mesh.files.indexingHere(chatId, [...scopeIds]);
67
+ else {
68
+ await mesh.files.indexingHere(chatId, ids);
47
69
  }
48
- const chatRows = await chatKnowledge();
49
- const initialFiles = await chatFiles();
50
- pendingFiles = pendingFileNames(initialFiles);
51
- chatFilesIndexingHere = indexingHere(initialFiles);
52
- const scopeRows = chatRows.filter((row) => scopeIds.has(row.id));
53
- const project = off.has('project')
70
+ }
71
+ const initialFiles = await chatFiles();
72
+ pendingFiles = pendingFileNames(initialFiles);
73
+ chatFilesIndexingHere = indexingHere(initialFiles);
74
+ const scopeRows = chatRows.filter((row) => scopeIds.has(row.id));
75
+ const project = off.has('project') || mesh.role !== 'project' ? null : mesh;
76
+ const kbEntries = project ? ((await project.files.knowledgeBase()).entries ?? []) : [];
77
+ const kbRows = kbEntries.filter((entry) => entry.loaded).map((entry) => entry.knowledge);
78
+ projectFilesNotReady = kbEntries.filter((entry) => !entry.loaded && entry.status !== 'failed').length;
79
+ let window;
80
+ const ctxSize = () => (window ??= effectiveCtxSize ? effectiveCtxSize(agent) : Promise.resolve(0));
81
+ const attached = scopeIds.size > 0 && scopeRows.length === scopeIds.size ? scopeRows : [];
82
+ const named = new Set(referencedDocuments(query, chatRows.map((row) => ({ documentId: row.id, title: row.fileName }))).map((document) => document.documentId));
83
+ const focused = named.size > 0 ? chatRows.filter((row) => named.has(row.id)) : chatRows;
84
+ let subject = [];
85
+ if (scopeIds.size > 0) {
86
+ subject = attached;
87
+ }
88
+ else if (kbRows.length === 0) {
89
+ subject = focused;
90
+ }
91
+ if (!toolMode && request.wholeDocument && subject.length > 1) {
92
+ manifest = await listFiles(mesh, { files: initialFiles, rows: subject });
93
+ }
94
+ const packet = async (rows, pick) => {
95
+ const unread = await unreadFiles(mesh, {
96
+ files: initialFiles,
97
+ rows,
98
+ ids: pick.pendingIds
99
+ });
100
+ const read = rows.filter((row) => !pick.pendingIds.includes(row.id));
101
+ const gaps = unread.states.map(({ name, status }) => `${name}: ${status}`);
102
+ return respond({
103
+ pendingFiles: [
104
+ ...new Set([
105
+ ...pendingFileNames(initialFiles, new Set(read.map((row) => row.id))),
106
+ ...unread.pending
107
+ ])
108
+ ],
109
+ searchedCount: pick.hits.length,
110
+ indexEmpty: false,
111
+ hasKnowledge: true,
112
+ availableFiles: [...new Set(read.map((row) => row.fileName))],
113
+ uncoveredItems: listed([...pick.uncovered, ...gaps]),
114
+ failedFiles: unread.failed,
115
+ outline: outlineBlock(pick),
116
+ hits: pick.hits.map(toWireHit)
117
+ });
118
+ };
119
+ const fromText = async (rows, { previousQuery: before, maxChars, whole = false }) => {
120
+ const allowed = agentAllowsRagSearch(agent);
121
+ const picks = !vectors && allowed;
122
+ const size = await ctxSize();
123
+ const budgetTokens = retrievalBudgetTokens(size) ?? DEFAULT_CHUNK_BUDGET_TOKENS;
124
+ const maxBudgetTokens = size > 0 ? Math.min(MAX_LIST_CONTEXT_TOKENS, Math.floor(size / 4)) : budgetTokens;
125
+ const wholeChars = Math.min(fullTextBudgetChars(size), WHOLE_CHARS);
126
+ const readWhole = (head) => !whole || excerptsOnly
54
127
  ? null
55
- : await chatProject(mesh, chatId, (id) => heldMesh(pool, id));
56
- const kbEntries = project ? ((await project.files.knowledgeBase()).entries ?? []) : [];
57
- const kbRows = kbEntries.filter((entry) => entry.loaded).map((entry) => entry.knowledge);
58
- projectFilesNotReady = kbEntries.filter((entry) => !entry.loaded && entry.status !== 'failed').length;
59
- if (scopeIds.size > 0 && scopeRows.length === scopeIds.size) {
60
- const ctxSize = effectiveCtxSize ? await effectiveCtxSize(agent) : 0;
61
- const fullText = await fullTextHits(mesh, scopeRows, {
62
- budgetChars: fullTextBudgetChars(ctxSize),
63
- head: !agentAllowsRagSearch(agent)
128
+ : fullTextHits(mesh, rows, { budgetChars: wholeChars, head, extractors: extractors?.() });
129
+ const wholeFirst = picks && request.synthesis;
130
+ const early = wholeFirst ? await readWhole(false) : null;
131
+ const texts = picks && !early ? await textsOf(mesh, rows, maxChars) : null;
132
+ const pick = texts &&
133
+ pickFromText(rows, texts, {
134
+ query,
135
+ budgetTokens,
136
+ maxBudgetTokens,
137
+ synthesis: request.synthesis,
138
+ previousQuery: before
64
139
  });
65
- if (fullText) {
66
- const read = new Set(scopeRows.map((row) => row.id));
67
- return respond({
68
- pendingFiles: pendingFileNames(initialFiles, read),
69
- chatFilesIndexingHere: 0,
70
- searchedCount: fullText.length,
71
- indexEmpty: false,
72
- hasKnowledge: true,
73
- availableFiles: [...new Set(scopeRows.map((row) => row.fileName))],
74
- hits: fullText
75
- });
76
- }
77
- }
78
- if (!toolMode && !agentAllowsRagSearch(agent)) {
79
- return respond({ enabled: false });
80
- }
81
- if (chatRows.length === 0 && kbRows.length === 0) {
82
- return respond({});
140
+ if (pick && pick.uncovered.length === 0) {
141
+ return packet(rows, pick);
83
142
  }
84
- const rows = [...chatRows, ...kbRows];
85
- const hasKnowledge = rows.length > 0;
86
- const chatIndex = index(meshId);
87
- const embed = embedder(meshId);
88
- if (!chatIndex || !embed) {
89
- return respond({ hasKnowledge, errorMessage: 'knowledge index unavailable on this device' });
143
+ const fullText = wholeFirst ? early : await readWhole(!allowed);
144
+ if (fullText) {
145
+ manifest = undefined;
146
+ return respond({
147
+ pendingFiles: pendingFileNames(initialFiles, new Set(rows.map((row) => row.id))),
148
+ chatFilesIndexingHere: 0,
149
+ searchedCount: fullText.length,
150
+ indexEmpty: false,
151
+ hasKnowledge: true,
152
+ availableFiles: [...new Set(rows.map((row) => row.fileName))],
153
+ hits: fullText
154
+ });
90
155
  }
91
- const projectIndex = project && kbRows.length ? index(b4a.toHex(project.key)) : null;
92
- const vectorIndex = projectIndex
93
- ? joinedIndex([chatIndex, projectIndex], {
94
- chatId,
95
- knowledgeBaseIds: new Set(kbRows.map((row) => row.id))
96
- })
97
- : chatIndex;
98
- const artifactByRow = new Map([
99
- ...chatRows.map((row) => [row, mesh.files.hydratedArtifact(row.id)]),
100
- ...kbRows.map((row) => [row, project?.files.hydratedArtifact(row.id) ?? null])
101
- ]);
102
- const hydratedDocuments = [];
103
- for (const [row, artifactId] of artifactByRow) {
104
- if (artifactId) {
105
- hydratedDocuments.push({ documentId: artifactId, title: row.fileName });
106
- }
156
+ return pick ? packet(rows, pick) : null;
157
+ };
158
+ if (attached.length > 0) {
159
+ const answer = await fromText(attached, { whole: true });
160
+ if (answer) {
161
+ return answer;
107
162
  }
108
- const indexEmpty = hydratedDocuments.length === 0;
109
- const availableFiles = [...new Set(hydratedDocuments.map((document) => document.title))];
110
- const failedRows = new Set();
111
- const fileStates = await chatFiles();
112
- const statusById = new Map(fileStates.map((file) => [file.fileId, file]));
113
- for (const row of rows) {
114
- if (artifactByRow.get(row))
115
- continue;
116
- if (statusById.get(row.id)?.state === 'failed') {
117
- failedRows.add(row);
118
- }
163
+ }
164
+ if (!toolMode && !agentAllowsRagSearch(agent)) {
165
+ return respond({ enabled: false });
166
+ }
167
+ if (chatRows.length === 0 && kbRows.length === 0) {
168
+ return respond({});
169
+ }
170
+ const followUp = !toolMode && !vectors && scopeIds.size === 0 && kbRows.length === 0;
171
+ if (followUp) {
172
+ const answer = await fromText(focused, {
173
+ maxChars: TEXT_CHARS,
174
+ previousQuery: named.size === 0 ? (previousQuery ?? undefined) : undefined
175
+ });
176
+ if (answer) {
177
+ return answer;
119
178
  }
120
- pendingFiles = pendingFileNames(fileStates, new Set(rows.filter((row) => artifactByRow.get(row)).map((row) => row.id)));
121
- const failedFiles = scopeRows.filter((row) => failedRows.has(row)).map((row) => row.fileName);
122
- const scopeDocumentIds = [];
123
- let scopePending = false;
124
- for (const row of scopeRows) {
125
- const artifactId = artifactByRow.get(row);
126
- if (artifactId) {
127
- scopeDocumentIds.push(artifactId);
128
- }
129
- else if (!failedRows.has(row)) {
130
- scopePending = true;
131
- }
179
+ }
180
+ if (!toolMode && scopeIds.size === 0 && chatRows.length > 0) {
181
+ await mesh.files.requestVectors(chatRows.map((row) => row.id));
182
+ }
183
+ const rows = [...chatRows, ...kbRows];
184
+ const hasKnowledge = rows.length > 0;
185
+ const chatIndex = index(meshId);
186
+ const embed = embedder(meshId);
187
+ if (!chatIndex || !embed) {
188
+ return respond({ hasKnowledge, errorMessage: 'knowledge index unavailable on this device' });
189
+ }
190
+ const projectIndex = project && kbRows.length ? index(b4a.toHex(project.key)) : null;
191
+ const vectorIndex = projectIndex
192
+ ? joinedIndex([chatIndex, projectIndex], {
193
+ chatId,
194
+ knowledgeBaseIds: new Set(kbRows.map((row) => row.id))
195
+ })
196
+ : chatIndex;
197
+ const artifactByRow = new Map([
198
+ ...chatRows.map((row) => [row, mesh.files.hydratedArtifact(row.id)]),
199
+ ...kbRows.map((row) => [row, project?.files.hydratedArtifact(row.id) ?? null])
200
+ ]);
201
+ const hydratedDocuments = [];
202
+ for (const [row, artifactId] of artifactByRow) {
203
+ if (artifactId) {
204
+ hydratedDocuments.push({ documentId: artifactId, title: row.fileName });
132
205
  }
133
- if (scopePending && scopeDocumentIds.length === 0) {
134
- return respond({
135
- pendingReferenced: true,
136
- indexEmpty,
137
- hasKnowledge,
138
- availableFiles,
139
- failedFiles
140
- });
206
+ }
207
+ const indexEmpty = hydratedDocuments.length === 0;
208
+ const availableFiles = [...new Set(hydratedDocuments.map((document) => document.title))];
209
+ const failedRows = new Set();
210
+ const fileStates = await chatFiles();
211
+ const statusById = new Map(fileStates.map((file) => [file.fileId, file]));
212
+ for (const row of rows) {
213
+ if (artifactByRow.get(row))
214
+ continue;
215
+ const rag = statusById
216
+ .get(row.id)
217
+ ?.processors.find((processor) => processor.processorId === 'rag');
218
+ if (rag?.state === 'failed') {
219
+ failedRows.add(row);
141
220
  }
142
- if (scopeIds.size > 0 && scopeDocumentIds.length === 0) {
143
- return respond({
144
- indexEmpty,
145
- hasKnowledge,
146
- availableFiles,
147
- failedFiles
148
- });
221
+ }
222
+ pendingFiles = pendingFileNames(fileStates, new Set(rows.filter((row) => artifactByRow.get(row)).map((row) => row.id)));
223
+ const failedFiles = scopeRows.filter((row) => failedRows.has(row)).map((row) => row.fileName);
224
+ const scopeDocumentIds = [];
225
+ let scopePending = false;
226
+ for (const row of scopeRows) {
227
+ const artifactId = artifactByRow.get(row);
228
+ if (artifactId) {
229
+ scopeDocumentIds.push(artifactId);
149
230
  }
150
- const pendingReferenced = scopeDocumentIds.length === 0 &&
151
- referencesPendingFile(query, rows.map((row) => ({
152
- title: row.fileName,
153
- pending: !artifactByRow.get(row) && !failedRows.has(row)
154
- })));
155
- if (pendingReferenced) {
156
- return respond({ pendingReferenced, indexEmpty, hasKnowledge, availableFiles, failedFiles });
231
+ else if (!failedRows.has(row)) {
232
+ scopePending = true;
157
233
  }
158
- const failedReferenced = scopeRows.length === 0 &&
159
- failedRows.size > 0 &&
160
- referencesPendingFile(query, rows.map((row) => ({ title: row.fileName, pending: failedRows.has(row) })));
161
- if (failedReferenced) {
162
- return respond({
163
- indexEmpty,
164
- hasKnowledge,
165
- availableFiles,
166
- failedFiles: rows.filter((row) => failedRows.has(row)).map((row) => row.fileName)
234
+ }
235
+ const chunkedOnly = scopeRows.some((row) => !artifactByRow.get(row) && !failedRows.has(row) && mesh.files.hasText(row.id));
236
+ if (!vectors && scopeDocumentIds.length === 0 && chunkedOnly) {
237
+ return ragSearch({
238
+ chatId,
239
+ query,
240
+ mode,
241
+ limit,
242
+ attachmentIds,
243
+ meshId,
244
+ immediate,
245
+ previousQuery,
246
+ excerptsOnly
247
+ }, { vectors: true });
248
+ }
249
+ if (scopePending && scopeDocumentIds.length === 0) {
250
+ return respond({
251
+ pendingReferenced: true,
252
+ indexEmpty,
253
+ hasKnowledge,
254
+ availableFiles,
255
+ failedFiles
256
+ });
257
+ }
258
+ if (scopeIds.size > 0 && scopeDocumentIds.length === 0) {
259
+ return respond({
260
+ indexEmpty,
261
+ hasKnowledge,
262
+ availableFiles,
263
+ failedFiles
264
+ });
265
+ }
266
+ const pendingReferenced = scopeDocumentIds.length === 0 &&
267
+ referencesPendingFile(query, rows.map((row) => ({
268
+ title: row.fileName,
269
+ pending: !artifactByRow.get(row) && !failedRows.has(row)
270
+ })));
271
+ if (pendingReferenced) {
272
+ return respond({ pendingReferenced, indexEmpty, hasKnowledge, availableFiles, failedFiles });
273
+ }
274
+ const failedReferenced = scopeRows.length === 0 &&
275
+ failedRows.size > 0 &&
276
+ referencesPendingFile(query, rows.map((row) => ({ title: row.fileName, pending: failedRows.has(row) })));
277
+ if (failedReferenced) {
278
+ return respond({
279
+ indexEmpty,
280
+ hasKnowledge,
281
+ availableFiles,
282
+ failedFiles: rows.filter((row) => failedRows.has(row)).map((row) => row.fileName)
283
+ });
284
+ }
285
+ const allFailed = indexEmpty && rows.length > 0 && rows.every((row) => failedRows.has(row));
286
+ if (allFailed) {
287
+ return respond({
288
+ indexEmpty,
289
+ hasKnowledge,
290
+ availableFiles,
291
+ failedFiles: rows.map((row) => row.fileName)
292
+ });
293
+ }
294
+ if (chatIndex instanceof PluginVectorIndex) {
295
+ const ctxSize = effectiveCtxSize ? await effectiveCtxSize(agent) : 0;
296
+ const documents = [...artifactByRow].flatMap(([row, artifactId]) => {
297
+ const owner = kbRows.includes(row) ? projectIndex : chatIndex;
298
+ return artifactId && owner instanceof PluginVectorIndex
299
+ ? [{ documentId: artifactId, title: row.fileName, index: owner }]
300
+ : [];
301
+ });
302
+ const scopes = [
303
+ ...(chatRows.length > 0 ? [chatIndex.scope(chatId)] : []),
304
+ ...(kbRows.length > 0 && projectIndex instanceof PluginVectorIndex
305
+ ? [projectIndex.scope(null)]
306
+ : [])
307
+ ];
308
+ try {
309
+ const found = await searchPlugin({
310
+ query,
311
+ scopes,
312
+ documents,
313
+ scopeDocumentIds,
314
+ budgetTokens: retrievalBudgetTokens(ctxSize),
315
+ previousQuery: previousQuery ?? undefined,
316
+ limit: toolMode ? clampLimit(limit) : undefined
167
317
  });
318
+ const hits = found.hits.map(toWireHit);
319
+ const searched = { searchedCount: hits.length, indexEmpty, hasKnowledge, availableFiles };
320
+ return toolMode
321
+ ? respond({ ...searched, hits })
322
+ : respond({
323
+ ...searched,
324
+ targetDocumentId: found.targetDocumentId,
325
+ failedFiles,
326
+ uncoveredItems: listed(found.uncovered),
327
+ outline: listed(found.outline),
328
+ hits
329
+ });
168
330
  }
169
- const allFailed = indexEmpty && rows.length > 0 && rows.every((row) => failedRows.has(row));
170
- if (allFailed) {
331
+ catch (err) {
171
332
  return respond({
172
333
  indexEmpty,
173
334
  hasKnowledge,
174
335
  availableFiles,
175
- failedFiles: rows.map((row) => row.fileName)
336
+ errorMessage: errorMessage(err)
176
337
  });
177
338
  }
178
- if (toolMode) {
179
- try {
180
- const hits = await searchRagChunks({
181
- query,
182
- chatId,
183
- limit: clampLimit(limit),
184
- liveSourceIds: rows.map((row) => row.id)
185
- }, {
186
- index: vectorIndex,
187
- embed: (texts) => embed.embed({ texts }),
188
- debug: (message) => mesh.log?.debug('rag', message)
189
- });
190
- return respond({
191
- searchedCount: hits.length,
192
- indexEmpty,
193
- hasKnowledge,
194
- availableFiles,
195
- hits: hits.map(toWireHit)
196
- });
197
- }
198
- catch (err) {
199
- return respond({
200
- indexEmpty,
201
- hasKnowledge,
202
- availableFiles,
203
- errorMessage: errorMessage(err)
204
- });
205
- }
206
- }
339
+ }
340
+ if (toolMode) {
207
341
  try {
208
- const ctxSize = effectiveCtxSize ? await effectiveCtxSize(agent) : 0;
209
- const result = await retrieveRagKnowledge({
342
+ const hits = await searchRagChunks({
210
343
  query,
211
344
  chatId,
212
- documents: hydratedDocuments,
213
- liveSourceIds: rows.map((row) => row.id),
214
- scopeDocumentIds: scopeDocumentIds.length > 0 ? scopeDocumentIds : undefined,
215
- chunkBudgetTokens: retrievalBudgetTokens(ctxSize)
345
+ limit: clampLimit(limit),
346
+ liveSourceIds: rows.map((row) => row.id)
216
347
  }, {
217
348
  index: vectorIndex,
218
349
  embed: (texts) => embed.embed({ texts }),
219
350
  debug: (message) => mesh.log?.debug('rag', message)
220
351
  });
221
352
  return respond({
222
- searchedCount: result.searchedCount,
353
+ searchedCount: hits.length,
223
354
  indexEmpty,
224
355
  hasKnowledge,
225
- targetDocumentId: result.targetDocumentId,
226
356
  availableFiles,
227
- failedFiles,
228
- hits: result.hits.map(toWireHit)
357
+ hits: hits.map(toWireHit)
229
358
  });
230
359
  }
231
360
  catch (err) {
@@ -233,13 +362,121 @@ export function createRagApi(pool, { index, embedder, effectiveCtxSize = null })
233
362
  indexEmpty,
234
363
  hasKnowledge,
235
364
  availableFiles,
236
- failedFiles,
237
365
  errorMessage: errorMessage(err)
238
366
  });
239
367
  }
240
368
  }
369
+ try {
370
+ const ctxSize = effectiveCtxSize ? await effectiveCtxSize(agent) : 0;
371
+ const result = await retrieveRagKnowledge({
372
+ query,
373
+ chatId,
374
+ documents: hydratedDocuments,
375
+ liveSourceIds: rows.map((row) => row.id),
376
+ scopeDocumentIds: scopeDocumentIds.length > 0 ? scopeDocumentIds : undefined,
377
+ chunkBudgetTokens: retrievalBudgetTokens(ctxSize)
378
+ }, {
379
+ index: vectorIndex,
380
+ embed: (texts) => embed.embed({ texts }),
381
+ debug: (message) => mesh.log?.debug('rag', message)
382
+ });
383
+ return respond({
384
+ searchedCount: result.searchedCount,
385
+ indexEmpty,
386
+ hasKnowledge,
387
+ targetDocumentId: result.targetDocumentId,
388
+ availableFiles,
389
+ failedFiles,
390
+ hits: result.hits.map(toWireHit)
391
+ });
392
+ }
393
+ catch (err) {
394
+ return respond({
395
+ indexEmpty,
396
+ hasKnowledge,
397
+ availableFiles,
398
+ failedFiles,
399
+ errorMessage: errorMessage(err)
400
+ });
401
+ }
241
402
  };
403
+ return { ragSearch: (request) => ragSearch(request) };
404
+ }
405
+ async function textsOf(mesh, rows, maxChars = Infinity) {
406
+ const texts = [];
407
+ let chars = 0;
408
+ for (const row of rows) {
409
+ const text = await mesh.files.textOf(row.id, row.blobId);
410
+ chars += text?.content.length ?? 0;
411
+ if (chars > maxChars) {
412
+ return null;
413
+ }
414
+ texts.push(text);
415
+ }
416
+ return texts;
417
+ }
418
+ function pickFromText(rows, texts, { query, budgetTokens, maxBudgetTokens, synthesis, previousQuery }) {
419
+ const pendingIds = rows.filter((_, at) => !texts[at]).map((row) => row.id);
420
+ if (pendingIds.length === rows.length) {
421
+ return null;
422
+ }
423
+ const chunks = texts.flatMap((text) => text?.chunks ?? []);
424
+ const pick = rankLexical(chunks.map((chunk) => ({ ...chunk, score: 0 })), query, { budgetTokens, maxBudgetTokens, synthesis, previousQuery });
425
+ return pick && { ...pick, pendingIds };
426
+ }
427
+ async function unreadFiles(mesh, { files, rows, ids }) {
428
+ const unread = { pending: [], failed: [], states: [] };
429
+ for (const row of rows.filter((candidate) => ids.includes(candidate.id))) {
430
+ const file = files.find((candidate) => candidate.fileId === row.id);
431
+ const state = (status) => unread.states.push({ id: row.id, name: row.fileName, status });
432
+ if (file && pendingFileNames([file]).length > 0) {
433
+ unread.pending.push(row.fileName);
434
+ state(UNREAD.reading);
435
+ }
436
+ else if (file?.state === 'failed') {
437
+ unread.failed.push(row.fileName);
438
+ const rag = file.processors.find((processor) => processor.processorId === 'rag');
439
+ state(file.reason === 'no readable text' ||
440
+ rag?.reason === 'no readable text' ||
441
+ (await noText(mesh, row.id))
442
+ ? UNREAD.noText
443
+ : UNREAD.unreadable);
444
+ }
445
+ else {
446
+ unread.pending.push(row.fileName);
447
+ state(UNREAD.remote);
448
+ }
449
+ }
450
+ return unread;
451
+ }
452
+ async function listFiles(mesh, { files, rows }) {
453
+ const details = new Map();
454
+ for (const row of rows) {
455
+ const text = await mesh.files.textOf(row.id, row.blobId);
456
+ if (text) {
457
+ details.set(row.id, readable(text));
458
+ }
459
+ }
460
+ const unread = await unreadFiles(mesh, {
461
+ files,
462
+ rows,
463
+ ids: rows.filter((row) => !details.has(row.id)).map((row) => row.id)
464
+ });
465
+ for (const { id, status } of unread.states) {
466
+ details.set(id, status);
467
+ }
468
+ return manifestBlock(rows.map((row) => ({ title: row.fileName, detail: details.get(row.id) ?? '' })));
469
+ }
470
+ async function noText(mesh, knowledgeId) {
471
+ const latest = (await listIndexingByKnowledge(mesh, { knowledgeId })).reduce((current, row) => (!current || row.at > current.at ? row : current), null);
472
+ return latest?.status === 'failed' && parseFailure(latest.error).reason === 'no-text';
473
+ }
474
+ function listed(items) {
475
+ return items?.length ? [...items] : undefined;
242
476
  }
477
+ const TEXT_CHARS = 4_000_000;
478
+ const WHOLE_CHARS = 32_000;
479
+ const MAX_LIST_CONTEXT_TOKENS = 3584;
243
480
  function toWireHit(hit) {
244
481
  return { id: hit.id, content: hit.content, metadataJson: hit.metadataJson, score: hit.score };
245
482
  }