@qvac/core 1.0.0-rc.8 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (226) hide show
  1. package/CHANGELOG.md +234 -3
  2. package/README.md +1 -0
  3. package/dist-lib/lib/api/attachments.d.ts +1 -1
  4. package/dist-lib/lib/api/attachments.js +11 -0
  5. package/dist-lib/lib/api/chunks.d.ts +4 -2
  6. package/dist-lib/lib/api/chunks.js +6 -3
  7. package/dist-lib/lib/api/devices.js +6 -18
  8. package/dist-lib/lib/api/execution.d.ts +6 -2
  9. package/dist-lib/lib/api/execution.js +29 -3
  10. package/dist-lib/lib/api/host-run-log.d.ts +7 -1
  11. package/dist-lib/lib/api/host-run-log.js +3 -1
  12. package/dist-lib/lib/api/index.d.ts +3 -1
  13. package/dist-lib/lib/api/index.js +17 -6
  14. package/dist-lib/lib/api/members.js +13 -29
  15. package/dist-lib/lib/api/mesh.d.ts +2 -1
  16. package/dist-lib/lib/api/projects.d.ts +1 -1
  17. package/dist-lib/lib/api/projects.js +9 -5
  18. package/dist-lib/lib/api/rag-label.d.ts +4 -0
  19. package/dist-lib/lib/api/rag-label.js +21 -0
  20. package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
  21. package/dist-lib/lib/api/rag-lexical.js +1058 -0
  22. package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
  23. package/dist-lib/lib/api/rag-manifest.js +46 -0
  24. package/dist-lib/lib/api/rag-outline.d.ts +36 -0
  25. package/dist-lib/lib/api/rag-outline.js +426 -0
  26. package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
  27. package/dist-lib/lib/api/rag-plugin.js +70 -0
  28. package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
  29. package/dist-lib/lib/api/rag-synthesis.js +102 -0
  30. package/dist-lib/lib/api/rag-targets.d.ts +23 -0
  31. package/dist-lib/lib/api/rag-targets.js +215 -0
  32. package/dist-lib/lib/api/rag-window.d.ts +5 -0
  33. package/dist-lib/lib/api/rag-window.js +46 -0
  34. package/dist-lib/lib/api/rag.d.ts +3 -2
  35. package/dist-lib/lib/api/rag.js +421 -181
  36. package/dist-lib/lib/api/served.js +8 -14
  37. package/dist-lib/lib/api/workspace.d.ts +8 -1
  38. package/dist-lib/lib/api/workspace.js +27 -13
  39. package/dist-lib/lib/assistant/harness.d.ts +9 -1
  40. package/dist-lib/lib/assistant/history.d.ts +11 -2
  41. package/dist-lib/lib/assistant/history.js +74 -4
  42. package/dist-lib/lib/assistant/media-round.d.ts +61 -0
  43. package/dist-lib/lib/assistant/media-round.js +111 -0
  44. package/dist-lib/lib/assistant/message.d.ts +7 -0
  45. package/dist-lib/lib/assistant/model-round.d.ts +12 -0
  46. package/dist-lib/lib/assistant/model-round.js +130 -17
  47. package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
  48. package/dist-lib/lib/assistant/pdf-pages.js +67 -3
  49. package/dist-lib/lib/assistant/round-host.d.ts +22 -0
  50. package/dist-lib/lib/assistant/round-host.js +142 -8
  51. package/dist-lib/lib/assistant/upload.d.ts +12 -0
  52. package/dist-lib/lib/assistant/upload.js +36 -0
  53. package/dist-lib/lib/assistant-client.d.ts +1 -0
  54. package/dist-lib/lib/assistant-client.js +40 -19
  55. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  56. package/dist-lib/lib/assistant-tree.js +41 -1
  57. package/dist-lib/lib/assistant.d.ts +15 -0
  58. package/dist-lib/lib/assistant.js +474 -89
  59. package/dist-lib/lib/client.d.ts +1 -0
  60. package/dist-lib/lib/client.js +4 -1
  61. package/dist-lib/lib/compaction.js +2 -2
  62. package/dist-lib/lib/core/agent.js +12 -45
  63. package/dist-lib/lib/core/approval-rules.js +1 -8
  64. package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
  65. package/dist-lib/lib/core/archived-mesh.js +12 -8
  66. package/dist-lib/lib/core/chat-copy.d.ts +18 -3
  67. package/dist-lib/lib/core/chat-copy.js +74 -25
  68. package/dist-lib/lib/core/chat-fork.js +15 -6
  69. package/dist-lib/lib/core/chat-group.js +16 -24
  70. package/dist-lib/lib/core/chat-move.d.ts +9 -0
  71. package/dist-lib/lib/core/chat-move.js +94 -3
  72. package/dist-lib/lib/core/chat-order.js +12 -18
  73. package/dist-lib/lib/core/chat.js +38 -31
  74. package/dist-lib/lib/core/chunk.d.ts +2 -2
  75. package/dist-lib/lib/core/chunk.js +18 -25
  76. package/dist-lib/lib/core/collections.d.ts +2 -0
  77. package/dist-lib/lib/core/collections.js +2 -0
  78. package/dist-lib/lib/core/contract/activity.d.ts +4 -0
  79. package/dist-lib/lib/core/contract/activity.js +34 -4
  80. package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
  81. package/dist-lib/lib/core/contract/chat-group.js +21 -1
  82. package/dist-lib/lib/core/contract/chat.d.ts +9 -0
  83. package/dist-lib/lib/core/contract/chat.js +127 -5
  84. package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
  85. package/dist-lib/lib/core/contract/chunks.js +37 -6
  86. package/dist-lib/lib/core/contract/features.d.ts +21 -0
  87. package/dist-lib/lib/core/contract/features.js +22 -0
  88. package/dist-lib/lib/core/contract/index.js +2 -0
  89. package/dist-lib/lib/core/contract/indexing.js +4 -2
  90. package/dist-lib/lib/core/contract/knowledge.js +5 -4
  91. package/dist-lib/lib/core/contract/members.js +16 -10
  92. package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
  93. package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
  94. package/dist-lib/lib/core/contract/projects.d.ts +5 -0
  95. package/dist-lib/lib/core/contract/projects.js +13 -5
  96. package/dist-lib/lib/core/contract/roles.js +32 -4
  97. package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
  98. package/dist-lib/lib/core/contract/run-requests.js +35 -2
  99. package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
  100. package/dist-lib/lib/core/contract/tool-configs.js +3 -2
  101. package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
  102. package/dist-lib/lib/core/contract/workspace.js +40 -7
  103. package/dist-lib/lib/core/contract/writers.js +2 -2
  104. package/dist-lib/lib/core/departures.d.ts +3 -1
  105. package/dist-lib/lib/core/departures.js +3 -0
  106. package/dist-lib/lib/core/devices.js +1 -8
  107. package/dist-lib/lib/core/files/attachments.d.ts +2 -0
  108. package/dist-lib/lib/core/files/attachments.js +19 -1
  109. package/dist-lib/lib/core/files/index.d.ts +20 -4
  110. package/dist-lib/lib/core/files/index.js +132 -21
  111. package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
  112. package/dist-lib/lib/core/files/processing/document.js +7 -0
  113. package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
  114. package/dist-lib/lib/core/files/processing/index.js +144 -12
  115. package/dist-lib/lib/core/files/processing/protocol.js +13 -34
  116. package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
  117. package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
  118. package/dist-lib/lib/core/files/rag/build.js +123 -19
  119. package/dist-lib/lib/core/files/rag/failure.js +2 -1
  120. package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
  121. package/dist-lib/lib/core/files/rag/index.js +13 -1
  122. package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
  123. package/dist-lib/lib/core/files/rag/load.js +27 -6
  124. package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
  125. package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
  126. package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
  127. package/dist-lib/lib/core/files/rag/produce.js +104 -16
  128. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
  129. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
  130. package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
  131. package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
  132. package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
  133. package/dist-lib/lib/core/files/records/knowledge.js +7 -40
  134. package/dist-lib/lib/core/files/records/reclaim.js +5 -10
  135. package/dist-lib/lib/core/files/upload.js +68 -41
  136. package/dist-lib/lib/core/fs-drives.js +14 -26
  137. package/dist-lib/lib/core/invite.d.ts +1 -0
  138. package/dist-lib/lib/core/invite.js +4 -2
  139. package/dist-lib/lib/core/local/files.d.ts +1 -1
  140. package/dist-lib/lib/core/local/files.js +5 -5
  141. package/dist-lib/lib/core/local/index.d.ts +13 -6
  142. package/dist-lib/lib/core/local/index.js +103 -69
  143. package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
  144. package/dist-lib/lib/core/local/run-logs.js +5 -5
  145. package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
  146. package/dist-lib/lib/core/mesh-processor.js +87 -0
  147. package/dist-lib/lib/core/mesh.d.ts +48 -5
  148. package/dist-lib/lib/core/mesh.js +318 -129
  149. package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
  150. package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
  151. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
  152. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
  153. package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
  154. package/dist-lib/lib/core/migrations/writer-links.js +32 -0
  155. package/dist-lib/lib/core/project-keys.d.ts +6 -0
  156. package/dist-lib/lib/core/project-keys.js +11 -0
  157. package/dist-lib/lib/core/public-mesh.d.ts +3 -0
  158. package/dist-lib/lib/core/public-mesh.js +10 -8
  159. package/dist-lib/lib/core/run-requests.d.ts +3 -0
  160. package/dist-lib/lib/core/run-requests.js +21 -1
  161. package/dist-lib/lib/core/skill-availability.d.ts +33 -0
  162. package/dist-lib/lib/core/skill-availability.js +67 -0
  163. package/dist-lib/lib/core/sync/background.d.ts +0 -1
  164. package/dist-lib/lib/core/sync/background.js +2 -1
  165. package/dist-lib/lib/core/sync/changed.d.ts +4 -2
  166. package/dist-lib/lib/core/sync/changed.js +1 -1
  167. package/dist-lib/lib/core/sync/index.js +52 -10
  168. package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
  169. package/dist-lib/lib/core/sync/view-updates.js +82 -91
  170. package/dist-lib/lib/core/sync-status.js +97 -41
  171. package/dist-lib/lib/core/tool-activity.js +6 -12
  172. package/dist-lib/lib/core/tool-configs.js +17 -29
  173. package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
  174. package/dist-lib/lib/core/upgrade-summary.js +67 -0
  175. package/dist-lib/lib/core/view-read.d.ts +4 -0
  176. package/dist-lib/lib/core/view-read.js +23 -0
  177. package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
  178. package/dist-lib/lib/core/workspace-upgrade.js +27 -16
  179. package/dist-lib/lib/core.d.ts +39 -42
  180. package/dist-lib/lib/core.js +661 -639
  181. package/dist-lib/lib/errors.d.ts +20 -0
  182. package/dist-lib/lib/errors.js +20 -0
  183. package/dist-lib/lib/llm-model-names.d.ts +1 -1
  184. package/dist-lib/lib/llm-model-names.js +1 -1
  185. package/dist-lib/lib/mesh-scoped-engine.js +4 -0
  186. package/dist-lib/lib/model-context.js +1 -2
  187. package/dist-lib/lib/probe-rounds.d.ts +7 -2
  188. package/dist-lib/lib/probe-rounds.js +6 -1
  189. package/dist-lib/lib/run-transcript.d.ts +4 -1
  190. package/dist-lib/lib/run-transcript.js +15 -6
  191. package/dist-lib/schema/engine.js +83 -6
  192. package/dist-lib/schema/indexes.d.ts +3 -0
  193. package/dist-lib/schema/indexes.js +5 -0
  194. package/dist-lib/schema/local.js +11 -2
  195. package/dist-lib/schema/qvac.js +49 -5
  196. package/dist-lib/spec/bind.js +34 -0
  197. package/dist-lib/spec/calls.js +5 -0
  198. package/dist-lib/spec/capabilities.d.ts +17 -0
  199. package/dist-lib/spec/hrpc/hrpc.json +65 -0
  200. package/dist-lib/spec/hrpc/index.js +68 -5
  201. package/dist-lib/spec/hrpc/messages.js +260 -23
  202. package/dist-lib/spec/hyperschema/index.js +260 -23
  203. package/dist-lib/spec/hyperschema/schema.json +181 -0
  204. package/dist-lib/spec/hyperschema/types.d.ts +48 -0
  205. package/dist-lib/spec/local/hyperdb/messages.js +270 -224
  206. package/dist-lib/spec/local/hyperschema/index.js +73 -38
  207. package/dist-lib/spec/local/hyperschema/schema.json +31 -0
  208. package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
  209. package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
  210. package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
  211. package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
  212. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
  213. package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
  214. package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
  215. package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
  216. package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
  217. package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
  218. package/package.json +75 -70
  219. package/tui/README.md +2 -1
  220. package/tui/app.mjs +28 -7
  221. package/tui/boot.mjs +5 -1
  222. package/tui/lib/attachment-files.mjs +82 -0
  223. package/tui/lib/reopen-meshes.mjs +15 -0
  224. package/tui/main.mjs +4 -43
  225. package/worker/assistant-sidecar.mjs +10 -3
  226. package/worker/stow-entry.mjs +3 -52
@@ -73,29 +73,50 @@ function describeCoverage(coverage) {
73
73
  function formatChars(count) {
74
74
  return count >= 1000 ? `${Math.round(count / 1000)}k` : String(count);
75
75
  }
76
- export async function fullTextHits(mesh, rows, { budgetChars, head = false }) {
76
+ function fullTextKind(kind, extractors) {
77
+ return FULL_TEXT_KINDS.has(kind) || (kind === RAG_DOCUMENT_KIND.office && !!extractors?.office);
78
+ }
79
+ export async function fullTextHits(mesh, rows, { budgetChars, head = false, extractors }) {
77
80
  if (rows.length === 0)
78
81
  return null;
79
82
  const kinds = rows.map((row) => classifyRagDocument({ fileName: row.fileName, mimeType: row.mimeType }));
80
- if (!kinds.every((kind) => FULL_TEXT_KINDS.has(kind))) {
81
- return null;
82
- }
83
+ const filesByChat = new Map();
83
84
  const hits = [];
84
85
  let used = 0;
85
86
  for (const [index, row] of rows.entries()) {
86
87
  const remaining = budgetChars - used;
87
- if (!head && kinds[index] !== RAG_DOCUMENT_KIND.pdf && row.byteLength > remaining) {
88
+ if (remaining <= 0) {
89
+ return null;
90
+ }
91
+ const content = (await mesh.files.textOf(row.id, row.blobId))?.content;
92
+ if (!content && !fullTextKind(kinds[index] ?? '', extractors)) {
93
+ return null;
94
+ }
95
+ if (!content && (await extractingHere(mesh, row, filesByChat))) {
96
+ return null;
97
+ }
98
+ if (!content &&
99
+ !head &&
100
+ FULL_TEXT_KINDS.has(kinds[index] ?? '') &&
101
+ kinds[index] !== RAG_DOCUMENT_KIND.pdf &&
102
+ row.byteLength > remaining) {
88
103
  return null;
89
104
  }
90
105
  let read;
91
106
  try {
92
- const bytes = await mesh.files.readBytes(row.blobId);
93
- read = await extractFullText(bytes, {
94
- kind: kinds[index],
95
- row,
96
- remainingChars: remaining,
97
- head
98
- });
107
+ if (content) {
108
+ read = { text: content, complete: true, totalChars: content.length };
109
+ }
110
+ else {
111
+ const bytes = await mesh.files.readBytes(row.blobId);
112
+ read = await extractFullText(bytes, {
113
+ kind: kinds[index],
114
+ row,
115
+ remainingChars: remaining,
116
+ head,
117
+ extractors
118
+ });
119
+ }
99
120
  }
100
121
  catch (err) {
101
122
  mesh.log?.warn('rag', 'full-text extraction failed; falling back to retrieval', { err });
@@ -133,6 +154,26 @@ export async function fullTextHits(mesh, rows, { budgetChars, head = false }) {
133
154
  }
134
155
  return hits;
135
156
  }
157
+ async function extractingHere(mesh, row, filesByChat) {
158
+ if (!row.chatId)
159
+ return false;
160
+ let files = filesByChat.get(row.chatId);
161
+ if (!files) {
162
+ files = mesh.files.status(row.chatId).then(({ files }) => files);
163
+ filesByChat.set(row.chatId, files);
164
+ }
165
+ try {
166
+ const file = (await files).find((entry) => entry.fileId === row.id);
167
+ return !!file && mesh.files.extractingHere(file);
168
+ }
169
+ catch (err) {
170
+ mesh.log?.warn('rag', 'checking local extraction state failed; skipping full-text fallback', {
171
+ id: row.id,
172
+ err
173
+ });
174
+ return true;
175
+ }
176
+ }
136
177
  function coverageOf(read, included) {
137
178
  if (read.pages && read.pages.read < read.pages.total) {
138
179
  return { unit: 'pages', included: read.pages.read, total: read.pages.total };
@@ -140,7 +181,7 @@ function coverageOf(read, included) {
140
181
  return { unit: 'chars', included, total: read.totalChars ?? read.text.length };
141
182
  }
142
183
  const MAX_BYTES_PER_CHAR = 4;
143
- async function extractFullText(bytes, { kind, row, remainingChars, head }) {
184
+ async function extractFullText(bytes, { kind, row, remainingChars, head, extractors }) {
144
185
  if (kind === RAG_DOCUMENT_KIND.pdf) {
145
186
  const pdf = await extractPdfHead(bytes, { maxChars: remainingChars });
146
187
  return {
@@ -149,6 +190,10 @@ async function extractFullText(bytes, { kind, row, remainingChars, head }) {
149
190
  pages: { read: pdf.pagesRead, total: pdf.totalPages }
150
191
  };
151
192
  }
193
+ if (kind === RAG_DOCUMENT_KIND.office) {
194
+ const text = await readOffice(bytes, row, extractors);
195
+ return { text, complete: true, totalChars: text.length };
196
+ }
152
197
  const maxPrefixBytes = remainingChars * MAX_BYTES_PER_CHAR;
153
198
  const prefix = head && bytes.length > maxPrefixBytes;
154
199
  const document = await extractRagDocumentFromBytes({
@@ -165,3 +210,7 @@ async function extractFullText(bytes, { kind, row, remainingChars, head }) {
165
210
  totalChars: bytes.length
166
211
  };
167
212
  }
213
+ async function readOffice(bytes, row, extractors) {
214
+ const document = await extractRagDocumentFromBytes({ fileName: row.fileName, mimeType: row.mimeType, bytes }, { extractors });
215
+ return document.content;
216
+ }
@@ -0,0 +1,28 @@
1
+ import type { RagPreparedChunk } from 'rag-extract';
2
+ export interface CachedText {
3
+ readonly blobId: string;
4
+ readonly content: string;
5
+ readonly chunks: ReadonlyArray<RagPreparedChunk>;
6
+ readonly artifactId?: string | undefined;
7
+ }
8
+ export declare class TextCache {
9
+ private readonly _dir;
10
+ private readonly _held;
11
+ private readonly _operations;
12
+ private _listed;
13
+ private readonly _recent;
14
+ private readonly _usable;
15
+ private _chars;
16
+ private _epoch;
17
+ constructor(dir: string);
18
+ has(id: string): boolean;
19
+ usable(id: string, blobId: string): boolean | undefined;
20
+ list(): Promise<void>;
21
+ read(id: string, blobId: string): Promise<CachedText | null>;
22
+ write(id: string, text: CachedText): Promise<void>;
23
+ remove(id: string): Promise<void>;
24
+ private _serialize;
25
+ private _remember;
26
+ private _forget;
27
+ private _file;
28
+ }
@@ -0,0 +1,141 @@
1
+ import fs from 'fs';
2
+ import path from 'path';
3
+ import { isRecord } from '../../../guards.js';
4
+ const MEMORY_CHARS = 6_000_000;
5
+ const MEMORY_FILES = 16;
6
+ export class TextCache {
7
+ _dir;
8
+ _held = new Set();
9
+ _operations = new Map();
10
+ _listed = null;
11
+ _recent = new Map();
12
+ _usable = new Map();
13
+ _chars = 0;
14
+ _epoch = 0;
15
+ constructor(dir) {
16
+ this._dir = dir;
17
+ }
18
+ has(id) {
19
+ return this._held.has(id);
20
+ }
21
+ usable(id, blobId) {
22
+ const known = this._usable.get(id);
23
+ return known?.blobId === blobId ? known.usable : undefined;
24
+ }
25
+ async list() {
26
+ this._listed ??= fs.promises.readdir(this._dir).then((names) => {
27
+ for (const name of names) {
28
+ if (name.endsWith('.json')) {
29
+ this._held.add(name.slice(0, -'.json'.length));
30
+ }
31
+ }
32
+ }, () => { });
33
+ await this._listed;
34
+ }
35
+ async read(id, blobId) {
36
+ const held = this._recent.get(id);
37
+ if (held?.text.blobId === blobId) {
38
+ this._recent.delete(id);
39
+ this._recent.set(id, held);
40
+ return held.text;
41
+ }
42
+ const epoch = this._epoch;
43
+ const parsed = await fs.promises
44
+ .readFile(this._file(id), 'utf8')
45
+ .then((json) => JSON.parse(json), noop);
46
+ if (epoch !== this._epoch) {
47
+ return this.read(id, blobId);
48
+ }
49
+ const cached = toCachedText(parsed);
50
+ if (cached?.blobId !== blobId) {
51
+ this._held.delete(id);
52
+ this._forget(id);
53
+ this._usable.delete(id);
54
+ return null;
55
+ }
56
+ this._held.add(id);
57
+ this._remember(id, cached);
58
+ return cached;
59
+ }
60
+ async write(id, text) {
61
+ await this._serialize(id, async () => {
62
+ const epoch = ++this._epoch;
63
+ this._forget(id);
64
+ await fs.promises.mkdir(this._dir, { recursive: true });
65
+ const partial = `${this._file(id)}.partial`;
66
+ await fs.promises.writeFile(partial, JSON.stringify(text));
67
+ await fs.promises.rename(partial, this._file(id));
68
+ this._held.add(id);
69
+ if (epoch === this._epoch) {
70
+ this._remember(id, text);
71
+ }
72
+ });
73
+ }
74
+ async remove(id) {
75
+ await this._serialize(id, async () => {
76
+ this._epoch++;
77
+ this._forget(id);
78
+ this._held.delete(id);
79
+ this._usable.delete(id);
80
+ await fs.promises.rm(this._file(id), { force: true });
81
+ await fs.promises.rm(`${this._file(id)}.partial`, { force: true });
82
+ });
83
+ }
84
+ async _serialize(id, operation) {
85
+ const previous = this._operations.get(id) ?? Promise.resolve();
86
+ const current = previous.catch(noop).then(operation);
87
+ this._operations.set(id, current);
88
+ try {
89
+ await current;
90
+ }
91
+ finally {
92
+ if (this._operations.get(id) === current) {
93
+ this._operations.delete(id);
94
+ }
95
+ }
96
+ }
97
+ _remember(id, text) {
98
+ this._forget(id);
99
+ this._usable.set(id, { blobId: text.blobId, usable: !!text.content.trim() });
100
+ const chars = text.content.length +
101
+ text.chunks.reduce((sum, chunk) => sum + chunk.content.length + chunk.metadataJson.length + chunk.id.length, 0);
102
+ if (chars > MEMORY_CHARS) {
103
+ return;
104
+ }
105
+ while (this._recent.size >= MEMORY_FILES || this._chars + chars > MEMORY_CHARS) {
106
+ const oldest = this._recent.keys().next().value;
107
+ if (oldest === undefined) {
108
+ break;
109
+ }
110
+ this._forget(oldest);
111
+ }
112
+ this._recent.set(id, { text, chars });
113
+ this._chars += chars;
114
+ }
115
+ _forget(id) {
116
+ const held = this._recent.get(id);
117
+ if (held) {
118
+ this._chars -= held.chars;
119
+ }
120
+ this._recent.delete(id);
121
+ }
122
+ _file(id) {
123
+ return path.join(this._dir, `${id.replace(/[^\w-]/g, '_')}.json`);
124
+ }
125
+ }
126
+ function toCachedText(value) {
127
+ if (!isRecord(value) || typeof value.blobId !== 'string' || typeof value.content !== 'string') {
128
+ return null;
129
+ }
130
+ const chunks = Array.isArray(value.chunks) ? value.chunks.filter(isChunk) : [];
131
+ const artifactId = typeof value.artifactId === 'string' ? value.artifactId : undefined;
132
+ return { blobId: value.blobId, content: value.content, chunks, artifactId };
133
+ }
134
+ function isChunk(value) {
135
+ return (isRecord(value) &&
136
+ typeof value.id === 'string' &&
137
+ typeof value.content === 'string' &&
138
+ typeof value.metadataJson === 'string' &&
139
+ typeof value.tokenCount === 'number');
140
+ }
141
+ function noop() { }
@@ -24,8 +24,6 @@ export declare function assertRemovableAt(at: number): void;
24
24
  export declare function removeKnowledge(store: Mesh, { knowledgeId, at }: QvacRemoveKnowledgeOperation): Promise<QvacKnowledge>;
25
25
  export type AddKnowledgeArtifactInput = Omit<QvacKnowledgeArtifact, 'createdAt'>;
26
26
  export declare function getKnowledgeArtifact(store: Mesh, id: string): Promise<QvacKnowledgeArtifact | null>;
27
- type KnowledgeSnapshot = ReturnType<Mesh['view']['snapshot']>;
28
27
  export declare function listKnowledgeArtifactsByKnowledge(store: Mesh, { knowledgeId }: KnowledgeScope): Promise<QvacKnowledgeArtifact[]>;
29
- export declare function findKnowledgeArtifactsByKnowledge(snap: KnowledgeSnapshot, knowledgeId: string): Promise<QvacKnowledgeArtifact[]>;
28
+ export declare function findKnowledgeArtifactsByKnowledge(snap: HyperDB, knowledgeId: string): Promise<QvacKnowledgeArtifact[]>;
30
29
  export declare function addKnowledgeArtifact(store: Mesh, input: AddKnowledgeArtifactInput): Promise<QvacKnowledgeArtifact>;
31
- export {};
@@ -5,14 +5,7 @@ import { DEFAULT_KNOWLEDGE_CAP_BYTES } from '../rag/limits.js';
5
5
  import { KNOWLEDGE_CAP_ROW } from '../../contract/knowledge.js';
6
6
  import { ACTIVE_KNOWLEDGE_BASE, ACTIVE_KNOWLEDGE_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_CAPS } from '../../collections.js';
7
7
  export async function getKnowledge(store, id) {
8
- const snap = store.view.snapshot();
9
- try {
10
- const row = await snap.get(KNOWLEDGE, { id });
11
- return row;
12
- }
13
- finally {
14
- await snap.close();
15
- }
8
+ return store.view.get(KNOWLEDGE, { id });
16
9
  }
17
10
  export async function mustGetKnowledge(store, id) {
18
11
  const row = await getKnowledge(store, id);
@@ -22,25 +15,12 @@ export async function mustGetKnowledge(store, id) {
22
15
  return row;
23
16
  }
24
17
  export async function listKnowledge(store, { chatId }) {
25
- const snap = store.view.snapshot();
26
- try {
27
- const rows = await snap
28
- .find(ACTIVE_KNOWLEDGE_BY_CHAT, { gte: { chatId }, lte: { chatId } })
29
- .toArray();
30
- return rows;
31
- }
32
- finally {
33
- await snap.close();
34
- }
18
+ return store.view
19
+ .find(ACTIVE_KNOWLEDGE_BY_CHAT, { gte: { chatId }, lte: { chatId } })
20
+ .toArray();
35
21
  }
36
22
  export async function listKnowledgeBase(store) {
37
- const snap = store.view.snapshot();
38
- try {
39
- return await snap.find(ACTIVE_KNOWLEDGE_BASE).toArray();
40
- }
41
- finally {
42
- await snap.close();
43
- }
23
+ return store.view.find(ACTIVE_KNOWLEDGE_BASE).toArray();
44
24
  }
45
25
  export function usedKnowledgeBytes(rows) {
46
26
  return rows.reduce((sum, row) => sum + row.byteLength, 0);
@@ -101,23 +81,10 @@ const ARTIFACT_STRING_FIELDS = [
101
81
  'artifactBlobId'
102
82
  ];
103
83
  export async function getKnowledgeArtifact(store, id) {
104
- const snap = store.view.snapshot();
105
- try {
106
- const artifact = await snap.get(KNOWLEDGE_ARTIFACTS, { id });
107
- return artifact;
108
- }
109
- finally {
110
- await snap.close();
111
- }
84
+ return store.view.get(KNOWLEDGE_ARTIFACTS, { id });
112
85
  }
113
86
  export async function listKnowledgeArtifactsByKnowledge(store, { knowledgeId }) {
114
- const snap = store.view.snapshot();
115
- try {
116
- return await findKnowledgeArtifactsByKnowledge(snap, knowledgeId);
117
- }
118
- finally {
119
- await snap.close();
120
- }
87
+ return findKnowledgeArtifactsByKnowledge(store.view, knowledgeId);
121
88
  }
122
89
  export async function findKnowledgeArtifactsByKnowledge(snap, knowledgeId) {
123
90
  return await snap
@@ -7,6 +7,7 @@ export async function syncKnowledgeTombstone(mesh, row) {
7
7
  if (!row.deletedAt)
8
8
  return;
9
9
  mesh.log?.debug('sync', 'knowledge tombstone observed - reclaiming blob bytes', { id: row.id });
10
+ await mesh.files.forgetText(row.id);
10
11
  if (typeof row.blobId === 'string' && !(await hasLiveSource(mesh, row.chatId, row.blobId))) {
11
12
  await mesh.files.clear(row.blobId, row.id);
12
13
  await mesh.fileIndex?.markCleared(row.blobId);
@@ -35,16 +36,10 @@ export async function sweepTombstonedBlobs(mesh, deviceId = null) {
35
36
  }
36
37
  await mesh.files.syncAttachments(await listChunks(mesh, { chatId, type: 'attachment' }));
37
38
  }
38
- const snap = mesh.view.snapshot();
39
- try {
40
- for await (const row of snap.find(KNOWLEDGE)) {
41
- if (mesh.closing)
42
- return;
43
- await syncKnowledgeTombstone(mesh, row);
44
- }
45
- }
46
- finally {
47
- await snap.close();
39
+ for await (const row of mesh.view.find(KNOWLEDGE)) {
40
+ if (mesh.closing)
41
+ return;
42
+ await syncKnowledgeTombstone(mesh, row);
48
43
  }
49
44
  }
50
45
  async function recascadeTombstonedChat(mesh, chatId, deviceId) {
@@ -50,60 +50,34 @@ export async function* uploadFile(mesh, frames, { kind, maxBytes, deviceId, inac
50
50
  uploadId = upload.id;
51
51
  input.onData = (bytes) => mesh.files.uploadProgress(upload.id, bytes);
52
52
  input.onFailure = (error) => mesh.files.uploadFailed(upload.id, error);
53
- const source = capped(dataFrames(it, first.value.data), limit, tooLarge);
54
- let stream = source;
55
- let storedMime = mimeType;
56
- const declaredSniffed = mapDeclaredToSniffed(storedMime);
57
- if (kind === 'knowledge') {
58
- const { head, rest } = await peekHead(source, 4096);
59
- if (declaredSniffed !== null && detectImageMime(head) !== declaredSniffed) {
60
- throw new CoreError('UPLOAD_MIME_MISMATCH', ['uploadKnowledge', mimeType]);
61
- }
62
- if (!isSupportedRagDocument({ fileName, mimeType, bytes: head })) {
63
- throw new CoreError('KNOWLEDGE_TYPE_UNSUPPORTED', [fileName]);
64
- }
65
- stream = rest;
66
- }
67
- else if (declaredSniffed !== null) {
68
- const image = yield* imageUpload(source, {
69
- raw,
70
- declared: declaredSniffed,
71
- mimeType: storedMime
53
+ const stored = first.value.blobId
54
+ ? existingBlob(first.value, { kind, mimeType })
55
+ : yield* writtenBlob(mesh, it, {
56
+ kind,
57
+ limit,
58
+ tooLarge,
59
+ inactivityMs,
60
+ file: { fileName, mimeType, raw: raw ?? undefined, data: first.value.data }
72
61
  });
73
- stream = image.stream;
74
- storedMime = image.mimeType;
75
- }
76
- let final = null;
77
- for await (const frame of mesh.files.write(stream, { inactivityMs })) {
78
- if ('blobId' in frame) {
79
- final = frame;
80
- }
81
- else {
82
- yield { bytesWritten: frame.bytesWritten };
83
- }
84
- }
85
- if (final === null) {
86
- throw new Error('blob write ended without its blobId frame');
87
- }
88
62
  const attachment = {
89
63
  id: upload.id,
90
64
  fileName,
91
- mimeType: storedMime,
92
- byteLength: final.byteLength,
93
- blobId: mesh.files.reference(final.blobId),
65
+ mimeType: stored.mimeType,
66
+ byteLength: stored.byteLength,
67
+ blobId: stored.reference,
94
68
  createdAt: Date.now()
95
69
  };
96
70
  if (!chatId) {
97
71
  const knowledge = await knowledgeBaseRow(mesh, input, attachment);
98
72
  mesh.files.uploadComplete(upload.id);
99
73
  input.onFailure = null;
100
- yield { bytesWritten: final.bytesWritten, knowledge };
74
+ yield { bytesWritten: stored.bytesWritten, knowledge };
101
75
  return;
102
76
  }
103
77
  let chunkId;
104
78
  try {
105
79
  input.check();
106
- const preview = IMAGE_MIMES.has(storedMime.toLowerCase());
80
+ const preview = IMAGE_MIMES.has(stored.mimeType.toLowerCase());
107
81
  const appended = await appendChunk(mesh, {
108
82
  id: preview ? upload.id : undefined,
109
83
  display: preview ? { preview: true } : undefined,
@@ -117,7 +91,9 @@ export async function* uploadFile(mesh, frames, { kind, maxBytes, deviceId, inac
117
91
  chunkId = appended.id;
118
92
  }
119
93
  catch (err) {
120
- await mesh.files.clear(attachment.blobId, attachment.id).catch(noop);
94
+ if (!first.value.blobId) {
95
+ await mesh.files.clear(attachment.blobId, attachment.id).catch(noop);
96
+ }
121
97
  throw err;
122
98
  }
123
99
  let knowledge;
@@ -142,7 +118,7 @@ export async function* uploadFile(mesh, frames, { kind, maxBytes, deviceId, inac
142
118
  mesh.files.uploadComplete(upload.id);
143
119
  input.onFailure = null;
144
120
  yield {
145
- bytesWritten: final.bytesWritten,
121
+ bytesWritten: stored.bytesWritten,
146
122
  attachment,
147
123
  knowledge
148
124
  };
@@ -158,6 +134,57 @@ export async function* uploadFile(mesh, frames, { kind, maxBytes, deviceId, inac
158
134
  }
159
135
  }
160
136
  }
137
+ function existingBlob({ blobId, raw, totalBytes, data }, { kind, mimeType }) {
138
+ if (!blobId || kind !== 'attachment' || !raw || !totalBytes || data) {
139
+ throw new CoreError('UPLOAD_BLOB_INCOMPLETE');
140
+ }
141
+ return { reference: blobId, byteLength: totalBytes, bytesWritten: 0, mimeType };
142
+ }
143
+ async function* writtenBlob(mesh, it, { kind, limit, tooLarge, inactivityMs, file }) {
144
+ const { fileName, mimeType, raw } = file;
145
+ const source = capped(dataFrames(it, file.data), limit, tooLarge);
146
+ let stream = source;
147
+ let storedMime = mimeType;
148
+ const declaredSniffed = mapDeclaredToSniffed(storedMime);
149
+ if (kind === 'knowledge') {
150
+ const { head, rest } = await peekHead(source, 4096);
151
+ if (declaredSniffed !== null && detectImageMime(head) !== declaredSniffed) {
152
+ throw new CoreError('UPLOAD_MIME_MISMATCH', ['uploadKnowledge', mimeType]);
153
+ }
154
+ if (!isSupportedRagDocument({ fileName, mimeType, bytes: head })) {
155
+ throw new CoreError('KNOWLEDGE_TYPE_UNSUPPORTED', [fileName]);
156
+ }
157
+ stream = rest;
158
+ }
159
+ else if (declaredSniffed !== null) {
160
+ const image = yield* imageUpload(source, {
161
+ raw,
162
+ declared: declaredSniffed,
163
+ mimeType: storedMime
164
+ });
165
+ stream = image.stream;
166
+ storedMime = image.mimeType;
167
+ }
168
+ let final = null;
169
+ for await (const frame of mesh.files.write(stream, { inactivityMs })) {
170
+ if ('blobId' in frame) {
171
+ final = frame;
172
+ }
173
+ else {
174
+ yield { bytesWritten: frame.bytesWritten };
175
+ }
176
+ }
177
+ if (final === null) {
178
+ throw new Error('blob write ended without its blobId frame');
179
+ }
180
+ const { byteLength, bytesWritten } = final;
181
+ return {
182
+ reference: mesh.files.reference(final.blobId),
183
+ byteLength,
184
+ bytesWritten,
185
+ mimeType: storedMime
186
+ };
187
+ }
161
188
  async function knowledgeBaseRow(mesh, input, attachment) {
162
189
  const { id, fileName, mimeType, byteLength, blobId } = attachment;
163
190
  try {
@@ -2,19 +2,7 @@ import BlindPeering from 'blind-peering';
2
2
  import Hyperdrive from 'hyperdrive';
3
3
  import HypercoreId from 'hypercore-id-encoding';
4
4
  const NAMESPACE = 'fs-drives';
5
- const DRIVE_OPEN_TIMEOUT_MS = 15_000;
6
- function withDeadline(promise, ms) {
7
- const { promise: gated, resolve, reject } = Promise.withResolvers();
8
- const timer = setTimeout(() => reject(new Error('drive open timed out on the host')), ms);
9
- promise.then((value) => {
10
- clearTimeout(timer);
11
- resolve(value);
12
- }, (err) => {
13
- clearTimeout(timer);
14
- reject(err);
15
- });
16
- return gated;
17
- }
5
+ const noop = () => { };
18
6
  async function openDrive(store, swarm, key) {
19
7
  const drive = new Hyperdrive(store, HypercoreId.decode(key));
20
8
  await drive.ready();
@@ -26,29 +14,27 @@ async function openDrive(store, swarm, key) {
26
14
  export function createFsDriveHost(store, opts) {
27
15
  const namespaced = store.namespace(NAMESPACE);
28
16
  const drives = new Map();
29
- async function peer(held) {
17
+ function peer(held) {
30
18
  const peering = opts.blindPeering();
31
19
  if (!peering) {
32
20
  return;
33
21
  }
34
- const blobs = await held.drive.getBlobs();
35
22
  peering.addCoreBackground(held.drive.core, { keys: opts.blindPeerKeys, pick: 2 });
36
- if (blobs) {
37
- peering.addCoreBackground(blobs.core, { keys: opts.blindPeerKeys, pick: 2 });
38
- }
23
+ held.drive.getBlobs().then((blobs) => {
24
+ const client = opts.blindPeering();
25
+ if (blobs && client) {
26
+ client.addCoreBackground(blobs.core, { keys: opts.blindPeerKeys, pick: 2 });
27
+ }
28
+ }, noop);
39
29
  }
40
- async function acquire(key) {
30
+ function acquire(key) {
41
31
  const cached = drives.get(key);
42
32
  if (cached) {
43
33
  return cached;
44
34
  }
45
- const opening = withDeadline((async () => {
46
- const held = await openDrive(namespaced, opts.swarm, key);
47
- await peer(held);
48
- return held;
49
- })(), DRIVE_OPEN_TIMEOUT_MS);
35
+ const opening = openDrive(namespaced, opts.swarm, key);
50
36
  drives.set(key, opening);
51
- opening.catch(() => drives.delete(key));
37
+ opening.then(peer, () => drives.delete(key)).catch(noop);
52
38
  return opening;
53
39
  }
54
40
  async function opened() {
@@ -58,7 +44,9 @@ export function createFsDriveHost(store, opts) {
58
44
  .map(({ value }) => value);
59
45
  }
60
46
  async function startBlindPeering() {
61
- await Promise.all((await opened()).map(peer));
47
+ for (const held of await opened()) {
48
+ peer(held);
49
+ }
62
50
  }
63
51
  async function close() {
64
52
  const held = await opened();
@@ -20,6 +20,7 @@ export interface MeshInvite {
20
20
  readonly workspace?: boolean;
21
21
  readonly workspaceName?: string;
22
22
  readonly publicHeads?: ReadonlyArray<QvacInvitePublicHead>;
23
+ readonly inviterHeads?: ReadonlyArray<QvacAutobeeHead>;
23
24
  }
24
25
  export declare function encodeMeshInvite(invite: MeshInvite): string;
25
26
  export declare function decodeMeshInvite(raw: string): MeshInvite | null;
@@ -21,7 +21,8 @@ export function encodeMeshInvite(invite) {
21
21
  deviceProof: invite.deviceProof ?? null,
22
22
  workspace: invite.workspace ?? false,
23
23
  workspaceName: invite.workspaceName ?? null,
24
- publicHeads: [...(invite.publicHeads ?? [])]
24
+ publicHeads: [...(invite.publicHeads ?? [])],
25
+ inviterHeads: [...(invite.inviterHeads ?? [])]
25
26
  };
26
27
  const prefix = invite.memberInvite ? MEMBER_INVITE_PREFIX : DEVICE_INVITE_PREFIX;
27
28
  return prefix + z32.encode(encodeStruct(INVITE_STRUCT, payload));
@@ -56,6 +57,7 @@ export function decodeMeshInvite(raw) {
56
57
  deviceProof: payload.deviceProof ?? undefined,
57
58
  workspace: payload.workspace ?? false,
58
59
  workspaceName: payload.workspaceName ?? undefined,
59
- publicHeads: payload.publicHeads ?? []
60
+ publicHeads: payload.publicHeads ?? [],
61
+ inviterHeads: payload.inviterHeads ?? []
60
62
  };
61
63
  }
@@ -3,7 +3,7 @@ import type { LocalFile } from '../../../spec/local/hyperschema/types.d.ts';
3
3
  export type { LocalFile };
4
4
  export type KnownFile = Omit<LocalFile, 'lastAccessed' | 'cleared'>;
5
5
  export type FileKey = Pick<LocalFile, 'meshKey' | 'coreKey' | 'blockOffset'>;
6
- type Locked = <T>(fn: () => Promise<T>) => Promise<T>;
6
+ type Locked = <T>(op: string, fn: () => Promise<T>) => Promise<T>;
7
7
  export declare class LocalFiles {
8
8
  private readonly _db;
9
9
  private readonly _locked;