@qvac/core 0.9.0 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/CHANGELOG.md +112 -1
  2. package/README.md +4 -4
  3. package/dist-lib/index.d.ts +0 -1
  4. package/dist-lib/index.js +0 -1
  5. package/dist-lib/lib/api/agents.d.ts +7 -1
  6. package/dist-lib/lib/api/agents.js +2 -2
  7. package/dist-lib/lib/api/app-config.d.ts +2 -1
  8. package/dist-lib/lib/api/app-config.js +2 -2
  9. package/dist-lib/lib/api/attachments.js +3 -2
  10. package/dist-lib/lib/api/chat-move.d.ts +2 -1
  11. package/dist-lib/lib/api/chat-move.js +2 -2
  12. package/dist-lib/lib/api/chat.d.ts +3 -1
  13. package/dist-lib/lib/api/chat.js +5 -5
  14. package/dist-lib/lib/api/chunks.d.ts +2 -1
  15. package/dist-lib/lib/api/chunks.js +3 -3
  16. package/dist-lib/lib/api/config.d.ts +2 -1
  17. package/dist-lib/lib/api/config.js +2 -2
  18. package/dist-lib/lib/api/current-mesh.d.ts +1 -3
  19. package/dist-lib/lib/api/current-mesh.js +4 -10
  20. package/dist-lib/lib/api/custom-models.d.ts +7 -1
  21. package/dist-lib/lib/api/custom-models.js +10 -8
  22. package/dist-lib/lib/api/device-routes.d.ts +0 -2
  23. package/dist-lib/lib/api/device-routes.js +4 -5
  24. package/dist-lib/lib/api/devices.d.ts +3 -1
  25. package/dist-lib/lib/api/devices.js +9 -8
  26. package/dist-lib/lib/api/diagnostics.d.ts +2 -1
  27. package/dist-lib/lib/api/diagnostics.js +2 -2
  28. package/dist-lib/lib/api/index.d.ts +2 -0
  29. package/dist-lib/lib/api/index.js +39 -21
  30. package/dist-lib/lib/api/instrument.d.ts +2 -2
  31. package/dist-lib/lib/api/instrument.js +10 -11
  32. package/dist-lib/lib/api/knowledge.d.ts +3 -1
  33. package/dist-lib/lib/api/knowledge.js +9 -7
  34. package/dist-lib/lib/api/logs.js +3 -3
  35. package/dist-lib/lib/api/members.d.ts +6 -1
  36. package/dist-lib/lib/api/members.js +11 -10
  37. package/dist-lib/lib/api/mesh.d.ts +2 -2
  38. package/dist-lib/lib/api/mesh.js +6 -6
  39. package/dist-lib/lib/api/model-download.d.ts +2 -1
  40. package/dist-lib/lib/api/model-download.js +2 -2
  41. package/dist-lib/lib/api/network.d.ts +3 -2
  42. package/dist-lib/lib/api/network.js +4 -4
  43. package/dist-lib/lib/api/notifications.d.ts +3 -1
  44. package/dist-lib/lib/api/notifications.js +5 -4
  45. package/dist-lib/lib/api/rag.d.ts +0 -1
  46. package/dist-lib/lib/api/rag.js +3 -4
  47. package/dist-lib/lib/api/sync.d.ts +2 -1
  48. package/dist-lib/lib/api/sync.js +2 -2
  49. package/dist-lib/lib/api/tool-activity.d.ts +2 -1
  50. package/dist-lib/lib/api/tool-activity.js +4 -3
  51. package/dist-lib/lib/api/tool-configs.d.ts +2 -1
  52. package/dist-lib/lib/api/tool-configs.js +4 -3
  53. package/dist-lib/lib/api/upload-frames.d.ts +0 -1
  54. package/dist-lib/lib/api/upload-frames.js +0 -1
  55. package/dist-lib/lib/api/watchable.d.ts +3 -2
  56. package/dist-lib/lib/api/watchable.js +2 -3
  57. package/dist-lib/lib/assistant/fake-models.d.ts +2 -4
  58. package/dist-lib/lib/assistant/harness.js +8 -6
  59. package/dist-lib/lib/assistant-client.d.ts +2 -0
  60. package/dist-lib/lib/assistant-client.js +4 -3
  61. package/dist-lib/lib/assistant-tree.d.ts +3 -3
  62. package/dist-lib/lib/assistant.d.ts +5 -3
  63. package/dist-lib/lib/assistant.js +50 -13
  64. package/dist-lib/lib/client.d.ts +2 -2
  65. package/dist-lib/lib/core/agent.js +5 -4
  66. package/dist-lib/lib/core/archived-mesh.d.ts +2 -3
  67. package/dist-lib/lib/core/archived-mesh.js +3 -3
  68. package/dist-lib/lib/core/chat-activity.js +5 -3
  69. package/dist-lib/lib/core/chat-copy.d.ts +0 -1
  70. package/dist-lib/lib/core/chat-copy.js +2 -2
  71. package/dist-lib/lib/core/chat-fork.d.ts +0 -6
  72. package/dist-lib/lib/core/chat-fork.js +11 -17
  73. package/dist-lib/lib/core/chat-group.js +8 -7
  74. package/dist-lib/lib/core/chat-move.d.ts +4 -6
  75. package/dist-lib/lib/core/chat-move.js +14 -15
  76. package/dist-lib/lib/core/chat-order.d.ts +2 -1
  77. package/dist-lib/lib/core/chat-order.js +15 -2
  78. package/dist-lib/lib/core/chat.js +8 -7
  79. package/dist-lib/lib/core/chunk.js +13 -8
  80. package/dist-lib/lib/core/collections.d.ts +1 -0
  81. package/dist-lib/lib/core/collections.js +1 -0
  82. package/dist-lib/lib/core/contract/compute.d.ts +0 -4
  83. package/dist-lib/lib/core/contract/compute.js +2 -9
  84. package/dist-lib/lib/core/contract/types.d.ts +2 -2
  85. package/dist-lib/lib/core/departures.d.ts +2 -2
  86. package/dist-lib/lib/core/device-control.d.ts +0 -2
  87. package/dist-lib/lib/core/device-control.js +5 -6
  88. package/dist-lib/lib/core/files/attachments.js +5 -4
  89. package/dist-lib/lib/core/files/index.d.ts +2 -1
  90. package/dist-lib/lib/core/files/index.js +20 -8
  91. package/dist-lib/lib/core/files/processing/claims.js +3 -3
  92. package/dist-lib/lib/core/files/processing/document.d.ts +2 -0
  93. package/dist-lib/lib/core/files/processing/document.js +3 -0
  94. package/dist-lib/lib/core/files/processing/index.js +4 -2
  95. package/dist-lib/lib/core/files/processing/source.d.ts +3 -0
  96. package/dist-lib/lib/core/files/processing/source.js +5 -3
  97. package/dist-lib/lib/core/files/processing/state.js +1 -0
  98. package/dist-lib/lib/core/files/rag/index.d.ts +3 -0
  99. package/dist-lib/lib/core/files/rag/index.js +3 -1
  100. package/dist-lib/lib/core/files/rag/load.js +6 -3
  101. package/dist-lib/lib/core/files/rag/produce.js +3 -3
  102. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +1 -1
  103. package/dist-lib/lib/core/files/rag/similarity.d.ts +2 -2
  104. package/dist-lib/lib/core/files/rag/similarity.js +2 -3
  105. package/dist-lib/lib/core/files/records/chunks.js +1 -1
  106. package/dist-lib/lib/core/files/records/knowledge.js +2 -1
  107. package/dist-lib/lib/core/files/records/purge.js +9 -5
  108. package/dist-lib/lib/core/files/records/reclaim.js +20 -48
  109. package/dist-lib/lib/core/files/records/sweep.d.ts +3 -0
  110. package/dist-lib/lib/core/files/records/sweep.js +9 -3
  111. package/dist-lib/lib/core/files/store/blob.d.ts +2 -0
  112. package/dist-lib/lib/core/files/store/blob.js +2 -2
  113. package/dist-lib/lib/core/files/store/index.d.ts +1 -0
  114. package/dist-lib/lib/core/files/store/index.js +2 -0
  115. package/dist-lib/lib/core/files/store/reference.js +4 -2
  116. package/dist-lib/lib/core/files/upload.d.ts +1 -1
  117. package/dist-lib/lib/core/files/upload.js +10 -9
  118. package/dist-lib/lib/core/handoff-channel.d.ts +6 -4
  119. package/dist-lib/lib/core/handoff-channel.js +8 -9
  120. package/dist-lib/lib/core/harness-loan.d.ts +0 -2
  121. package/dist-lib/lib/core/harness-loan.js +3 -4
  122. package/dist-lib/lib/core/hosted-agents.d.ts +2 -2
  123. package/dist-lib/lib/core/inference-calls-handoff.d.ts +2 -2
  124. package/dist-lib/lib/core/inference-calls-handoff.js +6 -5
  125. package/dist-lib/lib/core/inference-calls-receiver.d.ts +0 -2
  126. package/dist-lib/lib/core/inference-calls-receiver.js +5 -7
  127. package/dist-lib/lib/core/local/index.d.ts +4 -1
  128. package/dist-lib/lib/core/local/index.js +23 -2
  129. package/dist-lib/lib/core/membership.d.ts +2 -2
  130. package/dist-lib/lib/core/mesh.d.ts +13 -7
  131. package/dist-lib/lib/core/mesh.js +55 -47
  132. package/dist-lib/lib/core/migrations/file-index.js +2 -4
  133. package/dist-lib/lib/core/next-frame.js +2 -1
  134. package/dist-lib/lib/core/pdf/rasterize.d.ts +2 -1
  135. package/dist-lib/lib/core/pdf/rasterize.js +5 -5
  136. package/dist-lib/lib/core/peer-duplex.d.ts +3 -3
  137. package/dist-lib/lib/core/peer-duplex.js +4 -4
  138. package/dist-lib/lib/core/secret-handoff.d.ts +2 -2
  139. package/dist-lib/lib/core/secret-handoff.js +7 -4
  140. package/dist-lib/lib/core/send.js +3 -2
  141. package/dist-lib/lib/core/skill-overlay-handoff.d.ts +2 -2
  142. package/dist-lib/lib/core/skill-overlay-handoff.js +6 -5
  143. package/dist-lib/lib/core/skill-overlay-receiver.js +5 -5
  144. package/dist-lib/lib/core/skill-secrets.d.ts +0 -5
  145. package/dist-lib/lib/core/skill-secrets.js +5 -9
  146. package/dist-lib/lib/core/sync/changed.d.ts +20 -0
  147. package/dist-lib/lib/core/sync/changed.js +7 -0
  148. package/dist-lib/lib/core/sync/devices.js +1 -1
  149. package/dist-lib/lib/core/sync/files.d.ts +1 -0
  150. package/dist-lib/lib/core/sync/files.js +50 -72
  151. package/dist-lib/lib/core/sync/index.d.ts +1 -2
  152. package/dist-lib/lib/core/sync/index.js +40 -62
  153. package/dist-lib/lib/core/sync/view-updates.d.ts +33 -0
  154. package/dist-lib/lib/core/sync/view-updates.js +133 -0
  155. package/dist-lib/lib/core/system-info.d.ts +2 -2
  156. package/dist-lib/lib/core/system-info.js +10 -11
  157. package/dist-lib/lib/core/tool-configs.js +6 -6
  158. package/dist-lib/lib/core.d.ts +1 -7
  159. package/dist-lib/lib/core.js +147 -131
  160. package/dist-lib/lib/custom-model-pusher.d.ts +2 -4
  161. package/dist-lib/lib/custom-model-task.d.ts +2 -1
  162. package/dist-lib/lib/custom-model-task.js +2 -2
  163. package/dist-lib/lib/downloads-allowed-pusher.d.ts +2 -4
  164. package/dist-lib/lib/errors.d.ts +603 -0
  165. package/dist-lib/lib/errors.js +552 -0
  166. package/dist-lib/lib/log.d.ts +7 -6
  167. package/dist-lib/lib/log.js +10 -6
  168. package/dist-lib/lib/logger-type.d.ts +2 -5
  169. package/dist-lib/lib/model-download-status.d.ts +2 -2
  170. package/dist-lib/lib/model-download.d.ts +2 -2
  171. package/dist-lib/lib/model-download.js +3 -2
  172. package/dist-lib/lib/model-idle-pusher.d.ts +2 -4
  173. package/dist-lib/lib/model-memory-cap.d.ts +2 -4
  174. package/dist-lib/lib/oauth/browser-connect.js +3 -2
  175. package/dist-lib/lib/oauth/flow-core.js +3 -3
  176. package/dist-lib/lib/oauth/loopback.js +10 -9
  177. package/dist-lib/lib/oauth/providers.js +4 -2
  178. package/dist-lib/lib/oauth/register.js +3 -2
  179. package/dist-lib/lib/oauth/token-exchange-core.js +5 -4
  180. package/dist-lib/lib/run-notices.d.ts +2 -0
  181. package/dist-lib/lib/run-notices.js +7 -2
  182. package/dist-lib/lib/run-transcript.d.ts +2 -2
  183. package/dist-lib/lib/task.js +2 -1
  184. package/dist-lib/schema/engine.js +6 -3
  185. package/dist-lib/schema/field-order.js +1 -3
  186. package/dist-lib/schema/local.js +11 -0
  187. package/dist-lib/spec/hrpc/messages.js +20 -9
  188. package/dist-lib/spec/hyperschema/index.js +20 -9
  189. package/dist-lib/spec/hyperschema/schema.json +15 -0
  190. package/dist-lib/spec/hyperschema/types.d.ts +3 -0
  191. package/dist-lib/spec/local/hyperdb/db.json +16 -1
  192. package/dist-lib/spec/local/hyperdb/index.js +71 -2
  193. package/dist-lib/spec/local/hyperdb/messages.js +300 -143
  194. package/dist-lib/spec/local/hyperschema/index.js +82 -2
  195. package/dist-lib/spec/local/hyperschema/schema.json +43 -1
  196. package/dist-lib/spec/local/hyperschema/types.d.ts +12 -1
  197. package/package.json +10 -3
  198. package/tui/app.mjs +1 -1
  199. package/worker/assistant-sidecar.mjs +7 -0
  200. package/dist-lib/lib/log-levels.d.ts +0 -7
  201. package/dist-lib/lib/log-levels.js +0 -1
  202. package/dist-lib/lib/logger.d.ts +0 -5
  203. package/dist-lib/lib/logger.js +0 -18
@@ -82,7 +82,8 @@ export declare class Files extends EventEmitter {
82
82
  artifacts(id: string): Promise<QvacKnowledgeArtifact[]>;
83
83
  publish(input: Parameters<typeof addKnowledgeArtifact>[1]): Promise<QvacKnowledgeArtifact>;
84
84
  hasLiveSource(chatId: string, reference: string): Promise<boolean>;
85
- syncAttachment(chunk: QvacChunk): Promise<void>;
85
+ syncAttachments(chunks: ReadonlyArray<QvacChunk>): Promise<void>;
86
+ reclaimAttachments(chunks: ReadonlyArray<QvacChunk>): Promise<void>;
86
87
  syncKnowledge(row: QvacKnowledge): Promise<void>;
87
88
  syncArtifact(row: QvacKnowledgeArtifact): Promise<void>;
88
89
  reclaimPurged(): Promise<number>;
@@ -12,6 +12,7 @@ import { BlobStore, parseBlobReference } from './store/index.js';
12
12
  import { AbortedError } from '@qvac/harness/race-abort';
13
13
  import { RagProcessor } from './rag/index.js';
14
14
  import { PreviewProcessor, PREVIEW_VERSION, previewId } from './preview/index.js';
15
+ import { CoreError } from '../../errors.js';
15
16
  import { errorMessage } from '../../guards.js';
16
17
  import { ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS, CHUNKS_BY_CHAT, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE } from '../collections.js';
17
18
  export { fullTextBudgetChars, fullTextHits, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
@@ -153,9 +154,16 @@ export class Files extends EventEmitter {
153
154
  hasLiveSource(chatId, reference) {
154
155
  return hasLiveSource(this._mesh, chatId, reference);
155
156
  }
156
- async syncAttachment(chunk) {
157
- await this._mesh.fileIndex?.refresh([chunk]);
158
- await syncAttachmentChunk(this._mesh, chunk);
157
+ async syncAttachments(chunks) {
158
+ await this._mesh.fileIndex?.refresh(chunks);
159
+ await this.reclaimAttachments(chunks);
160
+ }
161
+ async reclaimAttachments(chunks) {
162
+ for (const chunk of chunks) {
163
+ if (this._mesh.closing)
164
+ return;
165
+ await syncAttachmentChunk(this._mesh, chunk);
166
+ }
159
167
  }
160
168
  syncKnowledge(row) {
161
169
  return syncKnowledgeTombstone(this._mesh, row);
@@ -218,7 +226,8 @@ export class Files extends EventEmitter {
218
226
  this.setModel(options.embedder.model);
219
227
  this.startProcessor(new RagProcessor({
220
228
  ...options,
221
- model: options.embedder.model
229
+ model: options.embedder.model,
230
+ log: this._mesh.log
222
231
  }), { deviceId: options.deviceId });
223
232
  }
224
233
  async stopProcessing() {
@@ -348,13 +357,13 @@ export class Files extends EventEmitter {
348
357
  async beginUpload(input) {
349
358
  const { chatId, fileName, mimeType } = input;
350
359
  if (!chatId || !fileName || !mimeType) {
351
- throw new Error('upload metadata is incomplete');
360
+ throw new CoreError('UPLOAD_METADATA_INCOMPLETE');
352
361
  }
353
362
  const id = input.fileId || randomId();
354
363
  this._pruneUploads();
355
364
  const previous = this._uploads.get(id);
356
365
  if (previous && !previous.expiresAt) {
357
- throw new Error('file id already exists');
366
+ throw new CoreError('FILE_ID_EXISTS');
358
367
  }
359
368
  this._uploads.set(id, {
360
369
  status: {
@@ -377,7 +386,7 @@ export class Files extends EventEmitter {
377
386
  ((await getKnowledge(this._mesh, id)) ||
378
387
  (await getChunk(this._mesh, id)) ||
379
388
  (await listChunks(this._mesh, { chatId, type: 'attachment' })).some((chunk) => chunk.attachment?.id === id))) {
380
- throw new Error('file id already exists');
389
+ throw new CoreError('FILE_ID_EXISTS');
381
390
  }
382
391
  }
383
392
  catch (error) {
@@ -512,7 +521,7 @@ export class Files extends EventEmitter {
512
521
  else if (local?.work &&
513
522
  !published &&
514
523
  local.work.state !== 'running' &&
515
- (!terminalFailure || local.work.state !== 'pending')) {
524
+ (!terminalFailure || !yieldsToVerdict(local.work))) {
516
525
  Object.assign(processor, local.work);
517
526
  }
518
527
  else if (local?.load && published) {
@@ -636,6 +645,9 @@ export class Files extends EventEmitter {
636
645
  await this._store.close();
637
646
  }
638
647
  }
648
+ function yieldsToVerdict(work) {
649
+ return work.state === 'pending' || (work.state === 'failed' && work.retryAt === undefined);
650
+ }
639
651
  function fileState(processor, opts) {
640
652
  if (opts.readyForChat) {
641
653
  return { state: 'ready', reason: undefined };
@@ -42,7 +42,7 @@ export class IndexingClaims {
42
42
  !(await getIndexingStatus(this._mesh, artifactId))) {
43
43
  return { claimId: '' };
44
44
  }
45
- logger.debug('rag', 'claim not placed', { id: artifactId, err });
45
+ this._mesh.log?.debug('rag', 'claim not placed', { id: artifactId, err });
46
46
  return null;
47
47
  }
48
48
  }
@@ -87,11 +87,11 @@ export class IndexingClaims {
87
87
  status: 'ready'
88
88
  });
89
89
  if (applied) {
90
- logger.info('rag', 'repaired to ready', { id: artifactId });
90
+ this._mesh.log?.info('rag', 'repaired to ready', { id: artifactId });
91
91
  }
92
92
  }
93
93
  catch (err) {
94
- logger.debug('rag', 'repair resolve dropped', { id: artifactId, err });
94
+ this._mesh.log?.debug('rag', 'repair resolve dropped', { id: artifactId, err });
95
95
  }
96
96
  }
97
97
  }
@@ -1,3 +1,4 @@
1
+ import type { LoggerInterface } from '@qvac/logging';
1
2
  import type { AbortSignal } from 'bare-abort-controller';
2
3
  import { type IndexingProgress } from '../rag/index.ts';
3
4
  import { IndexingClaims } from './claims.ts';
@@ -42,6 +43,7 @@ export declare class FileDocument {
42
43
  private readonly _progress?;
43
44
  private readonly _processor;
44
45
  constructor(mesh: Mesh, options: FileDocumentOptions);
46
+ get log(): LoggerInterface | null;
45
47
  get localCoreKey(): string;
46
48
  artifact(id: string): import("./state.ts").ArtifactState;
47
49
  readArtifact(id: string): Readonly<import("./state.ts").ArtifactState> | undefined;
@@ -45,6 +45,9 @@ export class FileDocument {
45
45
  };
46
46
  this.ownership = new ArtifactOwnership(mesh, state);
47
47
  }
48
+ get log() {
49
+ return this._mesh.log;
50
+ }
48
51
  get localCoreKey() {
49
52
  return b4a.toHex(this._mesh.blobs.core.key);
50
53
  }
@@ -38,11 +38,13 @@ export class FileProcessing {
38
38
  const readBytes = mesh.files.readBytes.bind(mesh.files);
39
39
  const fetch = mesh.files.download.bind(mesh.files);
40
40
  this._source = new IndexingSource(readBytes, {
41
+ log: mesh.log,
41
42
  retryMs: options.retryMs,
42
43
  retries: mesh.files.processorState(options.processor.id).sourceRetries,
43
44
  download: { fetch, ready: (id) => this._schedule(id, Date.now()) }
44
45
  });
45
46
  this._output = new IndexingSource(readBytes, {
47
+ log: mesh.log,
46
48
  retryMs: options.retryMs,
47
49
  retries: mesh.files.processorState(options.processor.id).outputRetries,
48
50
  download: {
@@ -190,7 +192,7 @@ export class FileProcessing {
190
192
  await this.enqueue(await this._documents());
191
193
  }
192
194
  catch (err) {
193
- logger.warn('rag', 'file processing scan failed', { err });
195
+ this._mesh.log?.warn('rag', 'file processing scan failed', { err });
194
196
  }
195
197
  }
196
198
  }
@@ -292,7 +294,7 @@ export class FileProcessing {
292
294
  else {
293
295
  this._mesh.files.work(id, { state: 'failed', reason }, this._processor.id);
294
296
  }
295
- logger.warn('rag', 'file processing failed', { id, stage, err });
297
+ this._mesh.log?.warn('rag', 'file processing failed', { id, stage, err });
296
298
  }
297
299
  _queueLoad(id) {
298
300
  const loader = this._lanes.find((entry) => entry.stage === 'load');
@@ -1,3 +1,4 @@
1
+ import type { LoggerInterface } from '@qvac/logging';
1
2
  import { type BlobStore } from '../store/index.ts';
2
3
  import { type AbortSignal } from 'bare-abort-controller';
3
4
  import type { FileRetries } from './state.ts';
@@ -11,6 +12,7 @@ export type SourceRead = {
11
12
  kind: 'malformed';
12
13
  };
13
14
  export interface IndexingSourceOptions {
15
+ readonly log: LoggerInterface | null;
14
16
  readonly retryMs?: number;
15
17
  readonly retries: FileRetries;
16
18
  readonly download?: {
@@ -20,6 +22,7 @@ export interface IndexingSourceOptions {
20
22
  }
21
23
  export declare class IndexingSource {
22
24
  private readonly _readBytes;
25
+ private readonly _log;
23
26
  private readonly _retryMs;
24
27
  private readonly _download;
25
28
  private readonly _downloads;
@@ -4,12 +4,14 @@ import { AbortedError } from '@qvac/harness/race-abort';
4
4
  const BLOB_RETRY_MS = 30_000;
5
5
  export class IndexingSource {
6
6
  _readBytes;
7
+ _log;
7
8
  _retryMs;
8
9
  _download;
9
10
  _downloads = new Map();
10
11
  retry;
11
12
  constructor(readBytes, opts) {
12
13
  this._readBytes = readBytes;
14
+ this._log = opts.log;
13
15
  this._retryMs = opts.retryMs ?? BLOB_RETRY_MS;
14
16
  this.retry = opts.retries;
15
17
  this._download = opts.download;
@@ -40,10 +42,10 @@ export class IndexingSource {
40
42
  throw new AbortedError();
41
43
  }
42
44
  if (isInvalidBlobReferenceError(err)) {
43
- logger.warn('rag', 'blob ref is malformed, never readable', { id });
45
+ this._log?.warn('rag', 'blob ref is malformed, never readable', { id });
44
46
  return { kind: 'malformed' };
45
47
  }
46
- logger.debug('rag', 'blob bytes unavailable', { id, err });
48
+ this._log?.debug('rag', 'blob bytes unavailable', { id, err });
47
49
  this.retry.markMissing(id);
48
50
  const retryAt = this.retryMiss(id, Date.now());
49
51
  this._fetch(id, blobRef);
@@ -91,7 +93,7 @@ export class IndexingSource {
91
93
  download.ready(id);
92
94
  }, (err) => {
93
95
  if (!abort.signal.aborted) {
94
- logger.debug('rag', 'blob download interrupted', { id, err });
96
+ this._log?.debug('rag', 'blob download interrupted', { id, err });
95
97
  }
96
98
  })
97
99
  .finally(() => {
@@ -109,6 +109,7 @@ export class FileState {
109
109
  output.rebuildFailures = undefined;
110
110
  this.outputRetries.clear(artifactId);
111
111
  }
112
+ entry.artifactIdMemo = undefined;
112
113
  }
113
114
  reset() {
114
115
  this._documents.clear();
@@ -1,3 +1,4 @@
1
+ import type { LoggerInterface } from '@qvac/logging';
1
2
  import type { AbortSignal } from 'bare-abort-controller';
2
3
  import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts';
3
4
  import { type RagExtractors, type VectorIndex } from 'rag-extract';
@@ -9,6 +10,7 @@ export interface RagProcessorOptions {
9
10
  readonly embedder?: IndexingEmbedder;
10
11
  readonly index?: VectorIndex | null;
11
12
  readonly extractors?: RagExtractors;
13
+ readonly log: LoggerInterface | null;
12
14
  }
13
15
  export declare class RagProcessor implements FileProcessor {
14
16
  readonly id = "rag";
@@ -18,6 +20,7 @@ export declare class RagProcessor implements FileProcessor {
18
20
  private readonly _producer;
19
21
  readonly load?: (document: FileDocument, signal: AbortSignal) => Promise<void>;
20
22
  private readonly _index;
23
+ private readonly _log;
21
24
  constructor(options: RagProcessorOptions);
22
25
  applies(_document: QvacKnowledge): boolean;
23
26
  canProduce(document: QvacKnowledge, host: ProcessorHost): Promise<boolean>;
@@ -12,9 +12,11 @@ export class RagProcessor {
12
12
  _producer;
13
13
  load;
14
14
  _index;
15
+ _log;
15
16
  constructor(options) {
16
17
  this.version = pipelineVersionFor(options.model);
17
18
  this._index = options.index ?? null;
19
+ this._log = options.log;
18
20
  const lock = new ScopeLock();
19
21
  const source = options.embedder;
20
22
  const embedder = source && {
@@ -61,7 +63,7 @@ export class RagProcessor {
61
63
  return true;
62
64
  }
63
65
  catch (err) {
64
- logger.warn('rag', 'vector index clear failed on close; it must not be reused', { err });
66
+ this._log?.warn('rag', 'vector index clear failed on close; it must not be reused', { err });
65
67
  return false;
66
68
  }
67
69
  }
@@ -125,7 +125,10 @@ export class HydratePass {
125
125
  }
126
126
  catch (err) {
127
127
  document.artifact(pick.id).failedReference = pick.artifactBlobId;
128
- logger.warn('rag', 'artifact undecodable, pinned for the session', { id: pick.id, err });
128
+ document.log?.warn('rag', 'artifact undecodable, pinned for the session', {
129
+ id: pick.id,
130
+ err
131
+ });
129
132
  continue;
130
133
  }
131
134
  if (document.paused() || document.closed()) {
@@ -140,7 +143,7 @@ export class HydratePass {
140
143
  }
141
144
  catch (err) {
142
145
  document.artifact(pick.id).failedReference = pick.artifactBlobId;
143
- logger.warn('rag', 'artifact rejected by the vector index', { id: pick.id, err });
146
+ document.log?.warn('rag', 'artifact rejected by the vector index', { id: pick.id, err });
144
147
  continue;
145
148
  }
146
149
  const evict = entry && entry.artifactId !== pick.id ? entry.artifactId : undefined;
@@ -154,7 +157,7 @@ export class HydratePass {
154
157
  await this._index.remove(evict);
155
158
  local.loaded = { artifactId: pick.id, blobRef: pick.artifactBlobId };
156
159
  }
157
- logger.info('rag', 'hydrated knowledge into the vector index', {
160
+ document.log?.info('rag', 'hydrated knowledge into the vector index', {
158
161
  artifactId: pick.id,
159
162
  chunks: chunks.length
160
163
  });
@@ -91,7 +91,7 @@ export class IndexingPass {
91
91
  if (kind === RAG_DOCUMENT_KIND.office && !this._extractors.office) {
92
92
  local.artifactIdMemo = UNSUPPORTED;
93
93
  document.work({
94
- state: 'pending',
94
+ state: 'failed',
95
95
  reason: 'extractor unavailable on this device'
96
96
  });
97
97
  return;
@@ -170,7 +170,7 @@ export class IndexingPass {
170
170
  processedChunks: payload.chunks.length,
171
171
  totalChunks: payload.chunks.length
172
172
  });
173
- logger.info('rag', applied ? 'indexed' : 'indexed, claim superseded', {
173
+ document.log?.info('rag', applied ? 'indexed' : 'indexed, claim superseded', {
174
174
  id: artifactId,
175
175
  chunks: payload.chunks.length
176
176
  });
@@ -220,7 +220,7 @@ export class IndexingPass {
220
220
  error: message
221
221
  });
222
222
  }
223
- logger.warn('rag', 'indexing failed', { id: artifactId, error: message });
223
+ document.log?.warn('rag', 'indexing failed', { id: artifactId, error: message });
224
224
  }
225
225
  }
226
226
  }
@@ -92,7 +92,7 @@ export async function fullTextHits(mesh, rows, { budgetChars, head = false }) {
92
92
  });
93
93
  }
94
94
  catch (err) {
95
- logger.warn('rag', 'full-text extraction failed; falling back to retrieval', { err });
95
+ mesh.log?.warn('rag', 'full-text extraction failed; falling back to retrieval', { err });
96
96
  return null;
97
97
  }
98
98
  if (!read.text) {
@@ -1,5 +1,5 @@
1
- import '../../../logger.ts';
1
+ import type { LoggerInterface } from '@qvac/logging';
2
2
  import { applySimilarityFloor, RAG_MIN_SIMILARITY } from 'rag-extract/retrieval';
3
3
  import type { VectorHit } from 'rag-extract/vector';
4
4
  export { applySimilarityFloor, RAG_MIN_SIMILARITY };
5
- export declare function logSimilarityScores(candidates: ReadonlyArray<VectorHit>, kept: ReadonlyArray<VectorHit>): void;
5
+ export declare function logSimilarityScores(candidates: ReadonlyArray<VectorHit>, kept: ReadonlyArray<VectorHit>, log?: LoggerInterface | null): void;
@@ -1,6 +1,5 @@
1
- import '../../../logger.js';
2
1
  import { applySimilarityFloor, logSimilarityScores as traceSimilarityScores, RAG_MIN_SIMILARITY } from 'rag-extract/retrieval';
3
2
  export { applySimilarityFloor, RAG_MIN_SIMILARITY };
4
- export function logSimilarityScores(candidates, kept) {
5
- traceSimilarityScores(candidates, kept, (message) => logger.debug('rag', message));
3
+ export function logSimilarityScores(candidates, kept, log) {
4
+ traceSimilarityScores(candidates, kept, (message) => log?.debug('rag', message));
6
5
  }
@@ -9,7 +9,7 @@ export async function syncAttachmentChunk(mesh, chunk) {
9
9
  if (await hasLiveSource(mesh, chunk.chatId, attachment.blobId)) {
10
10
  return;
11
11
  }
12
- logger.debug('sync', 'attachment tombstone observed — reclaiming blob bytes', { id: chunk.id });
12
+ mesh.log?.debug('sync', 'attachment tombstone observed - reclaiming blob bytes', { id: chunk.id });
13
13
  await mesh.files.clear(attachment.blobId, chunk.id);
14
14
  await mesh.fileIndex?.markCleared(attachment.blobId);
15
15
  }
@@ -1,3 +1,4 @@
1
+ import { CoreError } from '../../../errors.js';
1
2
  import { randomId } from '../../id.js';
2
3
  import { assertArtifactOutput } from './output.js';
3
4
  import { ACTIVE_KNOWLEDGE_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE } from '../../collections.js';
@@ -14,7 +15,7 @@ export async function getKnowledge(store, id) {
14
15
  async function mustGetKnowledge(store, id) {
15
16
  const row = await getKnowledge(store, id);
16
17
  if (!row) {
17
- throw new Error(`knowledge not found: ${id}`);
18
+ throw new CoreError('KNOWLEDGE_NOT_FOUND', [id]);
18
19
  }
19
20
  return row;
20
21
  }
@@ -8,7 +8,10 @@ export async function reclaimPurgedFiles(mesh) {
8
8
  return 0;
9
9
  }
10
10
  const rows = await index.list();
11
- const purged = await purgedChatIds(mesh, rows);
11
+ const purged = await purgedChats(mesh, rows);
12
+ if (purged.size) {
13
+ await index.retireChats(purged);
14
+ }
12
15
  const candidates = rows.filter((row) => !!row.chatId && purged.has(row.chatId));
13
16
  if (candidates.length === 0) {
14
17
  return 0;
@@ -80,12 +83,13 @@ export async function reclaimReferences(mesh, references) {
80
83
  void mesh.files.compact([...cores.values()]);
81
84
  }
82
85
  }
83
- async function purgedChatIds(mesh, rows) {
84
- const purged = new Set();
86
+ async function purgedChats(mesh, rows) {
87
+ const purged = new Map();
85
88
  const chatIds = rows.map((row) => row.chatId).filter((id) => !!id);
86
89
  for (const chatId of new Set(chatIds)) {
87
- if (await mesh.view.get(PURGED_CHATS, { chatId })) {
88
- purged.add(chatId);
90
+ const row = await mesh.view.get(PURGED_CHATS, { chatId });
91
+ if (row) {
92
+ purged.set(chatId, row.at);
89
93
  }
90
94
  }
91
95
  return purged;
@@ -2,12 +2,11 @@ import { hasLiveSource } from './routing.js';
2
2
  import { removeChatAttachments, removeChatKnowledge } from '../../chat.js';
3
3
  import { listChunks } from '../../chunk.js';
4
4
  import { getKnowledge, listKnowledgeArtifactsByKnowledge } from './knowledge.js';
5
- import { syncAttachmentChunk } from './chunks.js';
6
- import { CHATS, CHUNK_COUNTERS, KNOWLEDGE, PURGED_CHATS } from '../../collections.js';
5
+ import { CHATS, KNOWLEDGE, PURGED_CHATS } from '../../collections.js';
7
6
  export async function syncKnowledgeTombstone(mesh, row) {
8
7
  if (!row.deletedAt)
9
8
  return;
10
- logger.debug('sync', 'knowledge tombstone observed - reclaiming blob bytes', { id: row.id });
9
+ mesh.log?.debug('sync', 'knowledge tombstone observed - reclaiming blob bytes', { id: row.id });
11
10
  if (typeof row.blobId === 'string' && !(await hasLiveSource(mesh, row.chatId, row.blobId))) {
12
11
  await mesh.files.clear(row.blobId, row.id);
13
12
  await mesh.fileIndex?.markCleared(row.blobId);
@@ -26,67 +25,40 @@ export async function syncArtifactArrival(mesh, row) {
26
25
  await mesh.files.clear(row.artifactBlobId, row.id);
27
26
  }
28
27
  export async function sweepTombstonedBlobs(mesh, deviceId = null) {
29
- if (deviceId && mesh.writable) {
30
- const finished = await recascadeTombstonedChats(mesh, deviceId);
31
- if (!finished) {
28
+ const held = (await mesh.fileIndex?.list()) ?? [];
29
+ const chatIds = new Set(held.map((row) => row.chatId).filter((id) => !!id));
30
+ for (const chatId of chatIds) {
31
+ if (mesh.closing)
32
32
  return;
33
+ if (deviceId && mesh.writable) {
34
+ await recascadeTombstonedChat(mesh, chatId, deviceId);
33
35
  }
36
+ await mesh.files.syncAttachments(await listChunks(mesh, { chatId, type: 'attachment' }));
34
37
  }
35
- const knowledge = mesh.view.snapshot();
38
+ const snap = mesh.view.snapshot();
36
39
  try {
37
- for await (const row of knowledge.find(KNOWLEDGE)) {
40
+ for await (const row of snap.find(KNOWLEDGE)) {
38
41
  if (mesh.closing)
39
42
  return;
40
43
  await syncKnowledgeTombstone(mesh, row);
41
44
  }
42
45
  }
43
46
  finally {
44
- await knowledge.close();
45
- }
46
- const counters = mesh.view.snapshot();
47
- try {
48
- for await (const counter of counters.find(CHUNK_COUNTERS)) {
49
- if (mesh.closing)
50
- return;
51
- const finished = await syncChatAttachmentChunks(mesh, counter.chatId);
52
- if (!finished) {
53
- return;
54
- }
55
- }
56
- }
57
- finally {
58
- await counters.close();
47
+ await snap.close();
59
48
  }
60
49
  }
61
- async function recascadeTombstonedChats(mesh, deviceId) {
62
- const chats = mesh.view.snapshot();
50
+ async function recascadeTombstonedChat(mesh, chatId, deviceId) {
51
+ const chat = await mesh.view.get(CHATS, { id: chatId });
52
+ if (!chat?.deletedAt || (await mesh.view.get(PURGED_CHATS, { chatId }))) {
53
+ return;
54
+ }
63
55
  try {
64
- for await (const chat of chats.find(CHATS)) {
65
- if (mesh.closing) {
66
- return false;
67
- }
68
- if (!chat.deletedAt || (await chats.get(PURGED_CHATS, { chatId: chat.id }))) {
69
- continue;
70
- }
71
- await removeChatAttachments(mesh, chat.id, { deviceId, at: chat.deletedAt });
72
- await removeChatKnowledge(mesh, chat.id, chat.deletedAt);
73
- }
56
+ await removeChatAttachments(mesh, chatId, { deviceId, at: chat.deletedAt });
57
+ await removeChatKnowledge(mesh, chatId, chat.deletedAt);
74
58
  }
75
59
  catch (error) {
76
- logger.warn('sync', 'chat-tombstone sweep pass failed - continuing with local reclaims', {
60
+ mesh.log?.warn('sync', 'chat-tombstone sweep failed - continuing with local reclaims', {
77
61
  error
78
62
  });
79
63
  }
80
- finally {
81
- await chats.close();
82
- }
83
- return true;
84
- }
85
- async function syncChatAttachmentChunks(mesh, chatId) {
86
- for (const chunk of await listChunks(mesh, { chatId, type: 'attachment' })) {
87
- if (mesh.closing)
88
- return false;
89
- await mesh.files.syncAttachment(chunk);
90
- }
91
- return true;
92
64
  }
@@ -1,2 +1,5 @@
1
+ import type HyperDB from 'hyperdb';
2
+ import type { QvacChunk } from '../../../../spec/mesh/hyperschema/types.d.ts';
1
3
  import type { Mesh } from '../../mesh.ts';
2
4
  export declare function referencedBlobs(mesh: Mesh): Promise<string[]>;
5
+ export declare function attachmentChunks(snapshot: HyperDB): AsyncGenerator<QvacChunk>;
@@ -1,12 +1,13 @@
1
- import { CHUNKS, KNOWLEDGE, KNOWLEDGE_ARTIFACTS } from '../../collections.js';
1
+ import { findChunksOfType } from '../../chunk.js';
2
+ import { CHUNK_COUNTERS, KNOWLEDGE, KNOWLEDGE_ARTIFACTS } from '../../collections.js';
2
3
  export async function referencedBlobs(mesh) {
3
4
  const references = [];
4
5
  const snapshot = mesh.view.snapshot();
5
6
  try {
6
- for await (const chunk of snapshot.find(CHUNKS)) {
7
+ for await (const chunk of attachmentChunks(snapshot)) {
7
8
  if (mesh.closing)
8
9
  return references;
9
- if (chunk.type === 'attachment' && chunk.attachment) {
10
+ if (chunk.attachment) {
10
11
  references.push(chunk.attachment.blobId);
11
12
  }
12
13
  }
@@ -26,3 +27,8 @@ export async function referencedBlobs(mesh) {
26
27
  }
27
28
  return references;
28
29
  }
30
+ export async function* attachmentChunks(snapshot) {
31
+ for await (const { chatId } of snapshot.find(CHUNK_COUNTERS)) {
32
+ yield* await findChunksOfType(snapshot, { chatId, type: 'attachment' });
33
+ }
34
+ }
@@ -2,8 +2,10 @@ import type { BlobId } from 'hyperblobs';
2
2
  import type Hyperblobs from 'hyperblobs';
3
3
  import type { AbortSignal } from 'bare-abort-controller';
4
4
  import { type NextFrameOptions } from '../../next-frame.ts';
5
+ import type { LoggerInterface } from '@qvac/logging';
5
6
  export interface BlobHost {
6
7
  readonly blobs: Hyperblobs;
8
+ readonly log: LoggerInterface | null;
7
9
  openBlobs(key: Buffer): Promise<Hyperblobs>;
8
10
  }
9
11
  export type WriteProgress = {
@@ -87,7 +87,7 @@ export async function clearBlob(mesh, blobId, id) {
87
87
  return true;
88
88
  }
89
89
  catch (err) {
90
- logger.warn('sync', 'blob clear failed', { id, err });
90
+ mesh.log?.warn('sync', 'blob clear failed', { id, err });
91
91
  return false;
92
92
  }
93
93
  }
@@ -130,7 +130,7 @@ export async function compactBlobCores(mesh, coreKeys) {
130
130
  }
131
131
  }
132
132
  catch (err) {
133
- logger.warn('sync', 'blob compaction failed', { coreKey: b4a.toHex(key), err });
133
+ mesh.log?.warn('sync', 'blob compaction failed', { coreKey: b4a.toHex(key), err });
134
134
  }
135
135
  }
136
136
  }
@@ -11,6 +11,7 @@ export type { WriteProgress } from './blob.ts';
11
11
  export { MAX_INLINE_BYTES, MAX_REFERENCE_CHARS } from './inline.ts';
12
12
  export declare class BlobStore extends EventEmitter {
13
13
  readonly blobs: Hyperblobs;
14
+ readonly log: BlobHost['log'];
14
15
  readonly openBlobs: BlobHost['openBlobs'];
15
16
  private readonly _readers;
16
17
  private readonly _updated;
@@ -10,6 +10,7 @@ export { PENDING_BLOB_REFERENCE } from './reference.js';
10
10
  export { MAX_INLINE_BYTES, MAX_REFERENCE_CHARS } from './inline.js';
11
11
  export class BlobStore extends EventEmitter {
12
12
  blobs;
13
+ log;
13
14
  openBlobs;
14
15
  _readers = new Map();
15
16
  _updated = () => this.emit('update');
@@ -17,6 +18,7 @@ export class BlobStore extends EventEmitter {
17
18
  constructor(host) {
18
19
  super();
19
20
  this.blobs = host.blobs;
21
+ this.log = host.log;
20
22
  this.openBlobs = (key) => host.openBlobs(key);
21
23
  this.blobs.core.on('download', this._updated);
22
24
  }
@@ -2,12 +2,12 @@ import { readBlob } from './blob.js';
2
2
  import { AbortedError, raceAbort } from '@qvac/harness/race-abort';
3
3
  import { MAX_REFERENCE_CHARS, readInline } from './inline.js';
4
4
  import b4a from 'b4a';
5
+ import { CoreError } from '../../../errors.js';
5
6
  import { isRecord } from '../../../guards.js';
6
7
  import { PENDING_BLOB_REFERENCE } from '../../../constants.js';
7
8
  const CORE_KEY_PATTERN = /^[0-9a-f]{64}$/;
8
9
  const DEFAULT_REMOTE_TIMEOUT_MS = 15_000;
9
10
  const INVALID_BLOB_REFERENCE = 'invalid blob reference';
10
- const BLOB_UNAVAILABLE = 'blob unavailable';
11
11
  export { PENDING_BLOB_REFERENCE };
12
12
  export function isPendingBlobReference(blobReference) {
13
13
  return blobReference === PENDING_BLOB_REFERENCE;
@@ -19,7 +19,9 @@ export function isInvalidBlobReferenceError(error) {
19
19
  return error instanceof TypeError && error.message === INVALID_BLOB_REFERENCE;
20
20
  }
21
21
  function blobUnavailable(cause) {
22
- return new Error(BLOB_UNAVAILABLE, { cause });
22
+ return new CoreError('BLOB_UNAVAILABLE', [], {
23
+ cause: cause instanceof Error ? cause : undefined
24
+ });
23
25
  }
24
26
  function parseBlobReferenceRecord(blobReference) {
25
27
  if (blobReference.length > MAX_REFERENCE_CHARS) {