@qvac/core 0.6.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/CHANGELOG.md +112 -2
  2. package/dist-lib/lib/api/chat.d.ts +1 -1
  3. package/dist-lib/lib/api/chat.js +2 -1
  4. package/dist-lib/lib/api/index.d.ts +3 -1
  5. package/dist-lib/lib/api/index.js +2 -0
  6. package/dist-lib/lib/api/watchable.d.ts +1 -0
  7. package/dist-lib/lib/api/watchable.js +1 -1
  8. package/dist-lib/lib/assistant/harness.d.ts +2 -0
  9. package/dist-lib/lib/assistant/harness.js +13 -9
  10. package/dist-lib/lib/assistant-tree.js +4 -0
  11. package/dist-lib/lib/assistant.d.ts +5 -1
  12. package/dist-lib/lib/assistant.js +66 -20
  13. package/dist-lib/lib/client.js +2 -1
  14. package/dist-lib/lib/collabora/entry.js +13 -19
  15. package/dist-lib/lib/collabora/worker.js +77 -18
  16. package/dist-lib/lib/core/chat.d.ts +1 -0
  17. package/dist-lib/lib/core/chat.js +26 -4
  18. package/dist-lib/lib/core/collections.d.ts +1 -0
  19. package/dist-lib/lib/core/collections.js +1 -0
  20. package/dist-lib/lib/core/contract/chat.d.ts +2 -0
  21. package/dist-lib/lib/core/contract/chat.js +77 -4
  22. package/dist-lib/lib/core/contract/chunks.js +5 -0
  23. package/dist-lib/lib/core/contract/compute.d.ts +1 -1
  24. package/dist-lib/lib/core/contract/compute.js +2 -1
  25. package/dist-lib/lib/core/contract/tool-activity.js +5 -0
  26. package/dist-lib/lib/core/files/index.d.ts +2 -0
  27. package/dist-lib/lib/core/files/index.js +7 -1
  28. package/dist-lib/lib/core/files/local-index.d.ts +4 -1
  29. package/dist-lib/lib/core/files/processing/index.d.ts +3 -0
  30. package/dist-lib/lib/core/files/processing/index.js +9 -0
  31. package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
  32. package/dist-lib/lib/core/files/processing/state.js +5 -0
  33. package/dist-lib/lib/core/files/rag/index.d.ts +2 -0
  34. package/dist-lib/lib/core/files/rag/index.js +6 -0
  35. package/dist-lib/lib/core/files/records/index.d.ts +1 -0
  36. package/dist-lib/lib/core/files/records/index.js +1 -0
  37. package/dist-lib/lib/core/files/records/purge.d.ts +4 -0
  38. package/dist-lib/lib/core/files/records/purge.js +108 -0
  39. package/dist-lib/lib/core/files/records/reclaim.js +5 -3
  40. package/dist-lib/lib/core/files/store/blob.d.ts +1 -0
  41. package/dist-lib/lib/core/files/store/blob.js +29 -0
  42. package/dist-lib/lib/core/files/store/index.d.ts +1 -0
  43. package/dist-lib/lib/core/files/store/index.js +6 -1
  44. package/dist-lib/lib/core/local/files.d.ts +2 -0
  45. package/dist-lib/lib/core/local/files.js +31 -0
  46. package/dist-lib/lib/core/sync/index.js +14 -1
  47. package/dist-lib/lib/core.d.ts +12 -2
  48. package/dist-lib/lib/core.js +79 -16
  49. package/dist-lib/lib/credential-pusher.d.ts +8 -3
  50. package/dist-lib/lib/credential-pusher.js +4 -1
  51. package/dist-lib/lib/mesh-scoped-engine.js +3 -0
  52. package/dist-lib/lib/model-download.d.ts +11 -7
  53. package/dist-lib/lib/model-download.js +22 -21
  54. package/dist-lib/lib/models-watch.d.ts +6 -0
  55. package/dist-lib/lib/models-watch.js +45 -0
  56. package/dist-lib/schema/engine.d.ts +1 -0
  57. package/dist-lib/schema/engine.js +57 -3
  58. package/dist-lib/schema/field-order.js +3 -1
  59. package/dist-lib/schema/qvac.js +12 -1
  60. package/dist-lib/spec/bind.js +14 -0
  61. package/dist-lib/spec/calls.js +3 -0
  62. package/dist-lib/spec/capabilities.d.ts +6 -0
  63. package/dist-lib/spec/hrpc/hrpc.json +39 -0
  64. package/dist-lib/spec/hrpc/index.js +41 -4
  65. package/dist-lib/spec/hrpc/messages.js +788 -583
  66. package/dist-lib/spec/hyperschema/index.js +788 -583
  67. package/dist-lib/spec/hyperschema/schema.json +149 -0
  68. package/dist-lib/spec/hyperschema/types.d.ts +29 -0
  69. package/dist-lib/spec/mesh/hyperdb/db.json +16 -1
  70. package/dist-lib/spec/mesh/hyperdb/index.js +71 -2
  71. package/dist-lib/spec/mesh/hyperdb/messages.js +434 -227
  72. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
  73. package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
  74. package/dist-lib/spec/mesh/hyperdispatch/messages.js +134 -5
  75. package/dist-lib/spec/mesh/hyperschema/index.js +134 -5
  76. package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
  77. package/dist-lib/spec/mesh/hyperschema/types.d.ts +20 -1
  78. package/package.json +23 -10
  79. package/tui/app.mjs +3 -2
  80. package/tui/boot.mjs +1 -3
  81. package/dist-lib/lib/model-clamp.d.ts +0 -6
  82. package/dist-lib/lib/model-clamp.js +0 -23
@@ -2,17 +2,25 @@ import process from 'process';
2
2
  import ScopeLock from 'scope-lock';
3
3
  import safetyCatch from 'safety-catch';
4
4
  import { fileURLToPath } from 'url';
5
+ import { Worker as Thread } from 'worker_threads';
5
6
  import { asWorkerReply } from './protocol.js';
7
+ import { errorMessage } from '../guards.js';
6
8
  const DEADLINE_MS = 60_000;
7
9
  const lock = new ScopeLock();
8
10
  let active = null;
9
11
  export async function convertInWorker(job, signal) {
10
12
  await lock.lock();
11
13
  try {
12
- const worker = active ?? (active = await spawn());
14
+ const worker = active ?? (active = spawn());
13
15
  const id = worker.nextId++;
14
- worker.link.write({ id, ...job });
15
- await settle(worker, id, signal);
16
+ worker.link.ref();
17
+ try {
18
+ worker.link.write({ id, ...job });
19
+ await settle(worker, id, signal);
20
+ }
21
+ finally {
22
+ worker.link.unref();
23
+ }
16
24
  }
17
25
  finally {
18
26
  lock.unlock();
@@ -62,34 +70,85 @@ function abandon(worker) {
62
70
  }
63
71
  worker.link.kill();
64
72
  }
65
- async function spawn() {
73
+ function spawn() {
66
74
  if (typeof Bare === 'undefined') {
67
75
  throw new Error('the document engine needs the Bare runtime');
68
76
  }
69
- const { default: Channel } = await import('bare-channel');
70
- const channel = new Channel();
71
- const thread = new Bare.Thread(entry(), { data: { handle: channel.handle } });
72
- const port = channel.connect();
77
+ let thread;
78
+ try {
79
+ thread = new Thread(entry());
80
+ }
81
+ catch (error) {
82
+ throw new Error(`the document engine failed to load: ${errorMessage(error)}`);
83
+ }
84
+ const inbox = new Inbox(thread);
85
+ thread.unref();
73
86
  return {
74
87
  nextId: 1,
75
88
  link: {
76
- read: () => port.read(),
77
- write: (job) => void port.write(job),
89
+ read: () => inbox.read(),
90
+ write: (job) => thread.postMessage(job),
91
+ ref: () => thread.ref(),
92
+ unref: () => thread.unref(),
78
93
  kill() {
79
- try {
80
- thread.terminate();
81
- }
82
- catch (error) {
83
- safetyCatch(error);
84
- }
85
- port.close().catch(safetyCatch);
94
+ thread.unref();
95
+ thread.terminate().catch(safetyCatch);
86
96
  }
87
97
  }
88
98
  };
89
99
  }
100
+ class Inbox {
101
+ replies = [];
102
+ waiting = [];
103
+ ended = false;
104
+ failure = null;
105
+ constructor(thread) {
106
+ thread.on('message', (reply) => this.push(reply));
107
+ thread.on('error', (error) => this.end(error));
108
+ thread.on('exit', () => this.end());
109
+ }
110
+ read() {
111
+ if (this.replies.length > 0) {
112
+ return Promise.resolve(this.replies.shift());
113
+ }
114
+ if (this.failure) {
115
+ return Promise.reject(this.failure);
116
+ }
117
+ if (this.ended) {
118
+ return Promise.resolve(null);
119
+ }
120
+ return new Promise((resolve, reject) => this.waiting.push({ resolve, reject }));
121
+ }
122
+ push(reply) {
123
+ const next = this.waiting.shift();
124
+ if (next) {
125
+ next.resolve(reply);
126
+ }
127
+ else {
128
+ this.replies.push(reply);
129
+ }
130
+ }
131
+ end(error) {
132
+ if (this.ended) {
133
+ return;
134
+ }
135
+ this.ended = true;
136
+ if (error !== undefined) {
137
+ this.failure = new Error(`the document engine closed unexpectedly: ${errorMessage(error)}`);
138
+ }
139
+ for (const next of this.waiting.splice(0)) {
140
+ if (this.failure) {
141
+ next.reject(this.failure);
142
+ }
143
+ else {
144
+ next.resolve(null);
145
+ }
146
+ }
147
+ }
148
+ }
90
149
  function entry() {
91
150
  return (process.env.QVAC_COLLABORA_WORKER_ENTRY ||
92
- fileURLToPath(new URL('./entry.ts', import.meta.url).href));
151
+ fileURLToPath(import.meta.resolve('#collabora-worker-entry')));
93
152
  }
94
153
  function deadlineMs() {
95
154
  const ms = Number(process.env.QVAC_COLLABORA_TIMEOUT_MS);
@@ -38,6 +38,7 @@ export interface MoveChatOptions {
38
38
  }
39
39
  export declare function moveChat(store: Mesh, id: string, { groupId, afterId }: MoveChatOptions): Promise<ChatListing>;
40
40
  export declare function setChatWebSearchEnabled(store: Mesh, id: string, webSearchEnabled: boolean): Promise<ChatListing>;
41
+ export declare function markChatRead(store: Mesh, id: string): Promise<ChatListing>;
41
42
  export declare function removeChatKnowledge(store: Mesh, chatId: string, at: number): Promise<void>;
42
43
  export interface RemoveChatAttachmentsOptions {
43
44
  readonly deviceId: Buffer;
@@ -2,6 +2,7 @@ import { listChatGroupsIn, mustGetActiveChatGroup } from './chat-group.js';
2
2
  import { appendChunk, listChunks, updateOp } from './chunk.js';
3
3
  import { randomId } from './id.js';
4
4
  import { assertRemovableAt } from './files/index.js';
5
+ import { chatReferences, reclaimReferences } from './files/records/purge.js';
5
6
  import { place } from './rank.js';
6
7
  import { chatSortKey } from './chat-order.js';
7
8
  import b4a from 'b4a';
@@ -202,6 +203,23 @@ export async function moveChat(store, id, { groupId, afterId }) {
202
203
  export function setChatWebSearchEnabled(store, id, webSearchEnabled) {
203
204
  return patchChat(store, id, { webSearchEnabled });
204
205
  }
206
+ export async function markChatRead(store, id) {
207
+ const chat = await mustGetActiveChat(store, id);
208
+ const snap = store.view.snapshot();
209
+ let count = 0;
210
+ try {
211
+ count = (await snap.get(CHUNK_COUNTERS, { chatId: id }))?.count ?? 0;
212
+ }
213
+ finally {
214
+ await snap.close();
215
+ }
216
+ if (count <= (chat.readThrough || 0)) {
217
+ return enrichChat(store, chat);
218
+ }
219
+ const read = { ...chat, readThrough: count };
220
+ await store.dispatch('put-chat', { chat: read });
221
+ return enrichChat(store, read);
222
+ }
205
223
  async function cancelActiveRuns(store, deviceId, chatId) {
206
224
  const statuses = await listChunks(store, { chatId, type: 'run-status' });
207
225
  const terminal = new Set(statuses.filter((c) => TERMINAL_RUN_STATES.has(c.runStatus?.state ?? '')).map((c) => c.runId));
@@ -246,15 +264,19 @@ export async function removeChatAttachments(store, chatId, { deviceId, at }) {
246
264
  }
247
265
  export async function deleteChat(store, deviceId, id) {
248
266
  const current = await mustGetChat(store, id);
267
+ const references = await chatReferences(store, id);
249
268
  const now = Date.now();
250
269
  if (deviceId) {
251
270
  await cancelActiveRuns(store, deviceId, id);
252
- await removeChatAttachments(store, id, { deviceId, at: now });
253
271
  }
254
- await removeChatKnowledge(store, id, now);
255
272
  const deleted = { ...current, deletedAt: now, updatedAt: now };
256
- await store.dispatch('put-chat', { chat: deleted });
257
- return enrichChat(store, deleted);
273
+ const listing = await enrichChat(store, deleted);
274
+ const batch = store.batch();
275
+ batch.dispatch('put-chat', { chat: deleted });
276
+ batch.dispatch('purge-chat', { chatId: id, at: now });
277
+ await batch.flush();
278
+ await reclaimReferences(store, references).catch((err) => logger.warn('sync', "reclaiming a deleted chat's files failed", { id, err }));
279
+ return listing;
258
280
  }
259
281
  export async function softDeleteChatsForAgent(store, deviceId, agentId) {
260
282
  const snap = store.view.snapshot();
@@ -37,6 +37,7 @@ export declare const LIVE_WRITERS_BY_MOST_RECENT = "@qvac/live-writers-by-most-r
37
37
  export declare const JOINED_MESHES = "@qvac/joined-meshes";
38
38
  export declare const AGENTS_BY_PROVIDER = "@qvac/agents-by-provider";
39
39
  export declare const DEFAULT_AGENTS = "@qvac/default-agents";
40
+ export declare const PURGED_CHATS = "@qvac/purged-chats";
40
41
  export declare const LOCAL_CONFIG = "@local/config";
41
42
  export declare const LOCAL_DEVICE = "@local/device";
42
43
  export declare const LOCAL_APP_CONFIG = "@local/app-config";
@@ -37,6 +37,7 @@ export const LIVE_WRITERS_BY_MOST_RECENT = '@qvac/live-writers-by-most-recent';
37
37
  export const JOINED_MESHES = '@qvac/joined-meshes';
38
38
  export const AGENTS_BY_PROVIDER = '@qvac/agents-by-provider';
39
39
  export const DEFAULT_AGENTS = '@qvac/default-agents';
40
+ export const PURGED_CHATS = '@qvac/purged-chats';
40
41
  export const LOCAL_CONFIG = '@local/config';
41
42
  export const LOCAL_DEVICE = '@local/device';
42
43
  export const LOCAL_APP_CONFIG = '@local/app-config';
@@ -1,2 +1,4 @@
1
+ import type HyperDB from 'hyperdb';
1
2
  import type { Router } from '../../../spec/mesh/hyperdispatch/index.js';
3
+ export declare function isPurged(tx: HyperDB, chatId: string): Promise<boolean>;
2
4
  export declare function registerChatRoutes(router: Router): void;
@@ -1,5 +1,8 @@
1
1
  import { trackActivity } from './activity.js';
2
- import { CHATS } from '../collections.js';
2
+ import { CHATS, CHUNKS, CHUNKS_BY_CHAT, CHUNK_COUNTERS, INDEXING_STATUS, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS, TOOL_ACTIVITY, TOOL_ACTIVITY_BY_AGENT } from '../collections.js';
3
+ export async function isPurged(tx, chatId) {
4
+ return (await tx.get(PURGED_CHATS, { chatId })) !== null;
5
+ }
3
6
  export function registerChatRoutes(router) {
4
7
  router.add('@qvac/put-chat', trackActivity(async ({ chat }, { tx, log }) => {
5
8
  const existing = await tx.get(CHATS, { id: chat.id });
@@ -7,12 +10,82 @@ export function registerChatRoutes(router) {
7
10
  log?.debug('contract', 'put-chat dropped: tombstone is sticky', { id: chat.id });
8
11
  return;
9
12
  }
13
+ const readThrough = Math.max(existing?.readThrough || 0, chat.readThrough || 0) || undefined;
10
14
  if (existing && !chat.deletedAt && (existing.updatedAt ?? 0) > (chat.updatedAt ?? 0)) {
11
- log?.debug('contract', 'put-chat dropped: stale updatedAt', { id: chat.id });
15
+ if (readThrough === (existing.readThrough || undefined)) {
16
+ log?.debug('contract', 'put-chat dropped: stale updatedAt', { id: chat.id });
17
+ return;
18
+ }
19
+ await tx.insert(CHATS, { ...existing, readThrough });
20
+ log?.debug('contract', 'put-chat stale: read marker kept', { id: chat.id });
21
+ return;
22
+ }
23
+ if (existing && (await isPurged(tx, chat.id))) {
24
+ log?.debug('contract', 'put-chat dropped: chat is purged', { id: chat.id });
12
25
  return;
13
26
  }
14
- await tx.insert(CHATS, chat);
27
+ const row = { ...chat, readThrough };
28
+ await tx.insert(CHATS, row);
15
29
  log?.debug('contract', existing ? 'chat updated' : 'chat created', { id: chat.id });
16
- return chat;
30
+ return row;
31
+ }));
32
+ router.add('@qvac/purge-chat', trackActivity(async ({ chatId, at }, { tx, log }) => {
33
+ const chat = await tx.get(CHATS, { id: chatId });
34
+ if (!chat?.deletedAt) {
35
+ log?.debug('contract', 'purge-chat dropped: chat is not deleted', { chatId });
36
+ return;
37
+ }
38
+ if (await isPurged(tx, chatId)) {
39
+ return;
40
+ }
41
+ await purgeChat(tx, chat);
42
+ const purged = { chatId, at: at || chat.deletedAt };
43
+ await tx.insert(PURGED_CHATS, purged);
44
+ log?.debug('contract', 'chat purged', { chatId });
45
+ return purged;
17
46
  }));
18
47
  }
48
+ async function purgeChat(tx, chat) {
49
+ const chatId = chat.id;
50
+ const inChat = { gte: { chatId }, lte: { chatId } };
51
+ for (const id of await idsOf(tx.find(CHUNKS_BY_CHAT, inChat))) {
52
+ await tx.delete(CHUNKS, { id });
53
+ }
54
+ await tx.delete(CHUNK_COUNTERS, { chatId });
55
+ for (const knowledge of await tx.find(KNOWLEDGE_BY_CHAT, inChat).toArray()) {
56
+ const ofKnowledge = { gte: { knowledgeId: knowledge.id }, lte: { knowledgeId: knowledge.id } };
57
+ const artifacts = tx.find(KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, ofKnowledge);
58
+ for (const artifact of await artifacts.toArray()) {
59
+ await tx.delete(KNOWLEDGE_ARTIFACTS, { id: artifact.id });
60
+ }
61
+ await tx.delete(KNOWLEDGE, { id: knowledge.id });
62
+ }
63
+ for (const status of await tx
64
+ .find(INDEXING_STATUS_BY_CHAT, inChat)
65
+ .toArray()) {
66
+ await tx.delete(INDEXING_STATUS, { id: status.id });
67
+ }
68
+ const ofAgent = { gte: { agentId: chat.agentId }, lte: { agentId: chat.agentId } };
69
+ const activity = tx.find(TOOL_ACTIVITY_BY_AGENT, ofAgent);
70
+ for (const id of await idsOf(activity, (row) => row.chatId === chatId)) {
71
+ await tx.delete(TOOL_ACTIVITY, { id });
72
+ }
73
+ const tombstone = {
74
+ id: chatId,
75
+ agentId: chat.agentId,
76
+ title: '',
77
+ createdAt: chat.createdAt,
78
+ deletedAt: chat.deletedAt,
79
+ updatedAt: chat.updatedAt
80
+ };
81
+ await tx.insert(CHATS, tombstone);
82
+ }
83
+ async function idsOf(rows, keep = () => true) {
84
+ const ids = [];
85
+ for await (const row of rows) {
86
+ if (keep(row)) {
87
+ ids.push(row.id);
88
+ }
89
+ }
90
+ return ids;
91
+ }
@@ -1,4 +1,5 @@
1
1
  import { devicesByWriterKey, trackActivity } from './activity.js';
2
+ import { isPurged } from './chat.js';
2
3
  import { CHUNK_PAYLOAD_FIELDS, ragSourcesWithinBounds } from '../../../schema/qvac.js';
3
4
  import { chunkTypeOrdinal } from '../../../schema/indexes.js';
4
5
  import b4a from 'b4a';
@@ -44,6 +45,10 @@ export function registerChunkRoutes(router) {
44
45
  if (chunk) {
45
46
  return;
46
47
  }
48
+ if (await isPurged(tx, op.chatId)) {
49
+ log?.debug('contract', 'append-chunk dropped: chat is purged', { id: op.id });
50
+ return;
51
+ }
47
52
  if (!payloadMatchesType(op)) {
48
53
  log?.warn('contract', 'append-chunk dropped: payload does not match its type', {
49
54
  id: op.id
@@ -1,5 +1,5 @@
1
1
  import type { Router } from '../../../spec/mesh/hyperdispatch/index.js';
2
- declare const DROPPED: readonly ['@qvac/append-chunk', '@qvac/update-chunk', '@qvac/resolve-approval', '@qvac/put-agent', '@qvac/put-approval-rule', '@qvac/seed-defaults', '@qvac/add-knowledge', '@qvac/remove-knowledge', '@qvac/add-knowledge-artifact', '@qvac/claim-indexing', '@qvac/resolve-indexing', '@qvac/put-chat', '@qvac/put-tool-configs', '@qvac/append-tool-activity', '@qvac/migrate-agent-catalog', '@qvac/reindex-agents', '@qvac/put-chat-group', '@qvac/put-joined-mesh'];
2
+ declare const DROPPED: readonly ['@qvac/append-chunk', '@qvac/update-chunk', '@qvac/resolve-approval', '@qvac/put-agent', '@qvac/put-approval-rule', '@qvac/seed-defaults', '@qvac/add-knowledge', '@qvac/remove-knowledge', '@qvac/add-knowledge-artifact', '@qvac/claim-indexing', '@qvac/resolve-indexing', '@qvac/put-chat', '@qvac/put-tool-configs', '@qvac/append-tool-activity', '@qvac/migrate-agent-catalog', '@qvac/reindex-agents', '@qvac/put-chat-group', '@qvac/put-joined-mesh', '@qvac/purge-chat'];
3
3
  export type DroppedComputeRoute = (typeof DROPPED)[number];
4
4
  export declare function droppedComputeRoutes(): ReadonlyArray<DroppedComputeRoute>;
5
5
  export declare class ComputeRouteDropped extends Error {
@@ -16,7 +16,8 @@ const DROPPED = [
16
16
  '@qvac/migrate-agent-catalog',
17
17
  '@qvac/reindex-agents',
18
18
  '@qvac/put-chat-group',
19
- '@qvac/put-joined-mesh'
19
+ '@qvac/put-joined-mesh',
20
+ '@qvac/purge-chat'
20
21
  ];
21
22
  const DROPPED_ROUTES = new Set(DROPPED);
22
23
  export function droppedComputeRoutes() {
@@ -1,3 +1,4 @@
1
+ import { isPurged } from './chat.js';
1
2
  import { capToolActivityArguments } from '../../../schema/qvac.js';
2
3
  import { TOOL_ACTIVITY, TOOL_ACTIVITY_COUNTER } from '../collections.js';
3
4
  export function registerToolActivityRoutes(router) {
@@ -6,6 +7,10 @@ export function registerToolActivityRoutes(router) {
6
7
  if (existing) {
7
8
  return;
8
9
  }
10
+ if (activity.chatId && (await isPurged(tx, activity.chatId))) {
11
+ log?.debug('contract', 'tool activity dropped: chat is purged', { id: activity.id });
12
+ return;
13
+ }
9
14
  const counter = await tx.get(TOOL_ACTIVITY_COUNTER, {
10
15
  agentId: activity.agentId
11
16
  });
@@ -69,6 +69,7 @@ export declare class Files extends EventEmitter {
69
69
  inline(bytes: Buffer, mimeType: string): string;
70
70
  reference(id: Parameters<BlobStore['reference']>[0]): string;
71
71
  clear(reference: string, id: string): Promise<boolean>;
72
+ compact(coreKeys: ReadonlyArray<Buffer>): Promise<void>;
72
73
  attachments(options?: Parameters<typeof attachmentFiles>[1]): import("./attachments.ts").BoundAttachments;
73
74
  knowledge(options?: Parameters<typeof knowledgeFiles>[1]): Pick<import("../../../spec/capabilities").CapabilityHandlers, "downloadKnowledge" | "listIndexing" | "listKnowledge" | "removeKnowledge" | "uploadKnowledge">;
74
75
  listKnowledge(chatId: string): Promise<QvacKnowledge[]>;
@@ -84,6 +85,7 @@ export declare class Files extends EventEmitter {
84
85
  syncAttachment(chunk: QvacChunk): Promise<void>;
85
86
  syncKnowledge(row: QvacKnowledge): Promise<void>;
86
87
  syncArtifact(row: QvacKnowledgeArtifact): Promise<void>;
88
+ reclaimPurged(): Promise<number>;
87
89
  sweepTombstones(deviceId?: Buffer | null): Promise<void>;
88
90
  sweepOrphans(coreLength: number): Promise<void>;
89
91
  processorState(id: string): FileState;
@@ -1,7 +1,7 @@
1
1
  import b4a from 'b4a';
2
2
  import EventEmitter from 'bare-events';
3
3
  import { attachmentFiles } from './attachments.js';
4
- import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
4
+ import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, reclaimPurgedFiles, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
5
5
  import { knowledgeFiles } from './knowledge.js';
6
6
  import { randomId } from '../id.js';
7
7
  import { getChunk, listChunks } from '../chunk.js';
@@ -75,6 +75,9 @@ export class Files extends EventEmitter {
75
75
  clear(reference, id) {
76
76
  return this._store.clear(reference, id);
77
77
  }
78
+ compact(coreKeys) {
79
+ return this._store.compact(coreKeys);
80
+ }
78
81
  attachments(options = {}) {
79
82
  return attachmentFiles(this._mesh, options);
80
83
  }
@@ -160,6 +163,9 @@ export class Files extends EventEmitter {
160
163
  syncArtifact(row) {
161
164
  return syncArtifactArrival(this._mesh, row);
162
165
  }
166
+ reclaimPurged() {
167
+ return reclaimPurgedFiles(this._mesh);
168
+ }
163
169
  sweepTombstones(deviceId = null) {
164
170
  return sweepTombstonedBlobs(this._mesh, deviceId);
165
171
  }
@@ -1,11 +1,14 @@
1
1
  import type { QvacChunk } from '../../../spec/mesh/hyperschema/types.d.ts';
2
- import type { FileKey, KnownFile } from '../local/files.ts';
2
+ import type { FileKey, KnownFile, LocalFile } from '../local/files.ts';
3
3
  export type FileChunk = Pick<QvacChunk, 'type' | 'chatId' | 'attachment'>;
4
4
  export interface MeshFileIndex {
5
5
  record(chunks: ReadonlyArray<FileChunk>): Promise<void>;
6
6
  refresh(chunks: ReadonlyArray<FileChunk>): Promise<void>;
7
7
  read(reference: string): void;
8
8
  markCleared(blobId: string): Promise<void>;
9
+ list(): Promise<LocalFile[]>;
10
+ retireChats(purged: ReadonlyMap<string, number>): Promise<void>;
11
+ forget(file: FileKey): Promise<void>;
9
12
  }
10
13
  export interface FileOwner {
11
14
  readonly meshKey: Buffer;
@@ -3,6 +3,7 @@ import { type IndexingProgress } from '../rag/index.ts';
3
3
  import { type FileInput, type FileRecord } from '../records/index.ts';
4
4
  import type { Mesh } from '../../mesh.ts';
5
5
  import { FileDocument } from './document.ts';
6
+ import type { LoadedArtifact } from './state.ts';
6
7
  import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
7
8
  export interface ProcessorHost {
8
9
  readonly mesh: Mesh;
@@ -19,6 +20,7 @@ export interface FileProcessor {
19
20
  canProduce(document: FileRecord, host: ProcessorHost): boolean | Promise<boolean>;
20
21
  produce(document: FileDocument, signal: AbortSignal): Promise<void>;
21
22
  load?(document: FileDocument, signal: AbortSignal): Promise<void>;
23
+ evict?(loaded: LoadedArtifact): Promise<void>;
22
24
  close(): Promise<boolean>;
23
25
  }
24
26
  export interface FileProcessingOptions {
@@ -67,6 +69,7 @@ export declare class FileProcessing {
67
69
  private _scan;
68
70
  private _documents;
69
71
  private _pump;
72
+ private _forget;
70
73
  private _drain;
71
74
  private _recordFailure;
72
75
  private _queueLoad;
@@ -214,6 +214,12 @@ export class FileProcessing {
214
214
  this._pump(lane);
215
215
  });
216
216
  }
217
+ async _forget(id) {
218
+ const loaded = this._mesh.files.processorState(this._processor.id).forget(id)?.loaded;
219
+ if (loaded) {
220
+ await this._processor.evict?.(loaded);
221
+ }
222
+ }
217
223
  async _drain(lane) {
218
224
  while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
219
225
  const [[id, wakes]] = lane.queue;
@@ -223,6 +229,9 @@ export class FileProcessing {
223
229
  let produced = false;
224
230
  try {
225
231
  const record = await fileRecord(this._mesh.view, id, this._processor.input);
232
+ if (!record && lane.stage === 'load') {
233
+ await this._forget(id);
234
+ }
226
235
  if (!record || record.deletedAt) {
227
236
  await this._source.cancel((key) => key === id);
228
237
  await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
@@ -55,6 +55,7 @@ export declare class FileState {
55
55
  readonly outputRetries: FileRetries;
56
56
  document(id: string): DocumentState;
57
57
  artifact(id: string): ArtifactState;
58
+ forget(id: string): DocumentState | undefined;
58
59
  get(id: string): Readonly<DocumentState> | undefined;
59
60
  readArtifact(id: string): Readonly<ArtifactState> | undefined;
60
61
  progress(chatId: string): IndexingProgressEntry[];
@@ -70,6 +70,11 @@ export class FileState {
70
70
  }
71
71
  return entry;
72
72
  }
73
+ forget(id) {
74
+ const entry = this._documents.get(id);
75
+ this._documents.delete(id);
76
+ return entry;
77
+ }
73
78
  get(id) {
74
79
  return this._documents.get(id);
75
80
  }
@@ -3,6 +3,7 @@ import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts
3
3
  import { type RagExtractors, type VectorIndex } from 'rag-extract';
4
4
  import { type IndexingEmbedder } from './produce.ts';
5
5
  import type { FileDocument, FileProcessor, ProcessorHost } from '../processing/index.ts';
6
+ import type { LoadedArtifact } from '../processing/state.ts';
6
7
  export interface RagProcessorOptions {
7
8
  readonly model: string;
8
9
  readonly embedder?: IndexingEmbedder;
@@ -21,6 +22,7 @@ export declare class RagProcessor implements FileProcessor {
21
22
  applies(_document: QvacKnowledge): boolean;
22
23
  canProduce(document: QvacKnowledge, host: ProcessorHost): Promise<boolean>;
23
24
  produce(document: FileDocument, signal: AbortSignal): Promise<void>;
25
+ evict(loaded: LoadedArtifact): Promise<void>;
24
26
  close(): Promise<boolean>;
25
27
  }
26
28
  export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
@@ -49,6 +49,12 @@ export class RagProcessor {
49
49
  async produce(document, signal) {
50
50
  await this._producer?.run(document, signal);
51
51
  }
52
+ async evict(loaded) {
53
+ if (loaded.evict) {
54
+ await this._index?.remove(loaded.evict);
55
+ }
56
+ await this._index?.remove(loaded.artifactId);
57
+ }
52
58
  async close() {
53
59
  try {
54
60
  await this._index?.clear();
@@ -5,6 +5,7 @@ export { groupAttachmentsByMessage } from './linkage.ts';
5
5
  export { syncAttachmentChunk } from './chunks.ts';
6
6
  export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from './reclaim.ts';
7
7
  export { referencedBlobs } from './sweep.ts';
8
+ export { chatReferences, reclaimPurgedFiles, reclaimReferences } from './purge.ts';
8
9
  export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from './output.ts';
9
10
  export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from './source.ts';
10
11
  export type { FileInput, FileRecord } from './source.ts';
@@ -4,5 +4,6 @@ export { groupAttachmentsByMessage } from './linkage.js';
4
4
  export { syncAttachmentChunk } from './chunks.js';
5
5
  export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from './reclaim.js';
6
6
  export { referencedBlobs } from './sweep.js';
7
+ export { chatReferences, reclaimPurgedFiles, reclaimReferences } from './purge.js';
7
8
  export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from './output.js';
8
9
  export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from './source.js';
@@ -0,0 +1,4 @@
1
+ import type { Mesh } from '../../mesh.ts';
2
+ export declare function reclaimPurgedFiles(mesh: Mesh): Promise<number>;
3
+ export declare function chatReferences(mesh: Mesh, chatId: string): Promise<string[]>;
4
+ export declare function reclaimReferences(mesh: Mesh, references: ReadonlyArray<string>): Promise<void>;