@qvac/core 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +19 -2
  2. package/dist-lib/lib/assistant.d.ts +1 -0
  3. package/dist-lib/lib/assistant.js +22 -12
  4. package/dist-lib/lib/core/chat.js +9 -4
  5. package/dist-lib/lib/core/collections.d.ts +1 -0
  6. package/dist-lib/lib/core/collections.js +1 -0
  7. package/dist-lib/lib/core/contract/chat.d.ts +2 -0
  8. package/dist-lib/lib/core/contract/chat.js +67 -1
  9. package/dist-lib/lib/core/contract/chunks.js +5 -0
  10. package/dist-lib/lib/core/contract/compute.d.ts +1 -1
  11. package/dist-lib/lib/core/contract/compute.js +2 -1
  12. package/dist-lib/lib/core/contract/tool-activity.js +5 -0
  13. package/dist-lib/lib/core/files/index.d.ts +2 -0
  14. package/dist-lib/lib/core/files/index.js +7 -1
  15. package/dist-lib/lib/core/files/local-index.d.ts +4 -1
  16. package/dist-lib/lib/core/files/processing/index.d.ts +3 -0
  17. package/dist-lib/lib/core/files/processing/index.js +9 -0
  18. package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
  19. package/dist-lib/lib/core/files/processing/state.js +5 -0
  20. package/dist-lib/lib/core/files/rag/index.d.ts +2 -0
  21. package/dist-lib/lib/core/files/rag/index.js +6 -0
  22. package/dist-lib/lib/core/files/records/index.d.ts +1 -0
  23. package/dist-lib/lib/core/files/records/index.js +1 -0
  24. package/dist-lib/lib/core/files/records/purge.d.ts +4 -0
  25. package/dist-lib/lib/core/files/records/purge.js +108 -0
  26. package/dist-lib/lib/core/files/records/reclaim.js +5 -3
  27. package/dist-lib/lib/core/files/store/blob.d.ts +1 -0
  28. package/dist-lib/lib/core/files/store/blob.js +29 -0
  29. package/dist-lib/lib/core/files/store/index.d.ts +1 -0
  30. package/dist-lib/lib/core/files/store/index.js +6 -1
  31. package/dist-lib/lib/core/local/files.d.ts +2 -0
  32. package/dist-lib/lib/core/local/files.js +31 -0
  33. package/dist-lib/lib/core/sync/index.js +14 -1
  34. package/dist-lib/lib/core.js +22 -2
  35. package/dist-lib/lib/credential-pusher.d.ts +8 -3
  36. package/dist-lib/lib/credential-pusher.js +4 -1
  37. package/dist-lib/schema/qvac.js +10 -0
  38. package/dist-lib/spec/mesh/hyperdb/db.json +16 -1
  39. package/dist-lib/spec/mesh/hyperdb/index.js +71 -2
  40. package/dist-lib/spec/mesh/hyperdb/messages.js +400 -221
  41. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
  42. package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
  43. package/dist-lib/spec/mesh/hyperdispatch/messages.js +117 -2
  44. package/dist-lib/spec/mesh/hyperschema/index.js +117 -2
  45. package/dist-lib/spec/mesh/hyperschema/schema.json +65 -1
  46. package/dist-lib/spec/mesh/hyperschema/types.d.ts +19 -1
  47. package/package.json +3 -3
  48. package/tui/boot.mjs +1 -3
package/CHANGELOG.md CHANGED
@@ -13,6 +13,22 @@ release.
13
13
 
14
14
  ## [Unreleased]
15
15
 
16
+ ### Fixed
17
+
18
+ - `duck` crashed at start on Bare 1.33 and later, `--help` and `--version`
19
+ included, with "Invalid left-hand side expression in prefix operation": Bare
20
+ now links what `import.meta.resolve()` names, and the TUI named a `SKILL.md`.
21
+
22
+ ## [0.6.1] - 2026-09-29
23
+
24
+ ### Fixed
25
+
26
+ - A connected skill's OAuth token and settings reach the harness again. Since
27
+ 0.4.0 the credential pusher listed agents without naming the account mesh, so
28
+ every push failed before `setCredentials` and a skill read `needs-setup` as if
29
+ it had no credential. The pusher now lists the account mesh's agents, and
30
+ between accounts pushes the shared scope alone. (#265)
31
+
16
32
  ## [0.6.0] - 2026-09-28
17
33
 
18
34
  ### Added
@@ -57,7 +73,7 @@ release.
57
73
  model, storing nothing; with no harness that can preview, it rejects.
58
74
  `removeCustomModel` also frees the downloaded copy and answers with the
59
75
  `deleteModel` summary in place of `{ ok }`. Needs `@qvac/harness` 0.6. See
60
- [custom models](docs/api/custom-models.md). (#215)
76
+ [custom models](docs/api/models.md#custom-models). (#215)
61
77
 
62
78
  ```js
63
79
  // 0.5.0
@@ -1036,7 +1052,8 @@ Published under the `release-0.1` dist-tag.
1036
1052
  await assistant.ready()
1037
1053
  ```
1038
1054
 
1039
- [Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.6.0...HEAD
1055
+ [Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.6.1...HEAD
1056
+ [0.6.1]: https://github.com/tetherto/qvac-core/compare/v0.6.0...v0.6.1
1040
1057
  [0.6.0]: https://github.com/tetherto/qvac-core/compare/v0.5.0...v0.6.0
1041
1058
  [0.5.0]: https://github.com/tetherto/qvac-core/compare/v0.4.1...v0.5.0
1042
1059
  [0.4.1]: https://github.com/tetherto/qvac-core/compare/v0.4.0...v0.4.1
@@ -142,6 +142,7 @@ export declare class QvacAssistant extends ReadyResource {
142
142
  tools(): Promise<ReadonlyArray<ToolCatalogEntry>>;
143
143
  private _supervisedWatch;
144
144
  private _reclaimDeletedChats;
145
+ private _dropDeletedChat;
145
146
  private _reclaimRekeyedAgents;
146
147
  private _reclaimAgentChatCaches;
147
148
  private _reconcileCompaction;
@@ -417,9 +417,9 @@ export class QvacAssistant extends ReadyResource {
417
417
  return DEFAULT_PROFILE;
418
418
  }
419
419
  }
420
- async _sweepImageCache() {
420
+ async _sweepImageCache(gone = new Set()) {
421
421
  try {
422
- const keep = await this._referencedImageNames();
422
+ const keep = await this._referencedImageNames(gone);
423
423
  await sweepImageCache(this._imageCacheDir, {
424
424
  maxBytes: this._imageCacheMaxBytes,
425
425
  maxAgeMs: this._imageCacheMaxAgeMs,
@@ -449,13 +449,14 @@ export class QvacAssistant extends ReadyResource {
449
449
  }
450
450
  return false;
451
451
  }
452
- async _referencedImageNames() {
452
+ async _referencedImageNames(gone) {
453
453
  try {
454
454
  const { files } = await this._engine.listLocalFiles({});
455
455
  if (!files?.length) {
456
456
  return null;
457
457
  }
458
- return new Set(files.map((file) => imageCacheKey(file.blobId)));
458
+ const live = files.filter((file) => !file.chatId || !gone.has(file.chatId));
459
+ return new Set(live.map((file) => imageCacheKey(file.blobId)));
459
460
  }
460
461
  catch (err) {
461
462
  this._log?.warn('assistant', 'reading the local file index failed', err);
@@ -936,21 +937,30 @@ export class QvacAssistant extends ReadyResource {
936
937
  if (this.closing || this.suspended)
937
938
  return;
938
939
  const ids = (frame.chats ?? []).map((chat) => chat.id);
940
+ const gone = new Set(known === null ? [] : disappearedChatIds(known, ids));
939
941
  if (known !== null) {
940
- for (const chatId of disappearedChatIds(known, ids)) {
941
- this._reconciledCompactions.delete(chatId);
942
- this._chatAttribution.delete(chatId);
943
- void this._deleteChatKvCache(chatId, 'chat deleted');
944
- void this._inferenceCalls
945
- .forget(chatId)
946
- .catch((err) => this._log?.warn('assistant', 'inspector records not dropped', { err }));
942
+ for (const chatId of gone) {
943
+ this._dropDeletedChat(chatId);
947
944
  }
948
945
  }
949
946
  known = new Set(ids);
950
- await this._sweepImageCache();
947
+ await this._sweepImageCache(gone);
951
948
  await this._sweepInferenceCalls();
952
949
  }
953
950
  }
951
+ _dropDeletedChat(chatId) {
952
+ for (const run of this._runs.values()) {
953
+ if (run.chatId === chatId) {
954
+ void RUN_SETTLED;
955
+ }
956
+ }
957
+ this._reconciledCompactions.delete(chatId);
958
+ this._chatAttribution.delete(chatId);
959
+ void this._deleteChatKvCache(chatId, 'chat deleted');
960
+ void this._inferenceCalls
961
+ .forget(chatId)
962
+ .catch((err) => this._log?.warn('assistant', 'inspector records not dropped', { err }));
963
+ }
954
964
  async _reclaimRekeyedAgents(meshId, frames) {
955
965
  let known = null;
956
966
  for await (const frame of frames) {
@@ -2,6 +2,7 @@ import { listChatGroupsIn, mustGetActiveChatGroup } from './chat-group.js';
2
2
  import { appendChunk, listChunks, updateOp } from './chunk.js';
3
3
  import { randomId } from './id.js';
4
4
  import { assertRemovableAt } from './files/index.js';
5
+ import { chatReferences, reclaimReferences } from './files/records/purge.js';
5
6
  import { place } from './rank.js';
6
7
  import { chatSortKey } from './chat-order.js';
7
8
  import b4a from 'b4a';
@@ -246,15 +247,19 @@ export async function removeChatAttachments(store, chatId, { deviceId, at }) {
246
247
  }
247
248
  export async function deleteChat(store, deviceId, id) {
248
249
  const current = await mustGetChat(store, id);
250
+ const references = await chatReferences(store, id);
249
251
  const now = Date.now();
250
252
  if (deviceId) {
251
253
  await cancelActiveRuns(store, deviceId, id);
252
- await removeChatAttachments(store, id, { deviceId, at: now });
253
254
  }
254
- await removeChatKnowledge(store, id, now);
255
255
  const deleted = { ...current, deletedAt: now, updatedAt: now };
256
- await store.dispatch('put-chat', { chat: deleted });
257
- return enrichChat(store, deleted);
256
+ const listing = await enrichChat(store, deleted);
257
+ const batch = store.batch();
258
+ batch.dispatch('put-chat', { chat: deleted });
259
+ batch.dispatch('purge-chat', { chatId: id, at: now });
260
+ await batch.flush();
261
+ await reclaimReferences(store, references).catch((err) => logger.warn('sync', "reclaiming a deleted chat's files failed", { id, err }));
262
+ return listing;
258
263
  }
259
264
  export async function softDeleteChatsForAgent(store, deviceId, agentId) {
260
265
  const snap = store.view.snapshot();
@@ -37,6 +37,7 @@ export declare const LIVE_WRITERS_BY_MOST_RECENT = "@qvac/live-writers-by-most-r
37
37
  export declare const JOINED_MESHES = "@qvac/joined-meshes";
38
38
  export declare const AGENTS_BY_PROVIDER = "@qvac/agents-by-provider";
39
39
  export declare const DEFAULT_AGENTS = "@qvac/default-agents";
40
+ export declare const PURGED_CHATS = "@qvac/purged-chats";
40
41
  export declare const LOCAL_CONFIG = "@local/config";
41
42
  export declare const LOCAL_DEVICE = "@local/device";
42
43
  export declare const LOCAL_APP_CONFIG = "@local/app-config";
@@ -37,6 +37,7 @@ export const LIVE_WRITERS_BY_MOST_RECENT = '@qvac/live-writers-by-most-recent';
37
37
  export const JOINED_MESHES = '@qvac/joined-meshes';
38
38
  export const AGENTS_BY_PROVIDER = '@qvac/agents-by-provider';
39
39
  export const DEFAULT_AGENTS = '@qvac/default-agents';
40
+ export const PURGED_CHATS = '@qvac/purged-chats';
40
41
  export const LOCAL_CONFIG = '@local/config';
41
42
  export const LOCAL_DEVICE = '@local/device';
42
43
  export const LOCAL_APP_CONFIG = '@local/app-config';
@@ -1,2 +1,4 @@
1
+ import type HyperDB from 'hyperdb';
1
2
  import type { Router } from '../../../spec/mesh/hyperdispatch/index.js';
3
+ export declare function isPurged(tx: HyperDB, chatId: string): Promise<boolean>;
2
4
  export declare function registerChatRoutes(router: Router): void;
@@ -1,5 +1,8 @@
1
1
  import { trackActivity } from './activity.js';
2
- import { CHATS } from '../collections.js';
2
+ import { CHATS, CHUNKS, CHUNKS_BY_CHAT, CHUNK_COUNTERS, INDEXING_STATUS, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS, TOOL_ACTIVITY, TOOL_ACTIVITY_BY_AGENT } from '../collections.js';
3
+ export async function isPurged(tx, chatId) {
4
+ return (await tx.get(PURGED_CHATS, { chatId })) !== null;
5
+ }
3
6
  export function registerChatRoutes(router) {
4
7
  router.add('@qvac/put-chat', trackActivity(async ({ chat }, { tx, log }) => {
5
8
  const existing = await tx.get(CHATS, { id: chat.id });
@@ -11,8 +14,71 @@ export function registerChatRoutes(router) {
11
14
  log?.debug('contract', 'put-chat dropped: stale updatedAt', { id: chat.id });
12
15
  return;
13
16
  }
17
+ if (existing && (await isPurged(tx, chat.id))) {
18
+ log?.debug('contract', 'put-chat dropped: chat is purged', { id: chat.id });
19
+ return;
20
+ }
14
21
  await tx.insert(CHATS, chat);
15
22
  log?.debug('contract', existing ? 'chat updated' : 'chat created', { id: chat.id });
16
23
  return chat;
17
24
  }));
25
+ router.add('@qvac/purge-chat', trackActivity(async ({ chatId, at }, { tx, log }) => {
26
+ const chat = await tx.get(CHATS, { id: chatId });
27
+ if (!chat?.deletedAt) {
28
+ log?.debug('contract', 'purge-chat dropped: chat is not deleted', { chatId });
29
+ return;
30
+ }
31
+ if (await isPurged(tx, chatId)) {
32
+ return;
33
+ }
34
+ await purgeChat(tx, chat);
35
+ const purged = { chatId, at: at || chat.deletedAt };
36
+ await tx.insert(PURGED_CHATS, purged);
37
+ log?.debug('contract', 'chat purged', { chatId });
38
+ return purged;
39
+ }));
40
+ }
41
+ async function purgeChat(tx, chat) {
42
+ const chatId = chat.id;
43
+ const inChat = { gte: { chatId }, lte: { chatId } };
44
+ for (const id of await idsOf(tx.find(CHUNKS_BY_CHAT, inChat))) {
45
+ await tx.delete(CHUNKS, { id });
46
+ }
47
+ await tx.delete(CHUNK_COUNTERS, { chatId });
48
+ for (const knowledge of await tx.find(KNOWLEDGE_BY_CHAT, inChat).toArray()) {
49
+ const ofKnowledge = { gte: { knowledgeId: knowledge.id }, lte: { knowledgeId: knowledge.id } };
50
+ const artifacts = tx.find(KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, ofKnowledge);
51
+ for (const artifact of await artifacts.toArray()) {
52
+ await tx.delete(KNOWLEDGE_ARTIFACTS, { id: artifact.id });
53
+ }
54
+ await tx.delete(KNOWLEDGE, { id: knowledge.id });
55
+ }
56
+ for (const status of await tx
57
+ .find(INDEXING_STATUS_BY_CHAT, inChat)
58
+ .toArray()) {
59
+ await tx.delete(INDEXING_STATUS, { id: status.id });
60
+ }
61
+ const ofAgent = { gte: { agentId: chat.agentId }, lte: { agentId: chat.agentId } };
62
+ const activity = tx.find(TOOL_ACTIVITY_BY_AGENT, ofAgent);
63
+ for (const id of await idsOf(activity, (row) => row.chatId === chatId)) {
64
+ await tx.delete(TOOL_ACTIVITY, { id });
65
+ }
66
+ const tombstone = {
67
+ id: chatId,
68
+ agentId: chat.agentId,
69
+ title: '',
70
+ createdAt: chat.createdAt,
71
+ deletedAt: chat.deletedAt,
72
+ updatedAt: chat.updatedAt
73
+ };
74
+ await tx.insert(CHATS, tombstone);
75
+ }
76
+ async function idsOf(rows, keep = () => true) {
77
+ const ids = [];
78
+ for await (const row of rows) {
79
+ if (keep(row)) {
80
+ ids.push(row.id);
81
+ }
82
+ }
83
+ return ids;
18
84
  }
@@ -1,4 +1,5 @@
1
1
  import { devicesByWriterKey, trackActivity } from './activity.js';
2
+ import { isPurged } from './chat.js';
2
3
  import { CHUNK_PAYLOAD_FIELDS, ragSourcesWithinBounds } from '../../../schema/qvac.js';
3
4
  import { chunkTypeOrdinal } from '../../../schema/indexes.js';
4
5
  import b4a from 'b4a';
@@ -44,6 +45,10 @@ export function registerChunkRoutes(router) {
44
45
  if (chunk) {
45
46
  return;
46
47
  }
48
+ if (await isPurged(tx, op.chatId)) {
49
+ log?.debug('contract', 'append-chunk dropped: chat is purged', { id: op.id });
50
+ return;
51
+ }
47
52
  if (!payloadMatchesType(op)) {
48
53
  log?.warn('contract', 'append-chunk dropped: payload does not match its type', {
49
54
  id: op.id
@@ -1,5 +1,5 @@
1
1
  import type { Router } from '../../../spec/mesh/hyperdispatch/index.js';
2
- declare const DROPPED: readonly ['@qvac/append-chunk', '@qvac/update-chunk', '@qvac/resolve-approval', '@qvac/put-agent', '@qvac/put-approval-rule', '@qvac/seed-defaults', '@qvac/add-knowledge', '@qvac/remove-knowledge', '@qvac/add-knowledge-artifact', '@qvac/claim-indexing', '@qvac/resolve-indexing', '@qvac/put-chat', '@qvac/put-tool-configs', '@qvac/append-tool-activity', '@qvac/migrate-agent-catalog', '@qvac/reindex-agents', '@qvac/put-chat-group', '@qvac/put-joined-mesh'];
2
+ declare const DROPPED: readonly ['@qvac/append-chunk', '@qvac/update-chunk', '@qvac/resolve-approval', '@qvac/put-agent', '@qvac/put-approval-rule', '@qvac/seed-defaults', '@qvac/add-knowledge', '@qvac/remove-knowledge', '@qvac/add-knowledge-artifact', '@qvac/claim-indexing', '@qvac/resolve-indexing', '@qvac/put-chat', '@qvac/put-tool-configs', '@qvac/append-tool-activity', '@qvac/migrate-agent-catalog', '@qvac/reindex-agents', '@qvac/put-chat-group', '@qvac/put-joined-mesh', '@qvac/purge-chat'];
3
3
  export type DroppedComputeRoute = (typeof DROPPED)[number];
4
4
  export declare function droppedComputeRoutes(): ReadonlyArray<DroppedComputeRoute>;
5
5
  export declare class ComputeRouteDropped extends Error {
@@ -16,7 +16,8 @@ const DROPPED = [
16
16
  '@qvac/migrate-agent-catalog',
17
17
  '@qvac/reindex-agents',
18
18
  '@qvac/put-chat-group',
19
- '@qvac/put-joined-mesh'
19
+ '@qvac/put-joined-mesh',
20
+ '@qvac/purge-chat'
20
21
  ];
21
22
  const DROPPED_ROUTES = new Set(DROPPED);
22
23
  export function droppedComputeRoutes() {
@@ -1,3 +1,4 @@
1
+ import { isPurged } from './chat.js';
1
2
  import { capToolActivityArguments } from '../../../schema/qvac.js';
2
3
  import { TOOL_ACTIVITY, TOOL_ACTIVITY_COUNTER } from '../collections.js';
3
4
  export function registerToolActivityRoutes(router) {
@@ -6,6 +7,10 @@ export function registerToolActivityRoutes(router) {
6
7
  if (existing) {
7
8
  return;
8
9
  }
10
+ if (activity.chatId && (await isPurged(tx, activity.chatId))) {
11
+ log?.debug('contract', 'tool activity dropped: chat is purged', { id: activity.id });
12
+ return;
13
+ }
9
14
  const counter = await tx.get(TOOL_ACTIVITY_COUNTER, {
10
15
  agentId: activity.agentId
11
16
  });
@@ -69,6 +69,7 @@ export declare class Files extends EventEmitter {
69
69
  inline(bytes: Buffer, mimeType: string): string;
70
70
  reference(id: Parameters<BlobStore['reference']>[0]): string;
71
71
  clear(reference: string, id: string): Promise<boolean>;
72
+ compact(coreKeys: ReadonlyArray<Buffer>): Promise<void>;
72
73
  attachments(options?: Parameters<typeof attachmentFiles>[1]): import("./attachments.ts").BoundAttachments;
73
74
  knowledge(options?: Parameters<typeof knowledgeFiles>[1]): Pick<import("../../../spec/capabilities").CapabilityHandlers, "downloadKnowledge" | "listIndexing" | "listKnowledge" | "removeKnowledge" | "uploadKnowledge">;
74
75
  listKnowledge(chatId: string): Promise<QvacKnowledge[]>;
@@ -84,6 +85,7 @@ export declare class Files extends EventEmitter {
84
85
  syncAttachment(chunk: QvacChunk): Promise<void>;
85
86
  syncKnowledge(row: QvacKnowledge): Promise<void>;
86
87
  syncArtifact(row: QvacKnowledgeArtifact): Promise<void>;
88
+ reclaimPurged(): Promise<number>;
87
89
  sweepTombstones(deviceId?: Buffer | null): Promise<void>;
88
90
  sweepOrphans(coreLength: number): Promise<void>;
89
91
  processorState(id: string): FileState;
@@ -1,7 +1,7 @@
1
1
  import b4a from 'b4a';
2
2
  import EventEmitter from 'bare-events';
3
3
  import { attachmentFiles } from './attachments.js';
4
- import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
4
+ import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, reclaimPurgedFiles, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
5
5
  import { knowledgeFiles } from './knowledge.js';
6
6
  import { randomId } from '../id.js';
7
7
  import { getChunk, listChunks } from '../chunk.js';
@@ -75,6 +75,9 @@ export class Files extends EventEmitter {
75
75
  clear(reference, id) {
76
76
  return this._store.clear(reference, id);
77
77
  }
78
+ compact(coreKeys) {
79
+ return this._store.compact(coreKeys);
80
+ }
78
81
  attachments(options = {}) {
79
82
  return attachmentFiles(this._mesh, options);
80
83
  }
@@ -160,6 +163,9 @@ export class Files extends EventEmitter {
160
163
  syncArtifact(row) {
161
164
  return syncArtifactArrival(this._mesh, row);
162
165
  }
166
+ reclaimPurged() {
167
+ return reclaimPurgedFiles(this._mesh);
168
+ }
163
169
  sweepTombstones(deviceId = null) {
164
170
  return sweepTombstonedBlobs(this._mesh, deviceId);
165
171
  }
@@ -1,11 +1,14 @@
1
1
  import type { QvacChunk } from '../../../spec/mesh/hyperschema/types.d.ts';
2
- import type { FileKey, KnownFile } from '../local/files.ts';
2
+ import type { FileKey, KnownFile, LocalFile } from '../local/files.ts';
3
3
  export type FileChunk = Pick<QvacChunk, 'type' | 'chatId' | 'attachment'>;
4
4
  export interface MeshFileIndex {
5
5
  record(chunks: ReadonlyArray<FileChunk>): Promise<void>;
6
6
  refresh(chunks: ReadonlyArray<FileChunk>): Promise<void>;
7
7
  read(reference: string): void;
8
8
  markCleared(blobId: string): Promise<void>;
9
+ list(): Promise<LocalFile[]>;
10
+ retireChats(purged: ReadonlyMap<string, number>): Promise<void>;
11
+ forget(file: FileKey): Promise<void>;
9
12
  }
10
13
  export interface FileOwner {
11
14
  readonly meshKey: Buffer;
@@ -3,6 +3,7 @@ import { type IndexingProgress } from '../rag/index.ts';
3
3
  import { type FileInput, type FileRecord } from '../records/index.ts';
4
4
  import type { Mesh } from '../../mesh.ts';
5
5
  import { FileDocument } from './document.ts';
6
+ import type { LoadedArtifact } from './state.ts';
6
7
  import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
7
8
  export interface ProcessorHost {
8
9
  readonly mesh: Mesh;
@@ -19,6 +20,7 @@ export interface FileProcessor {
19
20
  canProduce(document: FileRecord, host: ProcessorHost): boolean | Promise<boolean>;
20
21
  produce(document: FileDocument, signal: AbortSignal): Promise<void>;
21
22
  load?(document: FileDocument, signal: AbortSignal): Promise<void>;
23
+ evict?(loaded: LoadedArtifact): Promise<void>;
22
24
  close(): Promise<boolean>;
23
25
  }
24
26
  export interface FileProcessingOptions {
@@ -67,6 +69,7 @@ export declare class FileProcessing {
67
69
  private _scan;
68
70
  private _documents;
69
71
  private _pump;
72
+ private _forget;
70
73
  private _drain;
71
74
  private _recordFailure;
72
75
  private _queueLoad;
@@ -214,6 +214,12 @@ export class FileProcessing {
214
214
  this._pump(lane);
215
215
  });
216
216
  }
217
+ async _forget(id) {
218
+ const loaded = this._mesh.files.processorState(this._processor.id).forget(id)?.loaded;
219
+ if (loaded) {
220
+ await this._processor.evict?.(loaded);
221
+ }
222
+ }
217
223
  async _drain(lane) {
218
224
  while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
219
225
  const [[id, wakes]] = lane.queue;
@@ -223,6 +229,9 @@ export class FileProcessing {
223
229
  let produced = false;
224
230
  try {
225
231
  const record = await fileRecord(this._mesh.view, id, this._processor.input);
232
+ if (!record && lane.stage === 'load') {
233
+ await this._forget(id);
234
+ }
226
235
  if (!record || record.deletedAt) {
227
236
  await this._source.cancel((key) => key === id);
228
237
  await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
@@ -55,6 +55,7 @@ export declare class FileState {
55
55
  readonly outputRetries: FileRetries;
56
56
  document(id: string): DocumentState;
57
57
  artifact(id: string): ArtifactState;
58
+ forget(id: string): DocumentState | undefined;
58
59
  get(id: string): Readonly<DocumentState> | undefined;
59
60
  readArtifact(id: string): Readonly<ArtifactState> | undefined;
60
61
  progress(chatId: string): IndexingProgressEntry[];
@@ -70,6 +70,11 @@ export class FileState {
70
70
  }
71
71
  return entry;
72
72
  }
73
+ forget(id) {
74
+ const entry = this._documents.get(id);
75
+ this._documents.delete(id);
76
+ return entry;
77
+ }
73
78
  get(id) {
74
79
  return this._documents.get(id);
75
80
  }
@@ -3,6 +3,7 @@ import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts
3
3
  import { type RagExtractors, type VectorIndex } from 'rag-extract';
4
4
  import { type IndexingEmbedder } from './produce.ts';
5
5
  import type { FileDocument, FileProcessor, ProcessorHost } from '../processing/index.ts';
6
+ import type { LoadedArtifact } from '../processing/state.ts';
6
7
  export interface RagProcessorOptions {
7
8
  readonly model: string;
8
9
  readonly embedder?: IndexingEmbedder;
@@ -21,6 +22,7 @@ export declare class RagProcessor implements FileProcessor {
21
22
  applies(_document: QvacKnowledge): boolean;
22
23
  canProduce(document: QvacKnowledge, host: ProcessorHost): Promise<boolean>;
23
24
  produce(document: FileDocument, signal: AbortSignal): Promise<void>;
25
+ evict(loaded: LoadedArtifact): Promise<void>;
24
26
  close(): Promise<boolean>;
25
27
  }
26
28
  export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
@@ -49,6 +49,12 @@ export class RagProcessor {
49
49
  async produce(document, signal) {
50
50
  await this._producer?.run(document, signal);
51
51
  }
52
+ async evict(loaded) {
53
+ if (loaded.evict) {
54
+ await this._index?.remove(loaded.evict);
55
+ }
56
+ await this._index?.remove(loaded.artifactId);
57
+ }
52
58
  async close() {
53
59
  try {
54
60
  await this._index?.clear();
@@ -5,6 +5,7 @@ export { groupAttachmentsByMessage } from './linkage.ts';
5
5
  export { syncAttachmentChunk } from './chunks.ts';
6
6
  export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from './reclaim.ts';
7
7
  export { referencedBlobs } from './sweep.ts';
8
+ export { chatReferences, reclaimPurgedFiles, reclaimReferences } from './purge.ts';
8
9
  export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from './output.ts';
9
10
  export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from './source.ts';
10
11
  export type { FileInput, FileRecord } from './source.ts';
@@ -4,5 +4,6 @@ export { groupAttachmentsByMessage } from './linkage.js';
4
4
  export { syncAttachmentChunk } from './chunks.js';
5
5
  export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from './reclaim.js';
6
6
  export { referencedBlobs } from './sweep.js';
7
+ export { chatReferences, reclaimPurgedFiles, reclaimReferences } from './purge.js';
7
8
  export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from './output.js';
8
9
  export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from './source.js';
@@ -0,0 +1,4 @@
1
+ import type { Mesh } from '../../mesh.ts';
2
+ export declare function reclaimPurgedFiles(mesh: Mesh): Promise<number>;
3
+ export declare function chatReferences(mesh: Mesh, chatId: string): Promise<string[]>;
4
+ export declare function reclaimReferences(mesh: Mesh, references: ReadonlyArray<string>): Promise<void>;
@@ -0,0 +1,108 @@
1
+ import b4a from 'b4a';
2
+ import { fileKeyOf } from '../local-index.js';
3
+ import { referencedBlobs } from './sweep.js';
4
+ import { CHUNKS_BY_CHAT, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS } from '../../collections.js';
5
+ export async function reclaimPurgedFiles(mesh) {
6
+ const index = mesh.fileIndex;
7
+ if (!index) {
8
+ return 0;
9
+ }
10
+ const rows = await index.list();
11
+ const purged = await purgedChatIds(mesh, rows);
12
+ const candidates = rows.filter((row) => !!row.chatId && purged.has(row.chatId));
13
+ if (candidates.length === 0) {
14
+ return 0;
15
+ }
16
+ const owner = ownerOf(mesh);
17
+ const referenced = await referencedKeys(mesh, owner);
18
+ const cores = new Map();
19
+ let reclaimed = 0;
20
+ for (const row of candidates) {
21
+ if (mesh.closing)
22
+ break;
23
+ if (referenced.has(keyId(row)))
24
+ continue;
25
+ if (!(await mesh.files.clear(row.blobId, row.attachmentId ?? row.blobId))) {
26
+ continue;
27
+ }
28
+ await index.forget(row);
29
+ cores.set(b4a.toHex(row.coreKey), row.coreKey);
30
+ reclaimed++;
31
+ }
32
+ if (cores.size > 0 && !mesh.closing) {
33
+ await mesh.files.compact([...cores.values()]);
34
+ }
35
+ return reclaimed;
36
+ }
37
+ export async function chatReferences(mesh, chatId) {
38
+ const references = [];
39
+ const inChat = { gte: { chatId }, lte: { chatId } };
40
+ const snap = mesh.view.snapshot();
41
+ try {
42
+ for await (const chunk of snap.find(CHUNKS_BY_CHAT, inChat)) {
43
+ if (chunk.type === 'attachment' && chunk.attachment) {
44
+ references.push(chunk.attachment.blobId);
45
+ }
46
+ }
47
+ for (const row of await snap.find(KNOWLEDGE_BY_CHAT, inChat).toArray()) {
48
+ references.push(row.blobId);
49
+ const ofKnowledge = { gte: { knowledgeId: row.id }, lte: { knowledgeId: row.id } };
50
+ for await (const artifact of snap.find(KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, ofKnowledge)) {
51
+ references.push(artifact.artifactBlobId);
52
+ }
53
+ }
54
+ }
55
+ finally {
56
+ await snap.close();
57
+ }
58
+ return references;
59
+ }
60
+ export async function reclaimReferences(mesh, references) {
61
+ if (references.length === 0)
62
+ return;
63
+ const owner = ownerOf(mesh);
64
+ const referenced = await referencedKeys(mesh, owner);
65
+ const cores = new Map();
66
+ for (const reference of references) {
67
+ if (mesh.closing)
68
+ return;
69
+ const key = fileKeyOf(reference, owner);
70
+ if (!key || referenced.has(keyId(key))) {
71
+ continue;
72
+ }
73
+ if (!(await mesh.files.clear(reference, reference))) {
74
+ continue;
75
+ }
76
+ await mesh.fileIndex?.forget(key);
77
+ cores.set(b4a.toHex(key.coreKey), key.coreKey);
78
+ }
79
+ if (cores.size > 0) {
80
+ void mesh.files.compact([...cores.values()]);
81
+ }
82
+ }
83
+ async function purgedChatIds(mesh, rows) {
84
+ const purged = new Set();
85
+ const chatIds = rows.map((row) => row.chatId).filter((id) => !!id);
86
+ for (const chatId of new Set(chatIds)) {
87
+ if (await mesh.view.get(PURGED_CHATS, { chatId })) {
88
+ purged.add(chatId);
89
+ }
90
+ }
91
+ return purged;
92
+ }
93
+ function ownerOf(mesh) {
94
+ return { meshKey: mesh.key, ownCoreKey: mesh.blobs.core.key };
95
+ }
96
+ async function referencedKeys(mesh, owner) {
97
+ const keys = new Set();
98
+ for (const reference of await referencedBlobs(mesh)) {
99
+ const key = fileKeyOf(reference, owner);
100
+ if (key) {
101
+ keys.add(keyId(key));
102
+ }
103
+ }
104
+ return keys;
105
+ }
106
+ function keyId(key) {
107
+ return `${b4a.toHex(key.coreKey)}/${key.blockOffset}`;
108
+ }
@@ -3,7 +3,7 @@ import { removeChatAttachments, removeChatKnowledge } from '../../chat.js';
3
3
  import { listChunks } from '../../chunk.js';
4
4
  import { getKnowledge, listKnowledgeArtifactsByKnowledge } from './knowledge.js';
5
5
  import { syncAttachmentChunk } from './chunks.js';
6
- import { CHATS, CHUNK_COUNTERS, KNOWLEDGE } from '../../collections.js';
6
+ import { CHATS, CHUNK_COUNTERS, KNOWLEDGE, PURGED_CHATS } from '../../collections.js';
7
7
  export async function syncKnowledgeTombstone(mesh, row) {
8
8
  if (!row.deletedAt)
9
9
  return;
@@ -62,10 +62,12 @@ async function recascadeTombstonedChats(mesh, deviceId) {
62
62
  const chats = mesh.view.snapshot();
63
63
  try {
64
64
  for await (const chat of chats.find(CHATS)) {
65
- if (mesh.closing)
65
+ if (mesh.closing) {
66
66
  return false;
67
- if (!chat.deletedAt)
67
+ }
68
+ if (!chat.deletedAt || (await chats.get(PURGED_CHATS, { chatId: chat.id }))) {
68
69
  continue;
70
+ }
69
71
  await removeChatAttachments(mesh, chat.id, { deviceId, at: chat.deletedAt });
70
72
  await removeChatKnowledge(mesh, chat.id, chat.deletedAt);
71
73
  }
@@ -27,3 +27,4 @@ export interface BlobReadChunk {
27
27
  }
28
28
  export declare function readBlob(blobs: Hyperblobs, blobId: BlobId, options?: ReadBlobOptions): AsyncGenerator<BlobReadChunk>;
29
29
  export declare function clearBlob(mesh: BlobHost, blobId: string, id: string): Promise<boolean>;
30
+ export declare function compactBlobCores(mesh: BlobHost, coreKeys: ReadonlyArray<Buffer>): Promise<void>;