@qvac/core 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -2
- package/dist-lib/lib/assistant.d.ts +1 -0
- package/dist-lib/lib/assistant.js +22 -12
- package/dist-lib/lib/core/chat.js +9 -4
- package/dist-lib/lib/core/collections.d.ts +1 -0
- package/dist-lib/lib/core/collections.js +1 -0
- package/dist-lib/lib/core/contract/chat.d.ts +2 -0
- package/dist-lib/lib/core/contract/chat.js +67 -1
- package/dist-lib/lib/core/contract/chunks.js +5 -0
- package/dist-lib/lib/core/contract/compute.d.ts +1 -1
- package/dist-lib/lib/core/contract/compute.js +2 -1
- package/dist-lib/lib/core/contract/tool-activity.js +5 -0
- package/dist-lib/lib/core/files/index.d.ts +2 -0
- package/dist-lib/lib/core/files/index.js +7 -1
- package/dist-lib/lib/core/files/local-index.d.ts +4 -1
- package/dist-lib/lib/core/files/processing/index.d.ts +3 -0
- package/dist-lib/lib/core/files/processing/index.js +9 -0
- package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/state.js +5 -0
- package/dist-lib/lib/core/files/rag/index.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/index.js +6 -0
- package/dist-lib/lib/core/files/records/index.d.ts +1 -0
- package/dist-lib/lib/core/files/records/index.js +1 -0
- package/dist-lib/lib/core/files/records/purge.d.ts +4 -0
- package/dist-lib/lib/core/files/records/purge.js +108 -0
- package/dist-lib/lib/core/files/records/reclaim.js +5 -3
- package/dist-lib/lib/core/files/store/blob.d.ts +1 -0
- package/dist-lib/lib/core/files/store/blob.js +29 -0
- package/dist-lib/lib/core/files/store/index.d.ts +1 -0
- package/dist-lib/lib/core/files/store/index.js +6 -1
- package/dist-lib/lib/core/local/files.d.ts +2 -0
- package/dist-lib/lib/core/local/files.js +31 -0
- package/dist-lib/lib/core/sync/index.js +14 -1
- package/dist-lib/lib/core.js +22 -2
- package/dist-lib/lib/credential-pusher.d.ts +8 -3
- package/dist-lib/lib/credential-pusher.js +4 -1
- package/dist-lib/schema/qvac.js +10 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +16 -1
- package/dist-lib/spec/mesh/hyperdb/index.js +71 -2
- package/dist-lib/spec/mesh/hyperdb/messages.js +400 -221
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
- package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +117 -2
- package/dist-lib/spec/mesh/hyperschema/index.js +117 -2
- package/dist-lib/spec/mesh/hyperschema/schema.json +65 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +19 -1
- package/package.json +3 -3
- package/tui/boot.mjs +1 -3
package/CHANGELOG.md
CHANGED
|
@@ -13,6 +13,22 @@ release.
|
|
|
13
13
|
|
|
14
14
|
## [Unreleased]
|
|
15
15
|
|
|
16
|
+
### Fixed
|
|
17
|
+
|
|
18
|
+
- `duck` crashed at start on Bare 1.33 and later, `--help` and `--version`
|
|
19
|
+
included, with "Invalid left-hand side expression in prefix operation": Bare
|
|
20
|
+
now links what `import.meta.resolve()` names, and the TUI named a `SKILL.md`.
|
|
21
|
+
|
|
22
|
+
## [0.6.1] - 2026-09-29
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
|
|
26
|
+
- A connected skill's OAuth token and settings reach the harness again. Since
|
|
27
|
+
0.4.0 the credential pusher listed agents without naming the account mesh, so
|
|
28
|
+
every push failed before `setCredentials` and a skill read `needs-setup` as if
|
|
29
|
+
it had no credential. The pusher now lists the account mesh's agents, and
|
|
30
|
+
between accounts pushes the shared scope alone. (#265)
|
|
31
|
+
|
|
16
32
|
## [0.6.0] - 2026-09-28
|
|
17
33
|
|
|
18
34
|
### Added
|
|
@@ -57,7 +73,7 @@ release.
|
|
|
57
73
|
model, storing nothing; with no harness that can preview, it rejects.
|
|
58
74
|
`removeCustomModel` also frees the downloaded copy and answers with the
|
|
59
75
|
`deleteModel` summary in place of `{ ok }`. Needs `@qvac/harness` 0.6. See
|
|
60
|
-
[custom models](docs/api/custom-models
|
|
76
|
+
[custom models](docs/api/models.md#custom-models). (#215)
|
|
61
77
|
|
|
62
78
|
```js
|
|
63
79
|
// 0.5.0
|
|
@@ -1036,7 +1052,8 @@ Published under the `release-0.1` dist-tag.
|
|
|
1036
1052
|
await assistant.ready()
|
|
1037
1053
|
```
|
|
1038
1054
|
|
|
1039
|
-
[Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.6.
|
|
1055
|
+
[Unreleased]: https://github.com/tetherto/qvac-core/compare/v0.6.1...HEAD
|
|
1056
|
+
[0.6.1]: https://github.com/tetherto/qvac-core/compare/v0.6.0...v0.6.1
|
|
1040
1057
|
[0.6.0]: https://github.com/tetherto/qvac-core/compare/v0.5.0...v0.6.0
|
|
1041
1058
|
[0.5.0]: https://github.com/tetherto/qvac-core/compare/v0.4.1...v0.5.0
|
|
1042
1059
|
[0.4.1]: https://github.com/tetherto/qvac-core/compare/v0.4.0...v0.4.1
|
|
@@ -142,6 +142,7 @@ export declare class QvacAssistant extends ReadyResource {
|
|
|
142
142
|
tools(): Promise<ReadonlyArray<ToolCatalogEntry>>;
|
|
143
143
|
private _supervisedWatch;
|
|
144
144
|
private _reclaimDeletedChats;
|
|
145
|
+
private _dropDeletedChat;
|
|
145
146
|
private _reclaimRekeyedAgents;
|
|
146
147
|
private _reclaimAgentChatCaches;
|
|
147
148
|
private _reconcileCompaction;
|
|
@@ -417,9 +417,9 @@ export class QvacAssistant extends ReadyResource {
|
|
|
417
417
|
return DEFAULT_PROFILE;
|
|
418
418
|
}
|
|
419
419
|
}
|
|
420
|
-
async _sweepImageCache() {
|
|
420
|
+
async _sweepImageCache(gone = new Set()) {
|
|
421
421
|
try {
|
|
422
|
-
const keep = await this._referencedImageNames();
|
|
422
|
+
const keep = await this._referencedImageNames(gone);
|
|
423
423
|
await sweepImageCache(this._imageCacheDir, {
|
|
424
424
|
maxBytes: this._imageCacheMaxBytes,
|
|
425
425
|
maxAgeMs: this._imageCacheMaxAgeMs,
|
|
@@ -449,13 +449,14 @@ export class QvacAssistant extends ReadyResource {
|
|
|
449
449
|
}
|
|
450
450
|
return false;
|
|
451
451
|
}
|
|
452
|
-
async _referencedImageNames() {
|
|
452
|
+
async _referencedImageNames(gone) {
|
|
453
453
|
try {
|
|
454
454
|
const { files } = await this._engine.listLocalFiles({});
|
|
455
455
|
if (!files?.length) {
|
|
456
456
|
return null;
|
|
457
457
|
}
|
|
458
|
-
|
|
458
|
+
const live = files.filter((file) => !file.chatId || !gone.has(file.chatId));
|
|
459
|
+
return new Set(live.map((file) => imageCacheKey(file.blobId)));
|
|
459
460
|
}
|
|
460
461
|
catch (err) {
|
|
461
462
|
this._log?.warn('assistant', 'reading the local file index failed', err);
|
|
@@ -936,21 +937,30 @@ export class QvacAssistant extends ReadyResource {
|
|
|
936
937
|
if (this.closing || this.suspended)
|
|
937
938
|
return;
|
|
938
939
|
const ids = (frame.chats ?? []).map((chat) => chat.id);
|
|
940
|
+
const gone = new Set(known === null ? [] : disappearedChatIds(known, ids));
|
|
939
941
|
if (known !== null) {
|
|
940
|
-
for (const chatId of
|
|
941
|
-
this.
|
|
942
|
-
this._chatAttribution.delete(chatId);
|
|
943
|
-
void this._deleteChatKvCache(chatId, 'chat deleted');
|
|
944
|
-
void this._inferenceCalls
|
|
945
|
-
.forget(chatId)
|
|
946
|
-
.catch((err) => this._log?.warn('assistant', 'inspector records not dropped', { err }));
|
|
942
|
+
for (const chatId of gone) {
|
|
943
|
+
this._dropDeletedChat(chatId);
|
|
947
944
|
}
|
|
948
945
|
}
|
|
949
946
|
known = new Set(ids);
|
|
950
|
-
await this._sweepImageCache();
|
|
947
|
+
await this._sweepImageCache(gone);
|
|
951
948
|
await this._sweepInferenceCalls();
|
|
952
949
|
}
|
|
953
950
|
}
|
|
951
|
+
_dropDeletedChat(chatId) {
|
|
952
|
+
for (const run of this._runs.values()) {
|
|
953
|
+
if (run.chatId === chatId) {
|
|
954
|
+
void RUN_SETTLED;
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
this._reconciledCompactions.delete(chatId);
|
|
958
|
+
this._chatAttribution.delete(chatId);
|
|
959
|
+
void this._deleteChatKvCache(chatId, 'chat deleted');
|
|
960
|
+
void this._inferenceCalls
|
|
961
|
+
.forget(chatId)
|
|
962
|
+
.catch((err) => this._log?.warn('assistant', 'inspector records not dropped', { err }));
|
|
963
|
+
}
|
|
954
964
|
async _reclaimRekeyedAgents(meshId, frames) {
|
|
955
965
|
let known = null;
|
|
956
966
|
for await (const frame of frames) {
|
|
@@ -2,6 +2,7 @@ import { listChatGroupsIn, mustGetActiveChatGroup } from './chat-group.js';
|
|
|
2
2
|
import { appendChunk, listChunks, updateOp } from './chunk.js';
|
|
3
3
|
import { randomId } from './id.js';
|
|
4
4
|
import { assertRemovableAt } from './files/index.js';
|
|
5
|
+
import { chatReferences, reclaimReferences } from './files/records/purge.js';
|
|
5
6
|
import { place } from './rank.js';
|
|
6
7
|
import { chatSortKey } from './chat-order.js';
|
|
7
8
|
import b4a from 'b4a';
|
|
@@ -246,15 +247,19 @@ export async function removeChatAttachments(store, chatId, { deviceId, at }) {
|
|
|
246
247
|
}
|
|
247
248
|
export async function deleteChat(store, deviceId, id) {
|
|
248
249
|
const current = await mustGetChat(store, id);
|
|
250
|
+
const references = await chatReferences(store, id);
|
|
249
251
|
const now = Date.now();
|
|
250
252
|
if (deviceId) {
|
|
251
253
|
await cancelActiveRuns(store, deviceId, id);
|
|
252
|
-
await removeChatAttachments(store, id, { deviceId, at: now });
|
|
253
254
|
}
|
|
254
|
-
await removeChatKnowledge(store, id, now);
|
|
255
255
|
const deleted = { ...current, deletedAt: now, updatedAt: now };
|
|
256
|
-
await store
|
|
257
|
-
|
|
256
|
+
const listing = await enrichChat(store, deleted);
|
|
257
|
+
const batch = store.batch();
|
|
258
|
+
batch.dispatch('put-chat', { chat: deleted });
|
|
259
|
+
batch.dispatch('purge-chat', { chatId: id, at: now });
|
|
260
|
+
await batch.flush();
|
|
261
|
+
await reclaimReferences(store, references).catch((err) => logger.warn('sync', "reclaiming a deleted chat's files failed", { id, err }));
|
|
262
|
+
return listing;
|
|
258
263
|
}
|
|
259
264
|
export async function softDeleteChatsForAgent(store, deviceId, agentId) {
|
|
260
265
|
const snap = store.view.snapshot();
|
|
@@ -37,6 +37,7 @@ export declare const LIVE_WRITERS_BY_MOST_RECENT = "@qvac/live-writers-by-most-r
|
|
|
37
37
|
export declare const JOINED_MESHES = "@qvac/joined-meshes";
|
|
38
38
|
export declare const AGENTS_BY_PROVIDER = "@qvac/agents-by-provider";
|
|
39
39
|
export declare const DEFAULT_AGENTS = "@qvac/default-agents";
|
|
40
|
+
export declare const PURGED_CHATS = "@qvac/purged-chats";
|
|
40
41
|
export declare const LOCAL_CONFIG = "@local/config";
|
|
41
42
|
export declare const LOCAL_DEVICE = "@local/device";
|
|
42
43
|
export declare const LOCAL_APP_CONFIG = "@local/app-config";
|
|
@@ -37,6 +37,7 @@ export const LIVE_WRITERS_BY_MOST_RECENT = '@qvac/live-writers-by-most-recent';
|
|
|
37
37
|
export const JOINED_MESHES = '@qvac/joined-meshes';
|
|
38
38
|
export const AGENTS_BY_PROVIDER = '@qvac/agents-by-provider';
|
|
39
39
|
export const DEFAULT_AGENTS = '@qvac/default-agents';
|
|
40
|
+
export const PURGED_CHATS = '@qvac/purged-chats';
|
|
40
41
|
export const LOCAL_CONFIG = '@local/config';
|
|
41
42
|
export const LOCAL_DEVICE = '@local/device';
|
|
42
43
|
export const LOCAL_APP_CONFIG = '@local/app-config';
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { trackActivity } from './activity.js';
|
|
2
|
-
import { CHATS } from '../collections.js';
|
|
2
|
+
import { CHATS, CHUNKS, CHUNKS_BY_CHAT, CHUNK_COUNTERS, INDEXING_STATUS, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS, TOOL_ACTIVITY, TOOL_ACTIVITY_BY_AGENT } from '../collections.js';
|
|
3
|
+
export async function isPurged(tx, chatId) {
|
|
4
|
+
return (await tx.get(PURGED_CHATS, { chatId })) !== null;
|
|
5
|
+
}
|
|
3
6
|
export function registerChatRoutes(router) {
|
|
4
7
|
router.add('@qvac/put-chat', trackActivity(async ({ chat }, { tx, log }) => {
|
|
5
8
|
const existing = await tx.get(CHATS, { id: chat.id });
|
|
@@ -11,8 +14,71 @@ export function registerChatRoutes(router) {
|
|
|
11
14
|
log?.debug('contract', 'put-chat dropped: stale updatedAt', { id: chat.id });
|
|
12
15
|
return;
|
|
13
16
|
}
|
|
17
|
+
if (existing && (await isPurged(tx, chat.id))) {
|
|
18
|
+
log?.debug('contract', 'put-chat dropped: chat is purged', { id: chat.id });
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
14
21
|
await tx.insert(CHATS, chat);
|
|
15
22
|
log?.debug('contract', existing ? 'chat updated' : 'chat created', { id: chat.id });
|
|
16
23
|
return chat;
|
|
17
24
|
}));
|
|
25
|
+
router.add('@qvac/purge-chat', trackActivity(async ({ chatId, at }, { tx, log }) => {
|
|
26
|
+
const chat = await tx.get(CHATS, { id: chatId });
|
|
27
|
+
if (!chat?.deletedAt) {
|
|
28
|
+
log?.debug('contract', 'purge-chat dropped: chat is not deleted', { chatId });
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
if (await isPurged(tx, chatId)) {
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
await purgeChat(tx, chat);
|
|
35
|
+
const purged = { chatId, at: at || chat.deletedAt };
|
|
36
|
+
await tx.insert(PURGED_CHATS, purged);
|
|
37
|
+
log?.debug('contract', 'chat purged', { chatId });
|
|
38
|
+
return purged;
|
|
39
|
+
}));
|
|
40
|
+
}
|
|
41
|
+
async function purgeChat(tx, chat) {
|
|
42
|
+
const chatId = chat.id;
|
|
43
|
+
const inChat = { gte: { chatId }, lte: { chatId } };
|
|
44
|
+
for (const id of await idsOf(tx.find(CHUNKS_BY_CHAT, inChat))) {
|
|
45
|
+
await tx.delete(CHUNKS, { id });
|
|
46
|
+
}
|
|
47
|
+
await tx.delete(CHUNK_COUNTERS, { chatId });
|
|
48
|
+
for (const knowledge of await tx.find(KNOWLEDGE_BY_CHAT, inChat).toArray()) {
|
|
49
|
+
const ofKnowledge = { gte: { knowledgeId: knowledge.id }, lte: { knowledgeId: knowledge.id } };
|
|
50
|
+
const artifacts = tx.find(KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, ofKnowledge);
|
|
51
|
+
for (const artifact of await artifacts.toArray()) {
|
|
52
|
+
await tx.delete(KNOWLEDGE_ARTIFACTS, { id: artifact.id });
|
|
53
|
+
}
|
|
54
|
+
await tx.delete(KNOWLEDGE, { id: knowledge.id });
|
|
55
|
+
}
|
|
56
|
+
for (const status of await tx
|
|
57
|
+
.find(INDEXING_STATUS_BY_CHAT, inChat)
|
|
58
|
+
.toArray()) {
|
|
59
|
+
await tx.delete(INDEXING_STATUS, { id: status.id });
|
|
60
|
+
}
|
|
61
|
+
const ofAgent = { gte: { agentId: chat.agentId }, lte: { agentId: chat.agentId } };
|
|
62
|
+
const activity = tx.find(TOOL_ACTIVITY_BY_AGENT, ofAgent);
|
|
63
|
+
for (const id of await idsOf(activity, (row) => row.chatId === chatId)) {
|
|
64
|
+
await tx.delete(TOOL_ACTIVITY, { id });
|
|
65
|
+
}
|
|
66
|
+
const tombstone = {
|
|
67
|
+
id: chatId,
|
|
68
|
+
agentId: chat.agentId,
|
|
69
|
+
title: '',
|
|
70
|
+
createdAt: chat.createdAt,
|
|
71
|
+
deletedAt: chat.deletedAt,
|
|
72
|
+
updatedAt: chat.updatedAt
|
|
73
|
+
};
|
|
74
|
+
await tx.insert(CHATS, tombstone);
|
|
75
|
+
}
|
|
76
|
+
async function idsOf(rows, keep = () => true) {
|
|
77
|
+
const ids = [];
|
|
78
|
+
for await (const row of rows) {
|
|
79
|
+
if (keep(row)) {
|
|
80
|
+
ids.push(row.id);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return ids;
|
|
18
84
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { devicesByWriterKey, trackActivity } from './activity.js';
|
|
2
|
+
import { isPurged } from './chat.js';
|
|
2
3
|
import { CHUNK_PAYLOAD_FIELDS, ragSourcesWithinBounds } from '../../../schema/qvac.js';
|
|
3
4
|
import { chunkTypeOrdinal } from '../../../schema/indexes.js';
|
|
4
5
|
import b4a from 'b4a';
|
|
@@ -44,6 +45,10 @@ export function registerChunkRoutes(router) {
|
|
|
44
45
|
if (chunk) {
|
|
45
46
|
return;
|
|
46
47
|
}
|
|
48
|
+
if (await isPurged(tx, op.chatId)) {
|
|
49
|
+
log?.debug('contract', 'append-chunk dropped: chat is purged', { id: op.id });
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
47
52
|
if (!payloadMatchesType(op)) {
|
|
48
53
|
log?.warn('contract', 'append-chunk dropped: payload does not match its type', {
|
|
49
54
|
id: op.id
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Router } from '../../../spec/mesh/hyperdispatch/index.js';
|
|
2
|
-
declare const DROPPED: readonly ['@qvac/append-chunk', '@qvac/update-chunk', '@qvac/resolve-approval', '@qvac/put-agent', '@qvac/put-approval-rule', '@qvac/seed-defaults', '@qvac/add-knowledge', '@qvac/remove-knowledge', '@qvac/add-knowledge-artifact', '@qvac/claim-indexing', '@qvac/resolve-indexing', '@qvac/put-chat', '@qvac/put-tool-configs', '@qvac/append-tool-activity', '@qvac/migrate-agent-catalog', '@qvac/reindex-agents', '@qvac/put-chat-group', '@qvac/put-joined-mesh'];
|
|
2
|
+
declare const DROPPED: readonly ['@qvac/append-chunk', '@qvac/update-chunk', '@qvac/resolve-approval', '@qvac/put-agent', '@qvac/put-approval-rule', '@qvac/seed-defaults', '@qvac/add-knowledge', '@qvac/remove-knowledge', '@qvac/add-knowledge-artifact', '@qvac/claim-indexing', '@qvac/resolve-indexing', '@qvac/put-chat', '@qvac/put-tool-configs', '@qvac/append-tool-activity', '@qvac/migrate-agent-catalog', '@qvac/reindex-agents', '@qvac/put-chat-group', '@qvac/put-joined-mesh', '@qvac/purge-chat'];
|
|
3
3
|
export type DroppedComputeRoute = (typeof DROPPED)[number];
|
|
4
4
|
export declare function droppedComputeRoutes(): ReadonlyArray<DroppedComputeRoute>;
|
|
5
5
|
export declare class ComputeRouteDropped extends Error {
|
|
@@ -16,7 +16,8 @@ const DROPPED = [
|
|
|
16
16
|
'@qvac/migrate-agent-catalog',
|
|
17
17
|
'@qvac/reindex-agents',
|
|
18
18
|
'@qvac/put-chat-group',
|
|
19
|
-
'@qvac/put-joined-mesh'
|
|
19
|
+
'@qvac/put-joined-mesh',
|
|
20
|
+
'@qvac/purge-chat'
|
|
20
21
|
];
|
|
21
22
|
const DROPPED_ROUTES = new Set(DROPPED);
|
|
22
23
|
export function droppedComputeRoutes() {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isPurged } from './chat.js';
|
|
1
2
|
import { capToolActivityArguments } from '../../../schema/qvac.js';
|
|
2
3
|
import { TOOL_ACTIVITY, TOOL_ACTIVITY_COUNTER } from '../collections.js';
|
|
3
4
|
export function registerToolActivityRoutes(router) {
|
|
@@ -6,6 +7,10 @@ export function registerToolActivityRoutes(router) {
|
|
|
6
7
|
if (existing) {
|
|
7
8
|
return;
|
|
8
9
|
}
|
|
10
|
+
if (activity.chatId && (await isPurged(tx, activity.chatId))) {
|
|
11
|
+
log?.debug('contract', 'tool activity dropped: chat is purged', { id: activity.id });
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
9
14
|
const counter = await tx.get(TOOL_ACTIVITY_COUNTER, {
|
|
10
15
|
agentId: activity.agentId
|
|
11
16
|
});
|
|
@@ -69,6 +69,7 @@ export declare class Files extends EventEmitter {
|
|
|
69
69
|
inline(bytes: Buffer, mimeType: string): string;
|
|
70
70
|
reference(id: Parameters<BlobStore['reference']>[0]): string;
|
|
71
71
|
clear(reference: string, id: string): Promise<boolean>;
|
|
72
|
+
compact(coreKeys: ReadonlyArray<Buffer>): Promise<void>;
|
|
72
73
|
attachments(options?: Parameters<typeof attachmentFiles>[1]): import("./attachments.ts").BoundAttachments;
|
|
73
74
|
knowledge(options?: Parameters<typeof knowledgeFiles>[1]): Pick<import("../../../spec/capabilities").CapabilityHandlers, "downloadKnowledge" | "listIndexing" | "listKnowledge" | "removeKnowledge" | "uploadKnowledge">;
|
|
74
75
|
listKnowledge(chatId: string): Promise<QvacKnowledge[]>;
|
|
@@ -84,6 +85,7 @@ export declare class Files extends EventEmitter {
|
|
|
84
85
|
syncAttachment(chunk: QvacChunk): Promise<void>;
|
|
85
86
|
syncKnowledge(row: QvacKnowledge): Promise<void>;
|
|
86
87
|
syncArtifact(row: QvacKnowledgeArtifact): Promise<void>;
|
|
88
|
+
reclaimPurged(): Promise<number>;
|
|
87
89
|
sweepTombstones(deviceId?: Buffer | null): Promise<void>;
|
|
88
90
|
sweepOrphans(coreLength: number): Promise<void>;
|
|
89
91
|
processorState(id: string): FileState;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
2
|
import EventEmitter from 'bare-events';
|
|
3
3
|
import { attachmentFiles } from './attachments.js';
|
|
4
|
-
import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
|
|
4
|
+
import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, reclaimPurgedFiles, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
|
|
5
5
|
import { knowledgeFiles } from './knowledge.js';
|
|
6
6
|
import { randomId } from '../id.js';
|
|
7
7
|
import { getChunk, listChunks } from '../chunk.js';
|
|
@@ -75,6 +75,9 @@ export class Files extends EventEmitter {
|
|
|
75
75
|
clear(reference, id) {
|
|
76
76
|
return this._store.clear(reference, id);
|
|
77
77
|
}
|
|
78
|
+
compact(coreKeys) {
|
|
79
|
+
return this._store.compact(coreKeys);
|
|
80
|
+
}
|
|
78
81
|
attachments(options = {}) {
|
|
79
82
|
return attachmentFiles(this._mesh, options);
|
|
80
83
|
}
|
|
@@ -160,6 +163,9 @@ export class Files extends EventEmitter {
|
|
|
160
163
|
syncArtifact(row) {
|
|
161
164
|
return syncArtifactArrival(this._mesh, row);
|
|
162
165
|
}
|
|
166
|
+
reclaimPurged() {
|
|
167
|
+
return reclaimPurgedFiles(this._mesh);
|
|
168
|
+
}
|
|
163
169
|
sweepTombstones(deviceId = null) {
|
|
164
170
|
return sweepTombstonedBlobs(this._mesh, deviceId);
|
|
165
171
|
}
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import type { QvacChunk } from '../../../spec/mesh/hyperschema/types.d.ts';
|
|
2
|
-
import type { FileKey, KnownFile } from '../local/files.ts';
|
|
2
|
+
import type { FileKey, KnownFile, LocalFile } from '../local/files.ts';
|
|
3
3
|
export type FileChunk = Pick<QvacChunk, 'type' | 'chatId' | 'attachment'>;
|
|
4
4
|
export interface MeshFileIndex {
|
|
5
5
|
record(chunks: ReadonlyArray<FileChunk>): Promise<void>;
|
|
6
6
|
refresh(chunks: ReadonlyArray<FileChunk>): Promise<void>;
|
|
7
7
|
read(reference: string): void;
|
|
8
8
|
markCleared(blobId: string): Promise<void>;
|
|
9
|
+
list(): Promise<LocalFile[]>;
|
|
10
|
+
retireChats(purged: ReadonlyMap<string, number>): Promise<void>;
|
|
11
|
+
forget(file: FileKey): Promise<void>;
|
|
9
12
|
}
|
|
10
13
|
export interface FileOwner {
|
|
11
14
|
readonly meshKey: Buffer;
|
|
@@ -3,6 +3,7 @@ import { type IndexingProgress } from '../rag/index.ts';
|
|
|
3
3
|
import { type FileInput, type FileRecord } from '../records/index.ts';
|
|
4
4
|
import type { Mesh } from '../../mesh.ts';
|
|
5
5
|
import { FileDocument } from './document.ts';
|
|
6
|
+
import type { LoadedArtifact } from './state.ts';
|
|
6
7
|
import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
7
8
|
export interface ProcessorHost {
|
|
8
9
|
readonly mesh: Mesh;
|
|
@@ -19,6 +20,7 @@ export interface FileProcessor {
|
|
|
19
20
|
canProduce(document: FileRecord, host: ProcessorHost): boolean | Promise<boolean>;
|
|
20
21
|
produce(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
21
22
|
load?(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
23
|
+
evict?(loaded: LoadedArtifact): Promise<void>;
|
|
22
24
|
close(): Promise<boolean>;
|
|
23
25
|
}
|
|
24
26
|
export interface FileProcessingOptions {
|
|
@@ -67,6 +69,7 @@ export declare class FileProcessing {
|
|
|
67
69
|
private _scan;
|
|
68
70
|
private _documents;
|
|
69
71
|
private _pump;
|
|
72
|
+
private _forget;
|
|
70
73
|
private _drain;
|
|
71
74
|
private _recordFailure;
|
|
72
75
|
private _queueLoad;
|
|
@@ -214,6 +214,12 @@ export class FileProcessing {
|
|
|
214
214
|
this._pump(lane);
|
|
215
215
|
});
|
|
216
216
|
}
|
|
217
|
+
async _forget(id) {
|
|
218
|
+
const loaded = this._mesh.files.processorState(this._processor.id).forget(id)?.loaded;
|
|
219
|
+
if (loaded) {
|
|
220
|
+
await this._processor.evict?.(loaded);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
217
223
|
async _drain(lane) {
|
|
218
224
|
while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
|
|
219
225
|
const [[id, wakes]] = lane.queue;
|
|
@@ -223,6 +229,9 @@ export class FileProcessing {
|
|
|
223
229
|
let produced = false;
|
|
224
230
|
try {
|
|
225
231
|
const record = await fileRecord(this._mesh.view, id, this._processor.input);
|
|
232
|
+
if (!record && lane.stage === 'load') {
|
|
233
|
+
await this._forget(id);
|
|
234
|
+
}
|
|
226
235
|
if (!record || record.deletedAt) {
|
|
227
236
|
await this._source.cancel((key) => key === id);
|
|
228
237
|
await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
|
|
@@ -55,6 +55,7 @@ export declare class FileState {
|
|
|
55
55
|
readonly outputRetries: FileRetries;
|
|
56
56
|
document(id: string): DocumentState;
|
|
57
57
|
artifact(id: string): ArtifactState;
|
|
58
|
+
forget(id: string): DocumentState | undefined;
|
|
58
59
|
get(id: string): Readonly<DocumentState> | undefined;
|
|
59
60
|
readArtifact(id: string): Readonly<ArtifactState> | undefined;
|
|
60
61
|
progress(chatId: string): IndexingProgressEntry[];
|
|
@@ -3,6 +3,7 @@ import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts
|
|
|
3
3
|
import { type RagExtractors, type VectorIndex } from 'rag-extract';
|
|
4
4
|
import { type IndexingEmbedder } from './produce.ts';
|
|
5
5
|
import type { FileDocument, FileProcessor, ProcessorHost } from '../processing/index.ts';
|
|
6
|
+
import type { LoadedArtifact } from '../processing/state.ts';
|
|
6
7
|
export interface RagProcessorOptions {
|
|
7
8
|
readonly model: string;
|
|
8
9
|
readonly embedder?: IndexingEmbedder;
|
|
@@ -21,6 +22,7 @@ export declare class RagProcessor implements FileProcessor {
|
|
|
21
22
|
applies(_document: QvacKnowledge): boolean;
|
|
22
23
|
canProduce(document: QvacKnowledge, host: ProcessorHost): Promise<boolean>;
|
|
23
24
|
produce(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
25
|
+
evict(loaded: LoadedArtifact): Promise<void>;
|
|
24
26
|
close(): Promise<boolean>;
|
|
25
27
|
}
|
|
26
28
|
export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
|
|
@@ -49,6 +49,12 @@ export class RagProcessor {
|
|
|
49
49
|
async produce(document, signal) {
|
|
50
50
|
await this._producer?.run(document, signal);
|
|
51
51
|
}
|
|
52
|
+
async evict(loaded) {
|
|
53
|
+
if (loaded.evict) {
|
|
54
|
+
await this._index?.remove(loaded.evict);
|
|
55
|
+
}
|
|
56
|
+
await this._index?.remove(loaded.artifactId);
|
|
57
|
+
}
|
|
52
58
|
async close() {
|
|
53
59
|
try {
|
|
54
60
|
await this._index?.clear();
|
|
@@ -5,6 +5,7 @@ export { groupAttachmentsByMessage } from './linkage.ts';
|
|
|
5
5
|
export { syncAttachmentChunk } from './chunks.ts';
|
|
6
6
|
export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from './reclaim.ts';
|
|
7
7
|
export { referencedBlobs } from './sweep.ts';
|
|
8
|
+
export { chatReferences, reclaimPurgedFiles, reclaimReferences } from './purge.ts';
|
|
8
9
|
export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from './output.ts';
|
|
9
10
|
export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from './source.ts';
|
|
10
11
|
export type { FileInput, FileRecord } from './source.ts';
|
|
@@ -4,5 +4,6 @@ export { groupAttachmentsByMessage } from './linkage.js';
|
|
|
4
4
|
export { syncAttachmentChunk } from './chunks.js';
|
|
5
5
|
export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from './reclaim.js';
|
|
6
6
|
export { referencedBlobs } from './sweep.js';
|
|
7
|
+
export { chatReferences, reclaimPurgedFiles, reclaimReferences } from './purge.js';
|
|
7
8
|
export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from './output.js';
|
|
8
9
|
export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from './source.js';
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Mesh } from '../../mesh.ts';
|
|
2
|
+
export declare function reclaimPurgedFiles(mesh: Mesh): Promise<number>;
|
|
3
|
+
export declare function chatReferences(mesh: Mesh, chatId: string): Promise<string[]>;
|
|
4
|
+
export declare function reclaimReferences(mesh: Mesh, references: ReadonlyArray<string>): Promise<void>;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import { fileKeyOf } from '../local-index.js';
|
|
3
|
+
import { referencedBlobs } from './sweep.js';
|
|
4
|
+
import { CHUNKS_BY_CHAT, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS } from '../../collections.js';
|
|
5
|
+
export async function reclaimPurgedFiles(mesh) {
|
|
6
|
+
const index = mesh.fileIndex;
|
|
7
|
+
if (!index) {
|
|
8
|
+
return 0;
|
|
9
|
+
}
|
|
10
|
+
const rows = await index.list();
|
|
11
|
+
const purged = await purgedChatIds(mesh, rows);
|
|
12
|
+
const candidates = rows.filter((row) => !!row.chatId && purged.has(row.chatId));
|
|
13
|
+
if (candidates.length === 0) {
|
|
14
|
+
return 0;
|
|
15
|
+
}
|
|
16
|
+
const owner = ownerOf(mesh);
|
|
17
|
+
const referenced = await referencedKeys(mesh, owner);
|
|
18
|
+
const cores = new Map();
|
|
19
|
+
let reclaimed = 0;
|
|
20
|
+
for (const row of candidates) {
|
|
21
|
+
if (mesh.closing)
|
|
22
|
+
break;
|
|
23
|
+
if (referenced.has(keyId(row)))
|
|
24
|
+
continue;
|
|
25
|
+
if (!(await mesh.files.clear(row.blobId, row.attachmentId ?? row.blobId))) {
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
await index.forget(row);
|
|
29
|
+
cores.set(b4a.toHex(row.coreKey), row.coreKey);
|
|
30
|
+
reclaimed++;
|
|
31
|
+
}
|
|
32
|
+
if (cores.size > 0 && !mesh.closing) {
|
|
33
|
+
await mesh.files.compact([...cores.values()]);
|
|
34
|
+
}
|
|
35
|
+
return reclaimed;
|
|
36
|
+
}
|
|
37
|
+
export async function chatReferences(mesh, chatId) {
|
|
38
|
+
const references = [];
|
|
39
|
+
const inChat = { gte: { chatId }, lte: { chatId } };
|
|
40
|
+
const snap = mesh.view.snapshot();
|
|
41
|
+
try {
|
|
42
|
+
for await (const chunk of snap.find(CHUNKS_BY_CHAT, inChat)) {
|
|
43
|
+
if (chunk.type === 'attachment' && chunk.attachment) {
|
|
44
|
+
references.push(chunk.attachment.blobId);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
for (const row of await snap.find(KNOWLEDGE_BY_CHAT, inChat).toArray()) {
|
|
48
|
+
references.push(row.blobId);
|
|
49
|
+
const ofKnowledge = { gte: { knowledgeId: row.id }, lte: { knowledgeId: row.id } };
|
|
50
|
+
for await (const artifact of snap.find(KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, ofKnowledge)) {
|
|
51
|
+
references.push(artifact.artifactBlobId);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
await snap.close();
|
|
57
|
+
}
|
|
58
|
+
return references;
|
|
59
|
+
}
|
|
60
|
+
export async function reclaimReferences(mesh, references) {
|
|
61
|
+
if (references.length === 0)
|
|
62
|
+
return;
|
|
63
|
+
const owner = ownerOf(mesh);
|
|
64
|
+
const referenced = await referencedKeys(mesh, owner);
|
|
65
|
+
const cores = new Map();
|
|
66
|
+
for (const reference of references) {
|
|
67
|
+
if (mesh.closing)
|
|
68
|
+
return;
|
|
69
|
+
const key = fileKeyOf(reference, owner);
|
|
70
|
+
if (!key || referenced.has(keyId(key))) {
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
if (!(await mesh.files.clear(reference, reference))) {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
await mesh.fileIndex?.forget(key);
|
|
77
|
+
cores.set(b4a.toHex(key.coreKey), key.coreKey);
|
|
78
|
+
}
|
|
79
|
+
if (cores.size > 0) {
|
|
80
|
+
void mesh.files.compact([...cores.values()]);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
async function purgedChatIds(mesh, rows) {
|
|
84
|
+
const purged = new Set();
|
|
85
|
+
const chatIds = rows.map((row) => row.chatId).filter((id) => !!id);
|
|
86
|
+
for (const chatId of new Set(chatIds)) {
|
|
87
|
+
if (await mesh.view.get(PURGED_CHATS, { chatId })) {
|
|
88
|
+
purged.add(chatId);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return purged;
|
|
92
|
+
}
|
|
93
|
+
function ownerOf(mesh) {
|
|
94
|
+
return { meshKey: mesh.key, ownCoreKey: mesh.blobs.core.key };
|
|
95
|
+
}
|
|
96
|
+
async function referencedKeys(mesh, owner) {
|
|
97
|
+
const keys = new Set();
|
|
98
|
+
for (const reference of await referencedBlobs(mesh)) {
|
|
99
|
+
const key = fileKeyOf(reference, owner);
|
|
100
|
+
if (key) {
|
|
101
|
+
keys.add(keyId(key));
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return keys;
|
|
105
|
+
}
|
|
106
|
+
function keyId(key) {
|
|
107
|
+
return `${b4a.toHex(key.coreKey)}/${key.blockOffset}`;
|
|
108
|
+
}
|
|
@@ -3,7 +3,7 @@ import { removeChatAttachments, removeChatKnowledge } from '../../chat.js';
|
|
|
3
3
|
import { listChunks } from '../../chunk.js';
|
|
4
4
|
import { getKnowledge, listKnowledgeArtifactsByKnowledge } from './knowledge.js';
|
|
5
5
|
import { syncAttachmentChunk } from './chunks.js';
|
|
6
|
-
import { CHATS, CHUNK_COUNTERS, KNOWLEDGE } from '../../collections.js';
|
|
6
|
+
import { CHATS, CHUNK_COUNTERS, KNOWLEDGE, PURGED_CHATS } from '../../collections.js';
|
|
7
7
|
export async function syncKnowledgeTombstone(mesh, row) {
|
|
8
8
|
if (!row.deletedAt)
|
|
9
9
|
return;
|
|
@@ -62,10 +62,12 @@ async function recascadeTombstonedChats(mesh, deviceId) {
|
|
|
62
62
|
const chats = mesh.view.snapshot();
|
|
63
63
|
try {
|
|
64
64
|
for await (const chat of chats.find(CHATS)) {
|
|
65
|
-
if (mesh.closing)
|
|
65
|
+
if (mesh.closing) {
|
|
66
66
|
return false;
|
|
67
|
-
|
|
67
|
+
}
|
|
68
|
+
if (!chat.deletedAt || (await chats.get(PURGED_CHATS, { chatId: chat.id }))) {
|
|
68
69
|
continue;
|
|
70
|
+
}
|
|
69
71
|
await removeChatAttachments(mesh, chat.id, { deviceId, at: chat.deletedAt });
|
|
70
72
|
await removeChatKnowledge(mesh, chat.id, chat.deletedAt);
|
|
71
73
|
}
|
|
@@ -27,3 +27,4 @@ export interface BlobReadChunk {
|
|
|
27
27
|
}
|
|
28
28
|
export declare function readBlob(blobs: Hyperblobs, blobId: BlobId, options?: ReadBlobOptions): AsyncGenerator<BlobReadChunk>;
|
|
29
29
|
export declare function clearBlob(mesh: BlobHost, blobId: string, id: string): Promise<boolean>;
|
|
30
|
+
export declare function compactBlobCores(mesh: BlobHost, coreKeys: ReadonlyArray<Buffer>): Promise<void>;
|