@qvac/core 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +112 -2
- package/dist-lib/lib/api/chat.d.ts +1 -1
- package/dist-lib/lib/api/chat.js +2 -1
- package/dist-lib/lib/api/index.d.ts +3 -1
- package/dist-lib/lib/api/index.js +2 -0
- package/dist-lib/lib/api/watchable.d.ts +1 -0
- package/dist-lib/lib/api/watchable.js +1 -1
- package/dist-lib/lib/assistant/harness.d.ts +2 -0
- package/dist-lib/lib/assistant/harness.js +13 -9
- package/dist-lib/lib/assistant-tree.js +4 -0
- package/dist-lib/lib/assistant.d.ts +5 -1
- package/dist-lib/lib/assistant.js +66 -20
- package/dist-lib/lib/client.js +2 -1
- package/dist-lib/lib/collabora/entry.js +13 -19
- package/dist-lib/lib/collabora/worker.js +77 -18
- package/dist-lib/lib/core/chat.d.ts +1 -0
- package/dist-lib/lib/core/chat.js +26 -4
- package/dist-lib/lib/core/collections.d.ts +1 -0
- package/dist-lib/lib/core/collections.js +1 -0
- package/dist-lib/lib/core/contract/chat.d.ts +2 -0
- package/dist-lib/lib/core/contract/chat.js +77 -4
- package/dist-lib/lib/core/contract/chunks.js +5 -0
- package/dist-lib/lib/core/contract/compute.d.ts +1 -1
- package/dist-lib/lib/core/contract/compute.js +2 -1
- package/dist-lib/lib/core/contract/tool-activity.js +5 -0
- package/dist-lib/lib/core/files/index.d.ts +2 -0
- package/dist-lib/lib/core/files/index.js +7 -1
- package/dist-lib/lib/core/files/local-index.d.ts +4 -1
- package/dist-lib/lib/core/files/processing/index.d.ts +3 -0
- package/dist-lib/lib/core/files/processing/index.js +9 -0
- package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/state.js +5 -0
- package/dist-lib/lib/core/files/rag/index.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/index.js +6 -0
- package/dist-lib/lib/core/files/records/index.d.ts +1 -0
- package/dist-lib/lib/core/files/records/index.js +1 -0
- package/dist-lib/lib/core/files/records/purge.d.ts +4 -0
- package/dist-lib/lib/core/files/records/purge.js +108 -0
- package/dist-lib/lib/core/files/records/reclaim.js +5 -3
- package/dist-lib/lib/core/files/store/blob.d.ts +1 -0
- package/dist-lib/lib/core/files/store/blob.js +29 -0
- package/dist-lib/lib/core/files/store/index.d.ts +1 -0
- package/dist-lib/lib/core/files/store/index.js +6 -1
- package/dist-lib/lib/core/local/files.d.ts +2 -0
- package/dist-lib/lib/core/local/files.js +31 -0
- package/dist-lib/lib/core/sync/index.js +14 -1
- package/dist-lib/lib/core.d.ts +12 -2
- package/dist-lib/lib/core.js +79 -16
- package/dist-lib/lib/credential-pusher.d.ts +8 -3
- package/dist-lib/lib/credential-pusher.js +4 -1
- package/dist-lib/lib/mesh-scoped-engine.js +3 -0
- package/dist-lib/lib/model-download.d.ts +11 -7
- package/dist-lib/lib/model-download.js +22 -21
- package/dist-lib/lib/models-watch.d.ts +6 -0
- package/dist-lib/lib/models-watch.js +45 -0
- package/dist-lib/schema/engine.d.ts +1 -0
- package/dist-lib/schema/engine.js +57 -3
- package/dist-lib/schema/field-order.js +3 -1
- package/dist-lib/schema/qvac.js +12 -1
- package/dist-lib/spec/bind.js +14 -0
- package/dist-lib/spec/calls.js +3 -0
- package/dist-lib/spec/capabilities.d.ts +6 -0
- package/dist-lib/spec/hrpc/hrpc.json +39 -0
- package/dist-lib/spec/hrpc/index.js +41 -4
- package/dist-lib/spec/hrpc/messages.js +788 -583
- package/dist-lib/spec/hyperschema/index.js +788 -583
- package/dist-lib/spec/hyperschema/schema.json +149 -0
- package/dist-lib/spec/hyperschema/types.d.ts +29 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +16 -1
- package/dist-lib/spec/mesh/hyperdb/index.js +71 -2
- package/dist-lib/spec/mesh/hyperdb/messages.js +434 -227
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
- package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +134 -5
- package/dist-lib/spec/mesh/hyperschema/index.js +134 -5
- package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +20 -1
- package/package.json +23 -10
- package/tui/app.mjs +3 -2
- package/tui/boot.mjs +1 -3
- package/dist-lib/lib/model-clamp.d.ts +0 -6
- package/dist-lib/lib/model-clamp.js +0 -23
|
@@ -2,17 +2,25 @@ import process from 'process';
|
|
|
2
2
|
import ScopeLock from 'scope-lock';
|
|
3
3
|
import safetyCatch from 'safety-catch';
|
|
4
4
|
import { fileURLToPath } from 'url';
|
|
5
|
+
import { Worker as Thread } from 'worker_threads';
|
|
5
6
|
import { asWorkerReply } from './protocol.js';
|
|
7
|
+
import { errorMessage } from '../guards.js';
|
|
6
8
|
const DEADLINE_MS = 60_000;
|
|
7
9
|
const lock = new ScopeLock();
|
|
8
10
|
let active = null;
|
|
9
11
|
export async function convertInWorker(job, signal) {
|
|
10
12
|
await lock.lock();
|
|
11
13
|
try {
|
|
12
|
-
const worker = active ?? (active =
|
|
14
|
+
const worker = active ?? (active = spawn());
|
|
13
15
|
const id = worker.nextId++;
|
|
14
|
-
worker.link.
|
|
15
|
-
|
|
16
|
+
worker.link.ref();
|
|
17
|
+
try {
|
|
18
|
+
worker.link.write({ id, ...job });
|
|
19
|
+
await settle(worker, id, signal);
|
|
20
|
+
}
|
|
21
|
+
finally {
|
|
22
|
+
worker.link.unref();
|
|
23
|
+
}
|
|
16
24
|
}
|
|
17
25
|
finally {
|
|
18
26
|
lock.unlock();
|
|
@@ -62,34 +70,85 @@ function abandon(worker) {
|
|
|
62
70
|
}
|
|
63
71
|
worker.link.kill();
|
|
64
72
|
}
|
|
65
|
-
|
|
73
|
+
function spawn() {
|
|
66
74
|
if (typeof Bare === 'undefined') {
|
|
67
75
|
throw new Error('the document engine needs the Bare runtime');
|
|
68
76
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
77
|
+
let thread;
|
|
78
|
+
try {
|
|
79
|
+
thread = new Thread(entry());
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
throw new Error(`the document engine failed to load: ${errorMessage(error)}`);
|
|
83
|
+
}
|
|
84
|
+
const inbox = new Inbox(thread);
|
|
85
|
+
thread.unref();
|
|
73
86
|
return {
|
|
74
87
|
nextId: 1,
|
|
75
88
|
link: {
|
|
76
|
-
read: () =>
|
|
77
|
-
write: (job) =>
|
|
89
|
+
read: () => inbox.read(),
|
|
90
|
+
write: (job) => thread.postMessage(job),
|
|
91
|
+
ref: () => thread.ref(),
|
|
92
|
+
unref: () => thread.unref(),
|
|
78
93
|
kill() {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}
|
|
82
|
-
catch (error) {
|
|
83
|
-
safetyCatch(error);
|
|
84
|
-
}
|
|
85
|
-
port.close().catch(safetyCatch);
|
|
94
|
+
thread.unref();
|
|
95
|
+
thread.terminate().catch(safetyCatch);
|
|
86
96
|
}
|
|
87
97
|
}
|
|
88
98
|
};
|
|
89
99
|
}
|
|
100
|
+
class Inbox {
|
|
101
|
+
replies = [];
|
|
102
|
+
waiting = [];
|
|
103
|
+
ended = false;
|
|
104
|
+
failure = null;
|
|
105
|
+
constructor(thread) {
|
|
106
|
+
thread.on('message', (reply) => this.push(reply));
|
|
107
|
+
thread.on('error', (error) => this.end(error));
|
|
108
|
+
thread.on('exit', () => this.end());
|
|
109
|
+
}
|
|
110
|
+
read() {
|
|
111
|
+
if (this.replies.length > 0) {
|
|
112
|
+
return Promise.resolve(this.replies.shift());
|
|
113
|
+
}
|
|
114
|
+
if (this.failure) {
|
|
115
|
+
return Promise.reject(this.failure);
|
|
116
|
+
}
|
|
117
|
+
if (this.ended) {
|
|
118
|
+
return Promise.resolve(null);
|
|
119
|
+
}
|
|
120
|
+
return new Promise((resolve, reject) => this.waiting.push({ resolve, reject }));
|
|
121
|
+
}
|
|
122
|
+
push(reply) {
|
|
123
|
+
const next = this.waiting.shift();
|
|
124
|
+
if (next) {
|
|
125
|
+
next.resolve(reply);
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
this.replies.push(reply);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
end(error) {
|
|
132
|
+
if (this.ended) {
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
this.ended = true;
|
|
136
|
+
if (error !== undefined) {
|
|
137
|
+
this.failure = new Error(`the document engine closed unexpectedly: ${errorMessage(error)}`);
|
|
138
|
+
}
|
|
139
|
+
for (const next of this.waiting.splice(0)) {
|
|
140
|
+
if (this.failure) {
|
|
141
|
+
next.reject(this.failure);
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
next.resolve(null);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
90
149
|
function entry() {
|
|
91
150
|
return (process.env.QVAC_COLLABORA_WORKER_ENTRY ||
|
|
92
|
-
fileURLToPath(
|
|
151
|
+
fileURLToPath(import.meta.resolve('#collabora-worker-entry')));
|
|
93
152
|
}
|
|
94
153
|
function deadlineMs() {
|
|
95
154
|
const ms = Number(process.env.QVAC_COLLABORA_TIMEOUT_MS);
|
|
@@ -38,6 +38,7 @@ export interface MoveChatOptions {
|
|
|
38
38
|
}
|
|
39
39
|
export declare function moveChat(store: Mesh, id: string, { groupId, afterId }: MoveChatOptions): Promise<ChatListing>;
|
|
40
40
|
export declare function setChatWebSearchEnabled(store: Mesh, id: string, webSearchEnabled: boolean): Promise<ChatListing>;
|
|
41
|
+
export declare function markChatRead(store: Mesh, id: string): Promise<ChatListing>;
|
|
41
42
|
export declare function removeChatKnowledge(store: Mesh, chatId: string, at: number): Promise<void>;
|
|
42
43
|
export interface RemoveChatAttachmentsOptions {
|
|
43
44
|
readonly deviceId: Buffer;
|
|
@@ -2,6 +2,7 @@ import { listChatGroupsIn, mustGetActiveChatGroup } from './chat-group.js';
|
|
|
2
2
|
import { appendChunk, listChunks, updateOp } from './chunk.js';
|
|
3
3
|
import { randomId } from './id.js';
|
|
4
4
|
import { assertRemovableAt } from './files/index.js';
|
|
5
|
+
import { chatReferences, reclaimReferences } from './files/records/purge.js';
|
|
5
6
|
import { place } from './rank.js';
|
|
6
7
|
import { chatSortKey } from './chat-order.js';
|
|
7
8
|
import b4a from 'b4a';
|
|
@@ -202,6 +203,23 @@ export async function moveChat(store, id, { groupId, afterId }) {
|
|
|
202
203
|
export function setChatWebSearchEnabled(store, id, webSearchEnabled) {
|
|
203
204
|
return patchChat(store, id, { webSearchEnabled });
|
|
204
205
|
}
|
|
206
|
+
export async function markChatRead(store, id) {
|
|
207
|
+
const chat = await mustGetActiveChat(store, id);
|
|
208
|
+
const snap = store.view.snapshot();
|
|
209
|
+
let count = 0;
|
|
210
|
+
try {
|
|
211
|
+
count = (await snap.get(CHUNK_COUNTERS, { chatId: id }))?.count ?? 0;
|
|
212
|
+
}
|
|
213
|
+
finally {
|
|
214
|
+
await snap.close();
|
|
215
|
+
}
|
|
216
|
+
if (count <= (chat.readThrough || 0)) {
|
|
217
|
+
return enrichChat(store, chat);
|
|
218
|
+
}
|
|
219
|
+
const read = { ...chat, readThrough: count };
|
|
220
|
+
await store.dispatch('put-chat', { chat: read });
|
|
221
|
+
return enrichChat(store, read);
|
|
222
|
+
}
|
|
205
223
|
async function cancelActiveRuns(store, deviceId, chatId) {
|
|
206
224
|
const statuses = await listChunks(store, { chatId, type: 'run-status' });
|
|
207
225
|
const terminal = new Set(statuses.filter((c) => TERMINAL_RUN_STATES.has(c.runStatus?.state ?? '')).map((c) => c.runId));
|
|
@@ -246,15 +264,19 @@ export async function removeChatAttachments(store, chatId, { deviceId, at }) {
|
|
|
246
264
|
}
|
|
247
265
|
export async function deleteChat(store, deviceId, id) {
|
|
248
266
|
const current = await mustGetChat(store, id);
|
|
267
|
+
const references = await chatReferences(store, id);
|
|
249
268
|
const now = Date.now();
|
|
250
269
|
if (deviceId) {
|
|
251
270
|
await cancelActiveRuns(store, deviceId, id);
|
|
252
|
-
await removeChatAttachments(store, id, { deviceId, at: now });
|
|
253
271
|
}
|
|
254
|
-
await removeChatKnowledge(store, id, now);
|
|
255
272
|
const deleted = { ...current, deletedAt: now, updatedAt: now };
|
|
256
|
-
await store
|
|
257
|
-
|
|
273
|
+
const listing = await enrichChat(store, deleted);
|
|
274
|
+
const batch = store.batch();
|
|
275
|
+
batch.dispatch('put-chat', { chat: deleted });
|
|
276
|
+
batch.dispatch('purge-chat', { chatId: id, at: now });
|
|
277
|
+
await batch.flush();
|
|
278
|
+
await reclaimReferences(store, references).catch((err) => logger.warn('sync', "reclaiming a deleted chat's files failed", { id, err }));
|
|
279
|
+
return listing;
|
|
258
280
|
}
|
|
259
281
|
export async function softDeleteChatsForAgent(store, deviceId, agentId) {
|
|
260
282
|
const snap = store.view.snapshot();
|
|
@@ -37,6 +37,7 @@ export declare const LIVE_WRITERS_BY_MOST_RECENT = "@qvac/live-writers-by-most-r
|
|
|
37
37
|
export declare const JOINED_MESHES = "@qvac/joined-meshes";
|
|
38
38
|
export declare const AGENTS_BY_PROVIDER = "@qvac/agents-by-provider";
|
|
39
39
|
export declare const DEFAULT_AGENTS = "@qvac/default-agents";
|
|
40
|
+
export declare const PURGED_CHATS = "@qvac/purged-chats";
|
|
40
41
|
export declare const LOCAL_CONFIG = "@local/config";
|
|
41
42
|
export declare const LOCAL_DEVICE = "@local/device";
|
|
42
43
|
export declare const LOCAL_APP_CONFIG = "@local/app-config";
|
|
@@ -37,6 +37,7 @@ export const LIVE_WRITERS_BY_MOST_RECENT = '@qvac/live-writers-by-most-recent';
|
|
|
37
37
|
export const JOINED_MESHES = '@qvac/joined-meshes';
|
|
38
38
|
export const AGENTS_BY_PROVIDER = '@qvac/agents-by-provider';
|
|
39
39
|
export const DEFAULT_AGENTS = '@qvac/default-agents';
|
|
40
|
+
export const PURGED_CHATS = '@qvac/purged-chats';
|
|
40
41
|
export const LOCAL_CONFIG = '@local/config';
|
|
41
42
|
export const LOCAL_DEVICE = '@local/device';
|
|
42
43
|
export const LOCAL_APP_CONFIG = '@local/app-config';
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { trackActivity } from './activity.js';
|
|
2
|
-
import { CHATS } from '../collections.js';
|
|
2
|
+
import { CHATS, CHUNKS, CHUNKS_BY_CHAT, CHUNK_COUNTERS, INDEXING_STATUS, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS, TOOL_ACTIVITY, TOOL_ACTIVITY_BY_AGENT } from '../collections.js';
|
|
3
|
+
export async function isPurged(tx, chatId) {
|
|
4
|
+
return (await tx.get(PURGED_CHATS, { chatId })) !== null;
|
|
5
|
+
}
|
|
3
6
|
export function registerChatRoutes(router) {
|
|
4
7
|
router.add('@qvac/put-chat', trackActivity(async ({ chat }, { tx, log }) => {
|
|
5
8
|
const existing = await tx.get(CHATS, { id: chat.id });
|
|
@@ -7,12 +10,82 @@ export function registerChatRoutes(router) {
|
|
|
7
10
|
log?.debug('contract', 'put-chat dropped: tombstone is sticky', { id: chat.id });
|
|
8
11
|
return;
|
|
9
12
|
}
|
|
13
|
+
const readThrough = Math.max(existing?.readThrough || 0, chat.readThrough || 0) || undefined;
|
|
10
14
|
if (existing && !chat.deletedAt && (existing.updatedAt ?? 0) > (chat.updatedAt ?? 0)) {
|
|
11
|
-
|
|
15
|
+
if (readThrough === (existing.readThrough || undefined)) {
|
|
16
|
+
log?.debug('contract', 'put-chat dropped: stale updatedAt', { id: chat.id });
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
await tx.insert(CHATS, { ...existing, readThrough });
|
|
20
|
+
log?.debug('contract', 'put-chat stale: read marker kept', { id: chat.id });
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
if (existing && (await isPurged(tx, chat.id))) {
|
|
24
|
+
log?.debug('contract', 'put-chat dropped: chat is purged', { id: chat.id });
|
|
12
25
|
return;
|
|
13
26
|
}
|
|
14
|
-
|
|
27
|
+
const row = { ...chat, readThrough };
|
|
28
|
+
await tx.insert(CHATS, row);
|
|
15
29
|
log?.debug('contract', existing ? 'chat updated' : 'chat created', { id: chat.id });
|
|
16
|
-
return
|
|
30
|
+
return row;
|
|
31
|
+
}));
|
|
32
|
+
router.add('@qvac/purge-chat', trackActivity(async ({ chatId, at }, { tx, log }) => {
|
|
33
|
+
const chat = await tx.get(CHATS, { id: chatId });
|
|
34
|
+
if (!chat?.deletedAt) {
|
|
35
|
+
log?.debug('contract', 'purge-chat dropped: chat is not deleted', { chatId });
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
if (await isPurged(tx, chatId)) {
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
await purgeChat(tx, chat);
|
|
42
|
+
const purged = { chatId, at: at || chat.deletedAt };
|
|
43
|
+
await tx.insert(PURGED_CHATS, purged);
|
|
44
|
+
log?.debug('contract', 'chat purged', { chatId });
|
|
45
|
+
return purged;
|
|
17
46
|
}));
|
|
18
47
|
}
|
|
48
|
+
async function purgeChat(tx, chat) {
|
|
49
|
+
const chatId = chat.id;
|
|
50
|
+
const inChat = { gte: { chatId }, lte: { chatId } };
|
|
51
|
+
for (const id of await idsOf(tx.find(CHUNKS_BY_CHAT, inChat))) {
|
|
52
|
+
await tx.delete(CHUNKS, { id });
|
|
53
|
+
}
|
|
54
|
+
await tx.delete(CHUNK_COUNTERS, { chatId });
|
|
55
|
+
for (const knowledge of await tx.find(KNOWLEDGE_BY_CHAT, inChat).toArray()) {
|
|
56
|
+
const ofKnowledge = { gte: { knowledgeId: knowledge.id }, lte: { knowledgeId: knowledge.id } };
|
|
57
|
+
const artifacts = tx.find(KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, ofKnowledge);
|
|
58
|
+
for (const artifact of await artifacts.toArray()) {
|
|
59
|
+
await tx.delete(KNOWLEDGE_ARTIFACTS, { id: artifact.id });
|
|
60
|
+
}
|
|
61
|
+
await tx.delete(KNOWLEDGE, { id: knowledge.id });
|
|
62
|
+
}
|
|
63
|
+
for (const status of await tx
|
|
64
|
+
.find(INDEXING_STATUS_BY_CHAT, inChat)
|
|
65
|
+
.toArray()) {
|
|
66
|
+
await tx.delete(INDEXING_STATUS, { id: status.id });
|
|
67
|
+
}
|
|
68
|
+
const ofAgent = { gte: { agentId: chat.agentId }, lte: { agentId: chat.agentId } };
|
|
69
|
+
const activity = tx.find(TOOL_ACTIVITY_BY_AGENT, ofAgent);
|
|
70
|
+
for (const id of await idsOf(activity, (row) => row.chatId === chatId)) {
|
|
71
|
+
await tx.delete(TOOL_ACTIVITY, { id });
|
|
72
|
+
}
|
|
73
|
+
const tombstone = {
|
|
74
|
+
id: chatId,
|
|
75
|
+
agentId: chat.agentId,
|
|
76
|
+
title: '',
|
|
77
|
+
createdAt: chat.createdAt,
|
|
78
|
+
deletedAt: chat.deletedAt,
|
|
79
|
+
updatedAt: chat.updatedAt
|
|
80
|
+
};
|
|
81
|
+
await tx.insert(CHATS, tombstone);
|
|
82
|
+
}
|
|
83
|
+
async function idsOf(rows, keep = () => true) {
|
|
84
|
+
const ids = [];
|
|
85
|
+
for await (const row of rows) {
|
|
86
|
+
if (keep(row)) {
|
|
87
|
+
ids.push(row.id);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return ids;
|
|
91
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { devicesByWriterKey, trackActivity } from './activity.js';
|
|
2
|
+
import { isPurged } from './chat.js';
|
|
2
3
|
import { CHUNK_PAYLOAD_FIELDS, ragSourcesWithinBounds } from '../../../schema/qvac.js';
|
|
3
4
|
import { chunkTypeOrdinal } from '../../../schema/indexes.js';
|
|
4
5
|
import b4a from 'b4a';
|
|
@@ -44,6 +45,10 @@ export function registerChunkRoutes(router) {
|
|
|
44
45
|
if (chunk) {
|
|
45
46
|
return;
|
|
46
47
|
}
|
|
48
|
+
if (await isPurged(tx, op.chatId)) {
|
|
49
|
+
log?.debug('contract', 'append-chunk dropped: chat is purged', { id: op.id });
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
47
52
|
if (!payloadMatchesType(op)) {
|
|
48
53
|
log?.warn('contract', 'append-chunk dropped: payload does not match its type', {
|
|
49
54
|
id: op.id
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Router } from '../../../spec/mesh/hyperdispatch/index.js';
|
|
2
|
-
declare const DROPPED: readonly ['@qvac/append-chunk', '@qvac/update-chunk', '@qvac/resolve-approval', '@qvac/put-agent', '@qvac/put-approval-rule', '@qvac/seed-defaults', '@qvac/add-knowledge', '@qvac/remove-knowledge', '@qvac/add-knowledge-artifact', '@qvac/claim-indexing', '@qvac/resolve-indexing', '@qvac/put-chat', '@qvac/put-tool-configs', '@qvac/append-tool-activity', '@qvac/migrate-agent-catalog', '@qvac/reindex-agents', '@qvac/put-chat-group', '@qvac/put-joined-mesh'];
|
|
2
|
+
declare const DROPPED: readonly ['@qvac/append-chunk', '@qvac/update-chunk', '@qvac/resolve-approval', '@qvac/put-agent', '@qvac/put-approval-rule', '@qvac/seed-defaults', '@qvac/add-knowledge', '@qvac/remove-knowledge', '@qvac/add-knowledge-artifact', '@qvac/claim-indexing', '@qvac/resolve-indexing', '@qvac/put-chat', '@qvac/put-tool-configs', '@qvac/append-tool-activity', '@qvac/migrate-agent-catalog', '@qvac/reindex-agents', '@qvac/put-chat-group', '@qvac/put-joined-mesh', '@qvac/purge-chat'];
|
|
3
3
|
export type DroppedComputeRoute = (typeof DROPPED)[number];
|
|
4
4
|
export declare function droppedComputeRoutes(): ReadonlyArray<DroppedComputeRoute>;
|
|
5
5
|
export declare class ComputeRouteDropped extends Error {
|
|
@@ -16,7 +16,8 @@ const DROPPED = [
|
|
|
16
16
|
'@qvac/migrate-agent-catalog',
|
|
17
17
|
'@qvac/reindex-agents',
|
|
18
18
|
'@qvac/put-chat-group',
|
|
19
|
-
'@qvac/put-joined-mesh'
|
|
19
|
+
'@qvac/put-joined-mesh',
|
|
20
|
+
'@qvac/purge-chat'
|
|
20
21
|
];
|
|
21
22
|
const DROPPED_ROUTES = new Set(DROPPED);
|
|
22
23
|
export function droppedComputeRoutes() {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isPurged } from './chat.js';
|
|
1
2
|
import { capToolActivityArguments } from '../../../schema/qvac.js';
|
|
2
3
|
import { TOOL_ACTIVITY, TOOL_ACTIVITY_COUNTER } from '../collections.js';
|
|
3
4
|
export function registerToolActivityRoutes(router) {
|
|
@@ -6,6 +7,10 @@ export function registerToolActivityRoutes(router) {
|
|
|
6
7
|
if (existing) {
|
|
7
8
|
return;
|
|
8
9
|
}
|
|
10
|
+
if (activity.chatId && (await isPurged(tx, activity.chatId))) {
|
|
11
|
+
log?.debug('contract', 'tool activity dropped: chat is purged', { id: activity.id });
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
9
14
|
const counter = await tx.get(TOOL_ACTIVITY_COUNTER, {
|
|
10
15
|
agentId: activity.agentId
|
|
11
16
|
});
|
|
@@ -69,6 +69,7 @@ export declare class Files extends EventEmitter {
|
|
|
69
69
|
inline(bytes: Buffer, mimeType: string): string;
|
|
70
70
|
reference(id: Parameters<BlobStore['reference']>[0]): string;
|
|
71
71
|
clear(reference: string, id: string): Promise<boolean>;
|
|
72
|
+
compact(coreKeys: ReadonlyArray<Buffer>): Promise<void>;
|
|
72
73
|
attachments(options?: Parameters<typeof attachmentFiles>[1]): import("./attachments.ts").BoundAttachments;
|
|
73
74
|
knowledge(options?: Parameters<typeof knowledgeFiles>[1]): Pick<import("../../../spec/capabilities").CapabilityHandlers, "downloadKnowledge" | "listIndexing" | "listKnowledge" | "removeKnowledge" | "uploadKnowledge">;
|
|
74
75
|
listKnowledge(chatId: string): Promise<QvacKnowledge[]>;
|
|
@@ -84,6 +85,7 @@ export declare class Files extends EventEmitter {
|
|
|
84
85
|
syncAttachment(chunk: QvacChunk): Promise<void>;
|
|
85
86
|
syncKnowledge(row: QvacKnowledge): Promise<void>;
|
|
86
87
|
syncArtifact(row: QvacKnowledgeArtifact): Promise<void>;
|
|
88
|
+
reclaimPurged(): Promise<number>;
|
|
87
89
|
sweepTombstones(deviceId?: Buffer | null): Promise<void>;
|
|
88
90
|
sweepOrphans(coreLength: number): Promise<void>;
|
|
89
91
|
processorState(id: string): FileState;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
2
|
import EventEmitter from 'bare-events';
|
|
3
3
|
import { attachmentFiles } from './attachments.js';
|
|
4
|
-
import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
|
|
4
|
+
import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, reclaimPurgedFiles, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
|
|
5
5
|
import { knowledgeFiles } from './knowledge.js';
|
|
6
6
|
import { randomId } from '../id.js';
|
|
7
7
|
import { getChunk, listChunks } from '../chunk.js';
|
|
@@ -75,6 +75,9 @@ export class Files extends EventEmitter {
|
|
|
75
75
|
clear(reference, id) {
|
|
76
76
|
return this._store.clear(reference, id);
|
|
77
77
|
}
|
|
78
|
+
compact(coreKeys) {
|
|
79
|
+
return this._store.compact(coreKeys);
|
|
80
|
+
}
|
|
78
81
|
attachments(options = {}) {
|
|
79
82
|
return attachmentFiles(this._mesh, options);
|
|
80
83
|
}
|
|
@@ -160,6 +163,9 @@ export class Files extends EventEmitter {
|
|
|
160
163
|
syncArtifact(row) {
|
|
161
164
|
return syncArtifactArrival(this._mesh, row);
|
|
162
165
|
}
|
|
166
|
+
reclaimPurged() {
|
|
167
|
+
return reclaimPurgedFiles(this._mesh);
|
|
168
|
+
}
|
|
163
169
|
sweepTombstones(deviceId = null) {
|
|
164
170
|
return sweepTombstonedBlobs(this._mesh, deviceId);
|
|
165
171
|
}
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import type { QvacChunk } from '../../../spec/mesh/hyperschema/types.d.ts';
|
|
2
|
-
import type { FileKey, KnownFile } from '../local/files.ts';
|
|
2
|
+
import type { FileKey, KnownFile, LocalFile } from '../local/files.ts';
|
|
3
3
|
export type FileChunk = Pick<QvacChunk, 'type' | 'chatId' | 'attachment'>;
|
|
4
4
|
export interface MeshFileIndex {
|
|
5
5
|
record(chunks: ReadonlyArray<FileChunk>): Promise<void>;
|
|
6
6
|
refresh(chunks: ReadonlyArray<FileChunk>): Promise<void>;
|
|
7
7
|
read(reference: string): void;
|
|
8
8
|
markCleared(blobId: string): Promise<void>;
|
|
9
|
+
list(): Promise<LocalFile[]>;
|
|
10
|
+
retireChats(purged: ReadonlyMap<string, number>): Promise<void>;
|
|
11
|
+
forget(file: FileKey): Promise<void>;
|
|
9
12
|
}
|
|
10
13
|
export interface FileOwner {
|
|
11
14
|
readonly meshKey: Buffer;
|
|
@@ -3,6 +3,7 @@ import { type IndexingProgress } from '../rag/index.ts';
|
|
|
3
3
|
import { type FileInput, type FileRecord } from '../records/index.ts';
|
|
4
4
|
import type { Mesh } from '../../mesh.ts';
|
|
5
5
|
import { FileDocument } from './document.ts';
|
|
6
|
+
import type { LoadedArtifact } from './state.ts';
|
|
6
7
|
import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
7
8
|
export interface ProcessorHost {
|
|
8
9
|
readonly mesh: Mesh;
|
|
@@ -19,6 +20,7 @@ export interface FileProcessor {
|
|
|
19
20
|
canProduce(document: FileRecord, host: ProcessorHost): boolean | Promise<boolean>;
|
|
20
21
|
produce(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
21
22
|
load?(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
23
|
+
evict?(loaded: LoadedArtifact): Promise<void>;
|
|
22
24
|
close(): Promise<boolean>;
|
|
23
25
|
}
|
|
24
26
|
export interface FileProcessingOptions {
|
|
@@ -67,6 +69,7 @@ export declare class FileProcessing {
|
|
|
67
69
|
private _scan;
|
|
68
70
|
private _documents;
|
|
69
71
|
private _pump;
|
|
72
|
+
private _forget;
|
|
70
73
|
private _drain;
|
|
71
74
|
private _recordFailure;
|
|
72
75
|
private _queueLoad;
|
|
@@ -214,6 +214,12 @@ export class FileProcessing {
|
|
|
214
214
|
this._pump(lane);
|
|
215
215
|
});
|
|
216
216
|
}
|
|
217
|
+
async _forget(id) {
|
|
218
|
+
const loaded = this._mesh.files.processorState(this._processor.id).forget(id)?.loaded;
|
|
219
|
+
if (loaded) {
|
|
220
|
+
await this._processor.evict?.(loaded);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
217
223
|
async _drain(lane) {
|
|
218
224
|
while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
|
|
219
225
|
const [[id, wakes]] = lane.queue;
|
|
@@ -223,6 +229,9 @@ export class FileProcessing {
|
|
|
223
229
|
let produced = false;
|
|
224
230
|
try {
|
|
225
231
|
const record = await fileRecord(this._mesh.view, id, this._processor.input);
|
|
232
|
+
if (!record && lane.stage === 'load') {
|
|
233
|
+
await this._forget(id);
|
|
234
|
+
}
|
|
226
235
|
if (!record || record.deletedAt) {
|
|
227
236
|
await this._source.cancel((key) => key === id);
|
|
228
237
|
await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
|
|
@@ -55,6 +55,7 @@ export declare class FileState {
|
|
|
55
55
|
readonly outputRetries: FileRetries;
|
|
56
56
|
document(id: string): DocumentState;
|
|
57
57
|
artifact(id: string): ArtifactState;
|
|
58
|
+
forget(id: string): DocumentState | undefined;
|
|
58
59
|
get(id: string): Readonly<DocumentState> | undefined;
|
|
59
60
|
readArtifact(id: string): Readonly<ArtifactState> | undefined;
|
|
60
61
|
progress(chatId: string): IndexingProgressEntry[];
|
|
@@ -3,6 +3,7 @@ import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts
|
|
|
3
3
|
import { type RagExtractors, type VectorIndex } from 'rag-extract';
|
|
4
4
|
import { type IndexingEmbedder } from './produce.ts';
|
|
5
5
|
import type { FileDocument, FileProcessor, ProcessorHost } from '../processing/index.ts';
|
|
6
|
+
import type { LoadedArtifact } from '../processing/state.ts';
|
|
6
7
|
export interface RagProcessorOptions {
|
|
7
8
|
readonly model: string;
|
|
8
9
|
readonly embedder?: IndexingEmbedder;
|
|
@@ -21,6 +22,7 @@ export declare class RagProcessor implements FileProcessor {
|
|
|
21
22
|
applies(_document: QvacKnowledge): boolean;
|
|
22
23
|
canProduce(document: QvacKnowledge, host: ProcessorHost): Promise<boolean>;
|
|
23
24
|
produce(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
25
|
+
evict(loaded: LoadedArtifact): Promise<void>;
|
|
24
26
|
close(): Promise<boolean>;
|
|
25
27
|
}
|
|
26
28
|
export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
|
|
@@ -49,6 +49,12 @@ export class RagProcessor {
|
|
|
49
49
|
async produce(document, signal) {
|
|
50
50
|
await this._producer?.run(document, signal);
|
|
51
51
|
}
|
|
52
|
+
async evict(loaded) {
|
|
53
|
+
if (loaded.evict) {
|
|
54
|
+
await this._index?.remove(loaded.evict);
|
|
55
|
+
}
|
|
56
|
+
await this._index?.remove(loaded.artifactId);
|
|
57
|
+
}
|
|
52
58
|
async close() {
|
|
53
59
|
try {
|
|
54
60
|
await this._index?.clear();
|
|
@@ -5,6 +5,7 @@ export { groupAttachmentsByMessage } from './linkage.ts';
|
|
|
5
5
|
export { syncAttachmentChunk } from './chunks.ts';
|
|
6
6
|
export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from './reclaim.ts';
|
|
7
7
|
export { referencedBlobs } from './sweep.ts';
|
|
8
|
+
export { chatReferences, reclaimPurgedFiles, reclaimReferences } from './purge.ts';
|
|
8
9
|
export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from './output.ts';
|
|
9
10
|
export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from './source.ts';
|
|
10
11
|
export type { FileInput, FileRecord } from './source.ts';
|
|
@@ -4,5 +4,6 @@ export { groupAttachmentsByMessage } from './linkage.js';
|
|
|
4
4
|
export { syncAttachmentChunk } from './chunks.js';
|
|
5
5
|
export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from './reclaim.js';
|
|
6
6
|
export { referencedBlobs } from './sweep.js';
|
|
7
|
+
export { chatReferences, reclaimPurgedFiles, reclaimReferences } from './purge.js';
|
|
7
8
|
export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from './output.js';
|
|
8
9
|
export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from './source.js';
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Mesh } from '../../mesh.ts';
|
|
2
|
+
export declare function reclaimPurgedFiles(mesh: Mesh): Promise<number>;
|
|
3
|
+
export declare function chatReferences(mesh: Mesh, chatId: string): Promise<string[]>;
|
|
4
|
+
export declare function reclaimReferences(mesh: Mesh, references: ReadonlyArray<string>): Promise<void>;
|