@qvac/core 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +112 -2
- package/dist-lib/lib/api/chat.d.ts +1 -1
- package/dist-lib/lib/api/chat.js +2 -1
- package/dist-lib/lib/api/index.d.ts +3 -1
- package/dist-lib/lib/api/index.js +2 -0
- package/dist-lib/lib/api/watchable.d.ts +1 -0
- package/dist-lib/lib/api/watchable.js +1 -1
- package/dist-lib/lib/assistant/harness.d.ts +2 -0
- package/dist-lib/lib/assistant/harness.js +13 -9
- package/dist-lib/lib/assistant-tree.js +4 -0
- package/dist-lib/lib/assistant.d.ts +5 -1
- package/dist-lib/lib/assistant.js +66 -20
- package/dist-lib/lib/client.js +2 -1
- package/dist-lib/lib/collabora/entry.js +13 -19
- package/dist-lib/lib/collabora/worker.js +77 -18
- package/dist-lib/lib/core/chat.d.ts +1 -0
- package/dist-lib/lib/core/chat.js +26 -4
- package/dist-lib/lib/core/collections.d.ts +1 -0
- package/dist-lib/lib/core/collections.js +1 -0
- package/dist-lib/lib/core/contract/chat.d.ts +2 -0
- package/dist-lib/lib/core/contract/chat.js +77 -4
- package/dist-lib/lib/core/contract/chunks.js +5 -0
- package/dist-lib/lib/core/contract/compute.d.ts +1 -1
- package/dist-lib/lib/core/contract/compute.js +2 -1
- package/dist-lib/lib/core/contract/tool-activity.js +5 -0
- package/dist-lib/lib/core/files/index.d.ts +2 -0
- package/dist-lib/lib/core/files/index.js +7 -1
- package/dist-lib/lib/core/files/local-index.d.ts +4 -1
- package/dist-lib/lib/core/files/processing/index.d.ts +3 -0
- package/dist-lib/lib/core/files/processing/index.js +9 -0
- package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/state.js +5 -0
- package/dist-lib/lib/core/files/rag/index.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/index.js +6 -0
- package/dist-lib/lib/core/files/records/index.d.ts +1 -0
- package/dist-lib/lib/core/files/records/index.js +1 -0
- package/dist-lib/lib/core/files/records/purge.d.ts +4 -0
- package/dist-lib/lib/core/files/records/purge.js +108 -0
- package/dist-lib/lib/core/files/records/reclaim.js +5 -3
- package/dist-lib/lib/core/files/store/blob.d.ts +1 -0
- package/dist-lib/lib/core/files/store/blob.js +29 -0
- package/dist-lib/lib/core/files/store/index.d.ts +1 -0
- package/dist-lib/lib/core/files/store/index.js +6 -1
- package/dist-lib/lib/core/local/files.d.ts +2 -0
- package/dist-lib/lib/core/local/files.js +31 -0
- package/dist-lib/lib/core/sync/index.js +14 -1
- package/dist-lib/lib/core.d.ts +12 -2
- package/dist-lib/lib/core.js +79 -16
- package/dist-lib/lib/credential-pusher.d.ts +8 -3
- package/dist-lib/lib/credential-pusher.js +4 -1
- package/dist-lib/lib/mesh-scoped-engine.js +3 -0
- package/dist-lib/lib/model-download.d.ts +11 -7
- package/dist-lib/lib/model-download.js +22 -21
- package/dist-lib/lib/models-watch.d.ts +6 -0
- package/dist-lib/lib/models-watch.js +45 -0
- package/dist-lib/schema/engine.d.ts +1 -0
- package/dist-lib/schema/engine.js +57 -3
- package/dist-lib/schema/field-order.js +3 -1
- package/dist-lib/schema/qvac.js +12 -1
- package/dist-lib/spec/bind.js +14 -0
- package/dist-lib/spec/calls.js +3 -0
- package/dist-lib/spec/capabilities.d.ts +6 -0
- package/dist-lib/spec/hrpc/hrpc.json +39 -0
- package/dist-lib/spec/hrpc/index.js +41 -4
- package/dist-lib/spec/hrpc/messages.js +788 -583
- package/dist-lib/spec/hyperschema/index.js +788 -583
- package/dist-lib/spec/hyperschema/schema.json +149 -0
- package/dist-lib/spec/hyperschema/types.d.ts +29 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +16 -1
- package/dist-lib/spec/mesh/hyperdb/index.js +71 -2
- package/dist-lib/spec/mesh/hyperdb/messages.js +434 -227
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
- package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +134 -5
- package/dist-lib/spec/mesh/hyperschema/index.js +134 -5
- package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +20 -1
- package/package.json +23 -10
- package/tui/app.mjs +3 -2
- package/tui/boot.mjs +1 -3
- package/dist-lib/lib/model-clamp.d.ts +0 -6
- package/dist-lib/lib/model-clamp.js +0 -23
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import { fileKeyOf } from '../local-index.js';
|
|
3
|
+
import { referencedBlobs } from './sweep.js';
|
|
4
|
+
import { CHUNKS_BY_CHAT, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS } from '../../collections.js';
|
|
5
|
+
export async function reclaimPurgedFiles(mesh) {
|
|
6
|
+
const index = mesh.fileIndex;
|
|
7
|
+
if (!index) {
|
|
8
|
+
return 0;
|
|
9
|
+
}
|
|
10
|
+
const rows = await index.list();
|
|
11
|
+
const purged = await purgedChatIds(mesh, rows);
|
|
12
|
+
const candidates = rows.filter((row) => !!row.chatId && purged.has(row.chatId));
|
|
13
|
+
if (candidates.length === 0) {
|
|
14
|
+
return 0;
|
|
15
|
+
}
|
|
16
|
+
const owner = ownerOf(mesh);
|
|
17
|
+
const referenced = await referencedKeys(mesh, owner);
|
|
18
|
+
const cores = new Map();
|
|
19
|
+
let reclaimed = 0;
|
|
20
|
+
for (const row of candidates) {
|
|
21
|
+
if (mesh.closing)
|
|
22
|
+
break;
|
|
23
|
+
if (referenced.has(keyId(row)))
|
|
24
|
+
continue;
|
|
25
|
+
if (!(await mesh.files.clear(row.blobId, row.attachmentId ?? row.blobId))) {
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
await index.forget(row);
|
|
29
|
+
cores.set(b4a.toHex(row.coreKey), row.coreKey);
|
|
30
|
+
reclaimed++;
|
|
31
|
+
}
|
|
32
|
+
if (cores.size > 0 && !mesh.closing) {
|
|
33
|
+
await mesh.files.compact([...cores.values()]);
|
|
34
|
+
}
|
|
35
|
+
return reclaimed;
|
|
36
|
+
}
|
|
37
|
+
export async function chatReferences(mesh, chatId) {
|
|
38
|
+
const references = [];
|
|
39
|
+
const inChat = { gte: { chatId }, lte: { chatId } };
|
|
40
|
+
const snap = mesh.view.snapshot();
|
|
41
|
+
try {
|
|
42
|
+
for await (const chunk of snap.find(CHUNKS_BY_CHAT, inChat)) {
|
|
43
|
+
if (chunk.type === 'attachment' && chunk.attachment) {
|
|
44
|
+
references.push(chunk.attachment.blobId);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
for (const row of await snap.find(KNOWLEDGE_BY_CHAT, inChat).toArray()) {
|
|
48
|
+
references.push(row.blobId);
|
|
49
|
+
const ofKnowledge = { gte: { knowledgeId: row.id }, lte: { knowledgeId: row.id } };
|
|
50
|
+
for await (const artifact of snap.find(KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, ofKnowledge)) {
|
|
51
|
+
references.push(artifact.artifactBlobId);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
await snap.close();
|
|
57
|
+
}
|
|
58
|
+
return references;
|
|
59
|
+
}
|
|
60
|
+
export async function reclaimReferences(mesh, references) {
|
|
61
|
+
if (references.length === 0)
|
|
62
|
+
return;
|
|
63
|
+
const owner = ownerOf(mesh);
|
|
64
|
+
const referenced = await referencedKeys(mesh, owner);
|
|
65
|
+
const cores = new Map();
|
|
66
|
+
for (const reference of references) {
|
|
67
|
+
if (mesh.closing)
|
|
68
|
+
return;
|
|
69
|
+
const key = fileKeyOf(reference, owner);
|
|
70
|
+
if (!key || referenced.has(keyId(key))) {
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
if (!(await mesh.files.clear(reference, reference))) {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
await mesh.fileIndex?.forget(key);
|
|
77
|
+
cores.set(b4a.toHex(key.coreKey), key.coreKey);
|
|
78
|
+
}
|
|
79
|
+
if (cores.size > 0) {
|
|
80
|
+
void mesh.files.compact([...cores.values()]);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
async function purgedChatIds(mesh, rows) {
|
|
84
|
+
const purged = new Set();
|
|
85
|
+
const chatIds = rows.map((row) => row.chatId).filter((id) => !!id);
|
|
86
|
+
for (const chatId of new Set(chatIds)) {
|
|
87
|
+
if (await mesh.view.get(PURGED_CHATS, { chatId })) {
|
|
88
|
+
purged.add(chatId);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return purged;
|
|
92
|
+
}
|
|
93
|
+
function ownerOf(mesh) {
|
|
94
|
+
return { meshKey: mesh.key, ownCoreKey: mesh.blobs.core.key };
|
|
95
|
+
}
|
|
96
|
+
async function referencedKeys(mesh, owner) {
|
|
97
|
+
const keys = new Set();
|
|
98
|
+
for (const reference of await referencedBlobs(mesh)) {
|
|
99
|
+
const key = fileKeyOf(reference, owner);
|
|
100
|
+
if (key) {
|
|
101
|
+
keys.add(keyId(key));
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return keys;
|
|
105
|
+
}
|
|
106
|
+
function keyId(key) {
|
|
107
|
+
return `${b4a.toHex(key.coreKey)}/${key.blockOffset}`;
|
|
108
|
+
}
|
|
@@ -3,7 +3,7 @@ import { removeChatAttachments, removeChatKnowledge } from '../../chat.js';
|
|
|
3
3
|
import { listChunks } from '../../chunk.js';
|
|
4
4
|
import { getKnowledge, listKnowledgeArtifactsByKnowledge } from './knowledge.js';
|
|
5
5
|
import { syncAttachmentChunk } from './chunks.js';
|
|
6
|
-
import { CHATS, CHUNK_COUNTERS, KNOWLEDGE } from '../../collections.js';
|
|
6
|
+
import { CHATS, CHUNK_COUNTERS, KNOWLEDGE, PURGED_CHATS } from '../../collections.js';
|
|
7
7
|
export async function syncKnowledgeTombstone(mesh, row) {
|
|
8
8
|
if (!row.deletedAt)
|
|
9
9
|
return;
|
|
@@ -62,10 +62,12 @@ async function recascadeTombstonedChats(mesh, deviceId) {
|
|
|
62
62
|
const chats = mesh.view.snapshot();
|
|
63
63
|
try {
|
|
64
64
|
for await (const chat of chats.find(CHATS)) {
|
|
65
|
-
if (mesh.closing)
|
|
65
|
+
if (mesh.closing) {
|
|
66
66
|
return false;
|
|
67
|
-
|
|
67
|
+
}
|
|
68
|
+
if (!chat.deletedAt || (await chats.get(PURGED_CHATS, { chatId: chat.id }))) {
|
|
68
69
|
continue;
|
|
70
|
+
}
|
|
69
71
|
await removeChatAttachments(mesh, chat.id, { deviceId, at: chat.deletedAt });
|
|
70
72
|
await removeChatKnowledge(mesh, chat.id, chat.deletedAt);
|
|
71
73
|
}
|
|
@@ -27,3 +27,4 @@ export interface BlobReadChunk {
|
|
|
27
27
|
}
|
|
28
28
|
export declare function readBlob(blobs: Hyperblobs, blobId: BlobId, options?: ReadBlobOptions): AsyncGenerator<BlobReadChunk>;
|
|
29
29
|
export declare function clearBlob(mesh: BlobHost, blobId: string, id: string): Promise<boolean>;
|
|
30
|
+
export declare function compactBlobCores(mesh: BlobHost, coreKeys: ReadonlyArray<Buffer>): Promise<void>;
|
|
@@ -105,3 +105,32 @@ async function clearBlobRef(mesh, reference) {
|
|
|
105
105
|
await remote.close();
|
|
106
106
|
}
|
|
107
107
|
}
|
|
108
|
+
function compactable(core) {
|
|
109
|
+
return 'compact' in core && typeof core.compact === 'function';
|
|
110
|
+
}
|
|
111
|
+
async function compact(core) {
|
|
112
|
+
if (!compactable(core)) {
|
|
113
|
+
throw new Error('this hypercore cannot compact');
|
|
114
|
+
}
|
|
115
|
+
await core.compact();
|
|
116
|
+
}
|
|
117
|
+
export async function compactBlobCores(mesh, coreKeys) {
|
|
118
|
+
for (const key of coreKeys) {
|
|
119
|
+
try {
|
|
120
|
+
if (b4a.equals(key, mesh.blobs.core.key)) {
|
|
121
|
+
await compact(mesh.blobs.core);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const remote = await mesh.openBlobs(key);
|
|
125
|
+
try {
|
|
126
|
+
await compact(remote.core);
|
|
127
|
+
}
|
|
128
|
+
finally {
|
|
129
|
+
await remote.close();
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
logger.warn('sync', 'blob compaction failed', { coreKey: b4a.toHex(key), err });
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
@@ -26,6 +26,7 @@ export declare class BlobStore extends EventEmitter {
|
|
|
26
26
|
readBytes(reference: string, options?: Parameters<typeof readBlobReferenceBytes>[2]): Promise<Buffer<ArrayBufferLike>>;
|
|
27
27
|
download(reference: string, options: Parameters<typeof downloadBlobReference>[2]): Promise<void>;
|
|
28
28
|
clear(reference: string, id: string): Promise<boolean>;
|
|
29
|
+
compact(coreKeys: ReadonlyArray<Buffer>): Promise<void>;
|
|
29
30
|
available(reference: string): Promise<boolean>;
|
|
30
31
|
sweep(references: string[], coreLength: number): Promise<void>;
|
|
31
32
|
close(): Promise<void>;
|
|
@@ -2,7 +2,7 @@ import b4a from 'b4a';
|
|
|
2
2
|
import EventEmitter from 'bare-events';
|
|
3
3
|
import { sweepUnreferenced } from './sweep.js';
|
|
4
4
|
import { inlineReference } from './inline.js';
|
|
5
|
-
import { clearBlob, stringifyBlobId, writeBlob } from './blob.js';
|
|
5
|
+
import { clearBlob, compactBlobCores, stringifyBlobId, writeBlob } from './blob.js';
|
|
6
6
|
import { parseBlobReference, readBlobReference, readBlobReferenceBytes, downloadBlobReference } from './reference.js';
|
|
7
7
|
export { CoreBlobServer } from './server.js';
|
|
8
8
|
export { compareBlobReferences, isInvalidBlobReferenceError, parseBlobReference, parseBlobReferenceCoreKey } from './reference.js';
|
|
@@ -44,6 +44,11 @@ export class BlobStore extends EventEmitter {
|
|
|
44
44
|
clear(reference, id) {
|
|
45
45
|
return clearBlob(this, reference, id);
|
|
46
46
|
}
|
|
47
|
+
async compact(coreKeys) {
|
|
48
|
+
if (this._closed)
|
|
49
|
+
return;
|
|
50
|
+
await compactBlobCores(this, coreKeys);
|
|
51
|
+
}
|
|
47
52
|
async available(reference) {
|
|
48
53
|
if (this._closed)
|
|
49
54
|
return false;
|
|
@@ -14,6 +14,8 @@ export declare class LocalFiles {
|
|
|
14
14
|
touch(key: FileKey, now?: number): Promise<void>;
|
|
15
15
|
private _write;
|
|
16
16
|
markCleared(file: FileKey): Promise<void>;
|
|
17
|
+
retireChats(meshKey: Buffer, purged: ReadonlyMap<string, number>): Promise<void>;
|
|
18
|
+
forget(file: FileKey): Promise<void>;
|
|
17
19
|
meshOf(coreKey: Buffer, blockOffset: number): Promise<Buffer | null>;
|
|
18
20
|
backfilled(meshKey: Buffer): Promise<boolean>;
|
|
19
21
|
markBackfilled(meshKey: Buffer): Promise<void>;
|
|
@@ -83,6 +83,37 @@ export class LocalFiles {
|
|
|
83
83
|
await tx.flush();
|
|
84
84
|
});
|
|
85
85
|
}
|
|
86
|
+
retireChats(meshKey, purged) {
|
|
87
|
+
return this._locked(async () => {
|
|
88
|
+
const tx = this._db.transaction();
|
|
89
|
+
let changed = false;
|
|
90
|
+
for (const row of await tx.find(LOCAL_FILES, meshRange(meshKey)).toArray()) {
|
|
91
|
+
const at = row.chatId ? purged.get(row.chatId) : undefined;
|
|
92
|
+
if (!at || row.deletedAt) {
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
await tx.insert(LOCAL_FILES, { ...row, deletedAt: at });
|
|
96
|
+
changed = true;
|
|
97
|
+
}
|
|
98
|
+
if (changed) {
|
|
99
|
+
await tx.flush();
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
await tx.close();
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
forget(file) {
|
|
107
|
+
return this._locked(async () => {
|
|
108
|
+
const tx = this._db.transaction();
|
|
109
|
+
if (!(await tx.get(LOCAL_FILES, keyOf(file)))) {
|
|
110
|
+
await tx.close();
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
await tx.delete(LOCAL_FILES, keyOf(file));
|
|
114
|
+
await tx.flush();
|
|
115
|
+
});
|
|
116
|
+
}
|
|
86
117
|
async meshOf(coreKey, blockOffset) {
|
|
87
118
|
const key = { coreKey, blockOffset };
|
|
88
119
|
const row = await this._db.findOne(LOCAL_FILES_BY_BLOB, { gte: key, lte: key });
|
|
@@ -2,7 +2,7 @@ import { FileChanges } from './files.js';
|
|
|
2
2
|
import { syncDeviceRevocation, syncOwnMember } from './devices.js';
|
|
3
3
|
import { syncChatTombstone } from './chats.js';
|
|
4
4
|
import { syncNotificationChunk } from './chunks.js';
|
|
5
|
-
import { AGENTS, CHATS, CHUNKS, CHUNK_COUNTERS, DEVICES, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, MEMBERS } from '../collections.js';
|
|
5
|
+
import { AGENTS, CHATS, CHUNKS, CHUNK_COUNTERS, DEVICES, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, MEMBERS, PURGED_CHATS } from '../collections.js';
|
|
6
6
|
export async function syncChanges(mesh, view, from) {
|
|
7
7
|
if (mesh.closing)
|
|
8
8
|
return;
|
|
@@ -13,6 +13,7 @@ export async function syncChanges(mesh, view, from) {
|
|
|
13
13
|
let meshesChanged = false;
|
|
14
14
|
let grantsChanged = false;
|
|
15
15
|
const orderDirty = new Set();
|
|
16
|
+
const purged = new Map();
|
|
16
17
|
let kindChanged = false;
|
|
17
18
|
const snap = view.db.snapshot();
|
|
18
19
|
const files = new FileChanges(snap, from);
|
|
@@ -21,6 +22,9 @@ export async function syncChanges(mesh, view, from) {
|
|
|
21
22
|
if (mesh.closing)
|
|
22
23
|
return;
|
|
23
24
|
await files.add(change);
|
|
25
|
+
if (change.type === 'delete') {
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
24
28
|
if (changed(change, DEVICES)) {
|
|
25
29
|
membershipMoved = true;
|
|
26
30
|
unattestedDevice ||= !change.value.revoked && !change.value.memberId;
|
|
@@ -50,6 +54,9 @@ export async function syncChanges(mesh, view, from) {
|
|
|
50
54
|
else if (changed(change, KNOWLEDGE_ARTIFACTS)) {
|
|
51
55
|
await mesh.files.syncArtifact(change.value);
|
|
52
56
|
}
|
|
57
|
+
else if (changed(change, PURGED_CHATS)) {
|
|
58
|
+
purged.set(change.value.chatId, change.value.at);
|
|
59
|
+
}
|
|
53
60
|
else if (change.collection === '@qvac/joined-meshes') {
|
|
54
61
|
meshesChanged = true;
|
|
55
62
|
}
|
|
@@ -65,6 +72,12 @@ export async function syncChanges(mesh, view, from) {
|
|
|
65
72
|
finally {
|
|
66
73
|
await snap.close();
|
|
67
74
|
}
|
|
75
|
+
if (purged.size) {
|
|
76
|
+
await mesh.fileIndex?.retireChats(purged);
|
|
77
|
+
for (const chatId of purged.keys()) {
|
|
78
|
+
mesh.emit('chat-purged', chatId);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
68
81
|
if (indexingDirty.size) {
|
|
69
82
|
mesh.emit('indexing-dirty', indexingDirty);
|
|
70
83
|
}
|
package/dist-lib/lib/core.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { BootstrapNode } from 'hyperswarm';
|
|
|
4
4
|
import ReadyGuard from 'ready-guard';
|
|
5
5
|
import ReadyResource from 'ready-resource';
|
|
6
6
|
import type { Logger } from '../index.ts';
|
|
7
|
-
import type { AssistantArchivedMeshList, AssistantCompactChatResponse, AssistantCopySkillResponse, AssistantDeleteModelRequest, AssistantDeleteModelResponse, AssistantDeleteSkillResponse, AssistantKvCacheDeletionResponse, AssistantListModelsRequest, AssistantLoadModelResponse, AssistantManagedSkillsResponse, AssistantMeshList, AssistantAccount, AssistantModelCatalogRevision, AssistantModelInfoList, AssistantOkResponse, AssistantSkillCatalogResponse, AssistantSkillFileResponse, AssistantSpeakRequest, AssistantSpeakResponse, AssistantToolCatalogResponse, AssistantDetectLanguageRequest, AssistantDetectLanguageResponse, AssistantTranscribeRequest, AssistantTranscribeResponse, AssistantUnloadModelResponse, AssistantCustomModelPreview, AssistantPreviewCustomModelRequest } from '../spec/index.d.ts';
|
|
7
|
+
import type { AssistantArchivedMeshList, AssistantCompactChatResponse, AssistantCopySkillResponse, AssistantDeleteModelRequest, AssistantDeleteModelResponse, AssistantDeleteSkillResponse, AssistantKvCacheDeletionResponse, AssistantListModelsRequest, AssistantLoadModelResponse, AssistantManagedSkillsResponse, AssistantMeshList, AssistantAccount, AssistantModelCatalogRevision, AssistantModelInfoList, AssistantOkResponse, AssistantSkillCatalogResponse, AssistantSkillFileResponse, AssistantSpeakRequest, AssistantSpeakResponse, AssistantToolCatalogResponse, AssistantDetectLanguageRequest, AssistantDetectLanguageResponse, AssistantTranscribeRequest, AssistantTranscribeResponse, AssistantUnloadModelResponse, AssistantModelUsers, AssistantCustomModelPreview, AssistantPreviewCustomModelRequest } from '../spec/index.d.ts';
|
|
8
8
|
import type { InferenceCallSource } from './api/inference-calls.ts';
|
|
9
9
|
import { type SkillOptions } from './api/index.ts';
|
|
10
10
|
import { type BlindPeeringOpts } from './core/blind-peers.ts';
|
|
@@ -73,6 +73,9 @@ export interface CoreOptions {
|
|
|
73
73
|
readonly suspend: () => Promise<void>;
|
|
74
74
|
readonly resume: () => Promise<void>;
|
|
75
75
|
};
|
|
76
|
+
grantWritableMs?: number;
|
|
77
|
+
accountWritableMs?: number;
|
|
78
|
+
grantVisibleMs?: number;
|
|
76
79
|
}
|
|
77
80
|
export declare const ACCOUNT_MEMBER_UNKNOWN = "This device is not part of its account yet";
|
|
78
81
|
export declare const AWAITING_ADMISSION = "awaiting-admission";
|
|
@@ -155,6 +158,9 @@ export declare class Core extends ReadyResource {
|
|
|
155
158
|
private readonly _meshSeed;
|
|
156
159
|
private readonly _meshKey;
|
|
157
160
|
private readonly _additionalNodes;
|
|
161
|
+
private readonly _grantWritableMs;
|
|
162
|
+
private readonly _accountWritableMs;
|
|
163
|
+
private readonly _grantVisibleMs;
|
|
158
164
|
private _listModels;
|
|
159
165
|
private _deleteModel;
|
|
160
166
|
private _clearKvCache;
|
|
@@ -228,10 +234,11 @@ export declare class Core extends ReadyResource {
|
|
|
228
234
|
private readonly _diagnosticsProviders;
|
|
229
235
|
private _inferenceCalls;
|
|
230
236
|
private _modelResidency;
|
|
237
|
+
private _modelUsers;
|
|
231
238
|
private _systemInfo;
|
|
232
239
|
private _statusError;
|
|
233
240
|
private _upgradeRequired;
|
|
234
|
-
constructor(transport: Transport, { storagePath, logger, spec, bootstrapNodes, device, app, minSupportedVersion, otaConfigCore, blindPeerKeys, blindPeeringOpts, fileGc, meshSeed, meshKey, additionalNodes, listModels, deleteModel, clearKvCache, deleteChatKvCache, compactChat, modelCatalogWatch, retryModelDownload, transcribe, detectLanguage, previewModelSource, speak, speakStream, embedder, extractors, skills, skillCatalog, tools, hostsModelSource, vectorIndex, apiLifecycle }: CoreOptions);
|
|
241
|
+
constructor(transport: Transport, { storagePath, logger, spec, bootstrapNodes, device, app, minSupportedVersion, otaConfigCore, blindPeerKeys, blindPeeringOpts, fileGc, meshSeed, meshKey, additionalNodes, listModels, deleteModel, clearKvCache, deleteChatKvCache, compactChat, modelCatalogWatch, retryModelDownload, transcribe, detectLanguage, previewModelSource, speak, speakStream, embedder, extractors, skills, skillCatalog, tools, hostsModelSource, vectorIndex, apiLifecycle, grantWritableMs, accountWritableMs, grantVisibleMs }: CoreOptions);
|
|
235
242
|
get devicePublicKey(): Buffer | null;
|
|
236
243
|
private _requireDeviceKeyPair;
|
|
237
244
|
get fsDrives(): DriveHost | null;
|
|
@@ -457,6 +464,7 @@ export declare class Core extends ReadyResource {
|
|
|
457
464
|
compact(force?: boolean): Promise<void>;
|
|
458
465
|
registerDiagnostics(provider: () => object): () => void;
|
|
459
466
|
registerModelResidency(residency: ModelResidency): () => void;
|
|
467
|
+
registerModelUsers(users: (name: string) => Promise<AssistantModelUsers>): () => void;
|
|
460
468
|
registerInferenceCalls(source: InferenceCallSource): () => void;
|
|
461
469
|
registerListModels(hook: (options: AssistantListModelsRequest) => Promise<AssistantModelInfoList>): () => void;
|
|
462
470
|
registerDeleteModel(hook: (options: AssistantDeleteModelRequest) => Promise<AssistantDeleteModelResponse>): () => void;
|
|
@@ -465,6 +473,8 @@ export declare class Core extends ReadyResource {
|
|
|
465
473
|
registerCompactChat(hook: (chatId: string, meshId: string) => Promise<AssistantCompactChatResponse>): () => void;
|
|
466
474
|
registerModelCatalogWatch(hook: () => AsyncIterable<AssistantModelCatalogRevision>): () => void;
|
|
467
475
|
private _watchModelCatalog;
|
|
476
|
+
private _modelCatalogRevisions;
|
|
477
|
+
private _readModels;
|
|
468
478
|
private _openModelCatalogWatchSource;
|
|
469
479
|
private _resetModelCatalogWatchSource;
|
|
470
480
|
registerRetryModelDownload(hook: (name: string) => Promise<void>): () => void;
|
package/dist-lib/lib/core.js
CHANGED
|
@@ -13,6 +13,7 @@ import ReadyResource from 'ready-resource';
|
|
|
13
13
|
import ScopeLock from 'scope-lock';
|
|
14
14
|
import { createApi } from './api/index.js';
|
|
15
15
|
import { NO_SUCH_MESH } from './api/current-mesh.js';
|
|
16
|
+
import { asEncoded, watchModels } from './models-watch.js';
|
|
16
17
|
import { BLIND_PEER_KEYS, BLIND_RELAY_KEYS } from './core/blind-peers.js';
|
|
17
18
|
import { CoreBlobServer, MemoryVectorIndex } from './core/files/index.js';
|
|
18
19
|
import { getChat, removeChatAttachments, removeChatKnowledge } from './core/chat.js';
|
|
@@ -139,6 +140,9 @@ export class Core extends ReadyResource {
|
|
|
139
140
|
_meshSeed;
|
|
140
141
|
_meshKey;
|
|
141
142
|
_additionalNodes;
|
|
143
|
+
_grantWritableMs;
|
|
144
|
+
_accountWritableMs;
|
|
145
|
+
_grantVisibleMs;
|
|
142
146
|
_listModels;
|
|
143
147
|
_deleteModel;
|
|
144
148
|
_clearKvCache;
|
|
@@ -212,10 +216,11 @@ export class Core extends ReadyResource {
|
|
|
212
216
|
_diagnosticsProviders;
|
|
213
217
|
_inferenceCalls = null;
|
|
214
218
|
_modelResidency = null;
|
|
219
|
+
_modelUsers = null;
|
|
215
220
|
_systemInfo;
|
|
216
221
|
_statusError;
|
|
217
222
|
_upgradeRequired;
|
|
218
|
-
constructor(transport, { storagePath, logger = null, spec = openLocalSpec, bootstrapNodes = undefined, device = undefined, app = undefined, minSupportedVersion = undefined, otaConfigCore = undefined, blindPeerKeys = undefined, blindPeeringOpts = undefined, fileGc = null, meshSeed = undefined, meshKey = undefined, additionalNodes = undefined, listModels = undefined, deleteModel = undefined, clearKvCache = undefined, deleteChatKvCache = undefined, compactChat = undefined, modelCatalogWatch = undefined, retryModelDownload = undefined, transcribe = undefined, detectLanguage = undefined, previewModelSource = undefined, speak = undefined, speakStream = undefined, embedder = undefined, extractors = undefined, skills = undefined, skillCatalog = undefined, tools = undefined, hostsModelSource = false, vectorIndex = undefined, apiLifecycle = undefined }) {
|
|
223
|
+
constructor(transport, { storagePath, logger = null, spec = openLocalSpec, bootstrapNodes = undefined, device = undefined, app = undefined, minSupportedVersion = undefined, otaConfigCore = undefined, blindPeerKeys = undefined, blindPeeringOpts = undefined, fileGc = null, meshSeed = undefined, meshKey = undefined, additionalNodes = undefined, listModels = undefined, deleteModel = undefined, clearKvCache = undefined, deleteChatKvCache = undefined, compactChat = undefined, modelCatalogWatch = undefined, retryModelDownload = undefined, transcribe = undefined, detectLanguage = undefined, previewModelSource = undefined, speak = undefined, speakStream = undefined, embedder = undefined, extractors = undefined, skills = undefined, skillCatalog = undefined, tools = undefined, hostsModelSource = false, vectorIndex = undefined, apiLifecycle = undefined, grantWritableMs = GRANT_WRITABLE_MS, accountWritableMs = ACCOUNT_WRITABLE_MS, grantVisibleMs = GRANT_VISIBLE_MS }) {
|
|
219
224
|
super();
|
|
220
225
|
this._transport = transport;
|
|
221
226
|
this._apiLifecycle = apiLifecycle ?? {
|
|
@@ -257,6 +262,9 @@ export class Core extends ReadyResource {
|
|
|
257
262
|
this._meshSeed = meshSeed;
|
|
258
263
|
this._meshKey = meshKey;
|
|
259
264
|
this._additionalNodes = additionalNodes;
|
|
265
|
+
this._grantWritableMs = grantWritableMs;
|
|
266
|
+
this._accountWritableMs = accountWritableMs;
|
|
267
|
+
this._grantVisibleMs = grantVisibleMs;
|
|
260
268
|
this._listModels = listModels ?? null;
|
|
261
269
|
this._deleteModel = deleteModel ?? null;
|
|
262
270
|
this._clearKvCache = clearKvCache ?? null;
|
|
@@ -413,7 +421,7 @@ export class Core extends ReadyResource {
|
|
|
413
421
|
computeLender: () => this._computeLender(),
|
|
414
422
|
pushSkillSecret: (request) => this.pushSkillSecret(request),
|
|
415
423
|
listModels: this._hostsModelSource || this._listModels
|
|
416
|
-
? (
|
|
424
|
+
? async (request) => ({ models: (await this._readModels(request)) ?? [] })
|
|
417
425
|
: null,
|
|
418
426
|
deleteModel: this._hostsModelSource || this._deleteModel
|
|
419
427
|
? (options) => this._deleteModel?.(options) ??
|
|
@@ -427,6 +435,9 @@ export class Core extends ReadyResource {
|
|
|
427
435
|
? ({ name }) => this._modelResidency?.unload(name) ??
|
|
428
436
|
Promise.reject(new Error('no model runtime is bound'))
|
|
429
437
|
: null,
|
|
438
|
+
modelUsers: this._hostsModelSource
|
|
439
|
+
? ({ name }) => this._modelUsers?.(name) ?? Promise.reject(new Error('no model runtime is bound'))
|
|
440
|
+
: null,
|
|
430
441
|
clearKvCache: () => this._clearKvCache?.() ??
|
|
431
442
|
Promise.resolve({
|
|
432
443
|
status: 'unsupported',
|
|
@@ -442,6 +453,9 @@ export class Core extends ReadyResource {
|
|
|
442
453
|
modelCatalogWatch: this._hostsModelSource || this._modelCatalogWatch
|
|
443
454
|
? () => this._watchModelCatalog()
|
|
444
455
|
: null,
|
|
456
|
+
modelsWatch: this._hostsModelSource || this._modelCatalogWatch
|
|
457
|
+
? (request) => watchModels(() => this._modelCatalogRevisions(), () => this._readModels(request))
|
|
458
|
+
: null,
|
|
445
459
|
retryModelDownload: this._hostsModelSource || this._retryModelDownload
|
|
446
460
|
? ({ name }) => this._retryModelDownload?.(name) ??
|
|
447
461
|
Promise.reject(new Error('no model download source bound'))
|
|
@@ -1789,7 +1803,7 @@ export class Core extends ReadyResource {
|
|
|
1789
1803
|
const cancellation = {
|
|
1790
1804
|
canceled: false,
|
|
1791
1805
|
accepting: true,
|
|
1792
|
-
deadline: Date.now() +
|
|
1806
|
+
deadline: Date.now() + this._grantVisibleMs,
|
|
1793
1807
|
account: !invite.memberInvite
|
|
1794
1808
|
};
|
|
1795
1809
|
this._joinCancellation = cancellation;
|
|
@@ -1981,7 +1995,11 @@ export class Core extends ReadyResource {
|
|
|
1981
1995
|
return;
|
|
1982
1996
|
}
|
|
1983
1997
|
try {
|
|
1984
|
-
await Promise.race([
|
|
1998
|
+
await Promise.race([
|
|
1999
|
+
mesh.waitForWritable(),
|
|
2000
|
+
this._closeSignal,
|
|
2001
|
+
after(this._accountWritableMs)
|
|
2002
|
+
]);
|
|
1985
2003
|
if (!mesh.writable) {
|
|
1986
2004
|
this._log?.warn(['p2p', 'mesh'], `the old ${kind} never went writable, so its writer was not revoked`, { meshId });
|
|
1987
2005
|
return;
|
|
@@ -2045,7 +2063,11 @@ export class Core extends ReadyResource {
|
|
|
2045
2063
|
const held = key ? this._pooled(b4a.toHex(key)) : null;
|
|
2046
2064
|
const mesh = key ? await this._openPooledMesh(b4a.toHex(key)).catch(noop) : null;
|
|
2047
2065
|
if (mesh && !mesh.writable) {
|
|
2048
|
-
await Promise.race([
|
|
2066
|
+
await Promise.race([
|
|
2067
|
+
mesh.waitForWritable(),
|
|
2068
|
+
this._closeSignal,
|
|
2069
|
+
after(this._accountWritableMs)
|
|
2070
|
+
]);
|
|
2049
2071
|
}
|
|
2050
2072
|
if (mesh && !mesh.ourMemberId) {
|
|
2051
2073
|
await mesh.refreshOurMember();
|
|
@@ -2343,7 +2365,7 @@ export class Core extends ReadyResource {
|
|
|
2343
2365
|
? setTimeout(() => {
|
|
2344
2366
|
cleanup();
|
|
2345
2367
|
reject(new Error(GRANT_JOIN_TIMED_OUT));
|
|
2346
|
-
},
|
|
2368
|
+
}, this._grantWritableMs)
|
|
2347
2369
|
: undefined;
|
|
2348
2370
|
this._cancelCandidateJoin = onCanceled;
|
|
2349
2371
|
candidate.autobee.on('writable', onWritable);
|
|
@@ -2511,7 +2533,10 @@ export class Core extends ReadyResource {
|
|
|
2511
2533
|
if (key) {
|
|
2512
2534
|
await this.local.files.markCleared(key);
|
|
2513
2535
|
}
|
|
2514
|
-
}
|
|
2536
|
+
},
|
|
2537
|
+
list: () => this.local.files.list(mesh().key),
|
|
2538
|
+
retireChats: (purged) => this.local.files.retireChats(mesh().key, purged),
|
|
2539
|
+
forget: (file) => this.local.files.forget(file)
|
|
2515
2540
|
};
|
|
2516
2541
|
}
|
|
2517
2542
|
async _indexMeshFiles(mesh) {
|
|
@@ -2648,6 +2673,23 @@ export class Core extends ReadyResource {
|
|
|
2648
2673
|
?.forget(chatId)
|
|
2649
2674
|
.catch((error) => this._log?.warn(['p2p', 'mesh'], 'chat-tombstone inspector reclaim failed', error));
|
|
2650
2675
|
});
|
|
2676
|
+
const reclaimLock = new ScopeLock({ debounce: true });
|
|
2677
|
+
const reclaim = async () => {
|
|
2678
|
+
if (!(await reclaimLock.lock()))
|
|
2679
|
+
return;
|
|
2680
|
+
try {
|
|
2681
|
+
if (!mesh.closing) {
|
|
2682
|
+
await mesh.files.reclaimPurged();
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
catch (error) {
|
|
2686
|
+
this._log?.warn(['p2p', 'mesh'], 'purged file reclaim failed', error);
|
|
2687
|
+
}
|
|
2688
|
+
finally {
|
|
2689
|
+
reclaimLock.unlock();
|
|
2690
|
+
}
|
|
2691
|
+
};
|
|
2692
|
+
mesh.on('chat-purged', () => void reclaim());
|
|
2651
2693
|
const sweepLock = new ScopeLock({ debounce: true });
|
|
2652
2694
|
const sweep = async () => {
|
|
2653
2695
|
if (!(await sweepLock.lock()))
|
|
@@ -2665,7 +2707,7 @@ export class Core extends ReadyResource {
|
|
|
2665
2707
|
}
|
|
2666
2708
|
};
|
|
2667
2709
|
void sweep();
|
|
2668
|
-
void this._indexMeshFiles(mesh);
|
|
2710
|
+
void this._indexMeshFiles(mesh).then(reclaim);
|
|
2669
2711
|
mesh.autobee.on('move-to', () => void sweep());
|
|
2670
2712
|
entry.hostedAgents = new HostedAgents({
|
|
2671
2713
|
mesh,
|
|
@@ -3785,11 +3827,11 @@ export class Core extends ReadyResource {
|
|
|
3785
3827
|
if (!(await this._meshSeedHandoff.admit(sibling, mesh.key, claim).catch(noop))) {
|
|
3786
3828
|
continue;
|
|
3787
3829
|
}
|
|
3788
|
-
await mesh.waitForWritable(
|
|
3830
|
+
await mesh.waitForWritable(this._accountWritableMs);
|
|
3789
3831
|
}
|
|
3790
|
-
if (!mesh.writable && (await mesh.waitForGenesis(
|
|
3832
|
+
if (!mesh.writable && (await mesh.waitForGenesis(this._accountWritableMs)) === null) {
|
|
3791
3833
|
await mesh.appendBirthOp(this._requireDeviceKeyPair(), this._localDevice ?? undefined);
|
|
3792
|
-
await mesh.waitForWritable(
|
|
3834
|
+
await mesh.waitForWritable(this._accountWritableMs);
|
|
3793
3835
|
}
|
|
3794
3836
|
if (mesh.writable) {
|
|
3795
3837
|
await this.local.markAwaitingAdmission(mesh.key, { awaitingAdmission: false });
|
|
@@ -4374,6 +4416,14 @@ export class Core extends ReadyResource {
|
|
|
4374
4416
|
}
|
|
4375
4417
|
};
|
|
4376
4418
|
}
|
|
4419
|
+
registerModelUsers(users) {
|
|
4420
|
+
this._modelUsers = users;
|
|
4421
|
+
return () => {
|
|
4422
|
+
if (this._modelUsers === users) {
|
|
4423
|
+
this._modelUsers = null;
|
|
4424
|
+
}
|
|
4425
|
+
};
|
|
4426
|
+
}
|
|
4377
4427
|
registerInferenceCalls(source) {
|
|
4378
4428
|
this._inferenceCalls = source;
|
|
4379
4429
|
return () => {
|
|
@@ -4435,21 +4485,34 @@ export class Core extends ReadyResource {
|
|
|
4435
4485
|
};
|
|
4436
4486
|
}
|
|
4437
4487
|
async *_watchModelCatalog() {
|
|
4488
|
+
const revisions = await this._modelCatalogRevisions();
|
|
4489
|
+
if (revisions) {
|
|
4490
|
+
yield* revisions;
|
|
4491
|
+
}
|
|
4492
|
+
}
|
|
4493
|
+
async _modelCatalogRevisions() {
|
|
4438
4494
|
while (!this.closing) {
|
|
4439
4495
|
const source = this._modelCatalogWatchSource;
|
|
4440
4496
|
try {
|
|
4441
4497
|
await source.ready();
|
|
4442
4498
|
}
|
|
4443
4499
|
catch {
|
|
4444
|
-
return;
|
|
4500
|
+
return null;
|
|
4445
4501
|
}
|
|
4446
4502
|
const watch = this._modelCatalogWatch;
|
|
4447
|
-
if (
|
|
4448
|
-
|
|
4503
|
+
if (watch) {
|
|
4504
|
+
return watch();
|
|
4449
4505
|
}
|
|
4450
|
-
yield* watch();
|
|
4451
|
-
return;
|
|
4452
4506
|
}
|
|
4507
|
+
return null;
|
|
4508
|
+
}
|
|
4509
|
+
async _readModels({ excludeCustom, ...options }) {
|
|
4510
|
+
const list = await this._listModels?.(options);
|
|
4511
|
+
if (!list) {
|
|
4512
|
+
return null;
|
|
4513
|
+
}
|
|
4514
|
+
const models = list.models ?? [];
|
|
4515
|
+
return asEncoded(excludeCustom ? models.filter((model) => model.registrySource !== 'custom') : models);
|
|
4453
4516
|
}
|
|
4454
4517
|
_openModelCatalogWatchSource() {
|
|
4455
4518
|
if (!this._modelCatalogWatch || !this._modelCatalogWatchSource.enter()) {
|
|
@@ -35,10 +35,15 @@ export interface CredentialEngine extends ConfigEngine {
|
|
|
35
35
|
}): AsyncIterable<unknown> & {
|
|
36
36
|
destroy(): void;
|
|
37
37
|
};
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
account(req: {}): Promise<{
|
|
39
|
+
meshId?: string | null;
|
|
40
|
+
}>;
|
|
41
|
+
listAgents(req: {
|
|
42
|
+
meshId: string;
|
|
43
|
+
}): Promise<{
|
|
44
|
+
agents?: ReadonlyArray<{
|
|
40
45
|
id: string;
|
|
41
|
-
}
|
|
46
|
+
}> | null;
|
|
42
47
|
}>;
|
|
43
48
|
}
|
|
44
49
|
export interface CredentialTarget {
|
|
@@ -254,7 +254,10 @@ export function startCredentialPusher(harness, engine, logger) {
|
|
|
254
254
|
const owners = credentialOwners(skills);
|
|
255
255
|
const specs = oauthSpecs(skills);
|
|
256
256
|
syncWatches(specs.map((spec) => spec.credentialKey));
|
|
257
|
-
const
|
|
257
|
+
const { meshId } = await engine.account({});
|
|
258
|
+
const ids = meshId
|
|
259
|
+
? ((await engine.listAgents({ meshId })).agents ?? []).map((agent) => agent.id)
|
|
260
|
+
: [];
|
|
258
261
|
const scopes = [undefined, ...ids];
|
|
259
262
|
const resolution = { engine, last, owners, specs, logger };
|
|
260
263
|
for (const agentId of scopes) {
|
|
@@ -11,6 +11,7 @@ export const MESH_SCOPED = new Set([
|
|
|
11
11
|
'listChats',
|
|
12
12
|
'getChat',
|
|
13
13
|
'renameChat',
|
|
14
|
+
'markChatRead',
|
|
14
15
|
'setChatRagEnabled',
|
|
15
16
|
'setChatWebSearchEnabled',
|
|
16
17
|
'deleteChat',
|
|
@@ -102,7 +103,9 @@ const DEVICE_LOCAL = new Set([
|
|
|
102
103
|
'deleteModel',
|
|
103
104
|
'loadModel',
|
|
104
105
|
'unloadModel',
|
|
106
|
+
'modelUsers',
|
|
105
107
|
'modelCatalogWatch',
|
|
108
|
+
'modelsWatch',
|
|
106
109
|
'clearKvCache',
|
|
107
110
|
'deleteChatKvCache',
|
|
108
111
|
'transcribe',
|