@qvac/core 1.0.0-rc.11 → 1.0.0-rc.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -1
- package/dist-lib/lib/assistant/harness.d.ts +2 -2
- package/dist-lib/lib/assistant/history.js +30 -2
- package/dist-lib/lib/assistant.js +2 -1
- package/dist-lib/lib/core/archived-mesh.js +7 -3
- package/dist-lib/lib/core/chat-copy.d.ts +18 -3
- package/dist-lib/lib/core/chat-copy.js +73 -23
- package/dist-lib/lib/core/chat-group.js +14 -9
- package/dist-lib/lib/core/chat-move.d.ts +9 -0
- package/dist-lib/lib/core/chat-move.js +94 -3
- package/dist-lib/lib/core/chat.js +21 -8
- package/dist-lib/lib/core/collections.d.ts +1 -0
- package/dist-lib/lib/core/collections.js +1 -0
- package/dist-lib/lib/core/contract/activity.js +2 -2
- package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
- package/dist-lib/lib/core/contract/chat-group.js +21 -1
- package/dist-lib/lib/core/contract/chat.d.ts +9 -0
- package/dist-lib/lib/core/contract/chat.js +127 -5
- package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
- package/dist-lib/lib/core/contract/chunks.js +37 -6
- package/dist-lib/lib/core/contract/features.d.ts +21 -0
- package/dist-lib/lib/core/contract/features.js +22 -0
- package/dist-lib/lib/core/contract/indexing.js +4 -2
- package/dist-lib/lib/core/contract/knowledge.js +5 -4
- package/dist-lib/lib/core/contract/members.js +2 -2
- package/dist-lib/lib/core/contract/projects.d.ts +5 -0
- package/dist-lib/lib/core/contract/projects.js +12 -4
- package/dist-lib/lib/core/contract/roles.js +30 -4
- package/dist-lib/lib/core/contract/run-requests.js +2 -2
- package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
- package/dist-lib/lib/core/contract/tool-configs.js +3 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
- package/dist-lib/lib/core/contract/workspace.js +40 -7
- package/dist-lib/lib/core/contract/writers.js +2 -2
- package/dist-lib/lib/core/files/attachments.js +4 -1
- package/dist-lib/lib/core/files/index.d.ts +1 -0
- package/dist-lib/lib/core/files/index.js +19 -6
- package/dist-lib/lib/core/files/processing/index.js +4 -1
- package/dist-lib/lib/core/files/rag/build.js +23 -1
- package/dist-lib/lib/core/files/rag/load.d.ts +3 -0
- package/dist-lib/lib/core/files/rag/load.js +27 -6
- package/dist-lib/lib/core/local/index.d.ts +6 -2
- package/dist-lib/lib/core/local/index.js +12 -3
- package/dist-lib/lib/core/mesh.d.ts +6 -0
- package/dist-lib/lib/core/mesh.js +22 -12
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +85 -0
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +690 -0
- package/dist-lib/lib/core/migrations/writer-links.d.ts +0 -1
- package/dist-lib/lib/core/migrations/writer-links.js +12 -10
- package/dist-lib/lib/core/project-keys.d.ts +6 -0
- package/dist-lib/lib/core/project-keys.js +11 -0
- package/dist-lib/lib/core/public-mesh.d.ts +2 -0
- package/dist-lib/lib/core/public-mesh.js +5 -0
- package/dist-lib/lib/core/sync/changed.d.ts +4 -2
- package/dist-lib/lib/core/sync/changed.js +1 -1
- package/dist-lib/lib/core/sync/index.js +27 -2
- package/dist-lib/lib/core/workspace-upgrade.d.ts +6 -0
- package/dist-lib/lib/core/workspace-upgrade.js +26 -16
- package/dist-lib/lib/core.d.ts +4 -32
- package/dist-lib/lib/core.js +143 -482
- package/dist-lib/lib/errors.d.ts +4 -0
- package/dist-lib/lib/errors.js +4 -0
- package/dist-lib/lib/run-transcript.d.ts +2 -1
- package/dist-lib/lib/run-transcript.js +11 -5
- package/dist-lib/schema/engine.js +2 -1
- package/dist-lib/schema/qvac.js +26 -1
- package/dist-lib/spec/hrpc/messages.js +17 -8
- package/dist-lib/spec/hyperschema/index.js +17 -8
- package/dist-lib/spec/hyperschema/schema.json +5 -0
- package/dist-lib/spec/hyperschema/types.d.ts +1 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +15 -0
- package/dist-lib/spec/mesh/hyperdb/index.js +70 -1
- package/dist-lib/spec/mesh/hyperdb/messages.js +806 -438
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +18 -0
- package/dist-lib/spec/mesh/hyperdispatch/index.js +52 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +303 -3
- package/dist-lib/spec/mesh/hyperschema/index.js +303 -3
- package/dist-lib/spec/mesh/hyperschema/schema.json +146 -0
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +39 -0
- package/package.json +10 -8
- package/tui/README.md +2 -1
- package/tui/boot.mjs +2 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { trackActivity } from './activity.js';
|
|
2
|
-
import {
|
|
2
|
+
import { FEATURES, isFeatureSupported } from './features.js';
|
|
3
3
|
import { AGENTS, CHATS, MESH_BLIND_PEERS } from '../collections.js';
|
|
4
4
|
async function seedsChat({ role = 'account' }, tx) {
|
|
5
|
-
return role !== 'account' || (await
|
|
5
|
+
return role !== 'account' || !(await isFeatureSupported(tx, FEATURES.ACCOUNT_SEEDS_NO_CHAT));
|
|
6
6
|
}
|
|
7
7
|
export function registerSeedDefaultsRoutes(router, options = {}) {
|
|
8
8
|
router.add('@qvac/seed-defaults', trackActivity(async (op, { tx, log }) => {
|
|
@@ -2,7 +2,7 @@ import { toolConfigsWithinBounds } from '../../../schema/qvac.js';
|
|
|
2
2
|
import { toolConfigsCarrySecret } from '../tool-configs.js';
|
|
3
3
|
import { trackActivity } from './activity.js';
|
|
4
4
|
import { AGENTS, TOOL_CONFIGS } from '../collections.js';
|
|
5
|
-
import {
|
|
5
|
+
import { FEATURES, isFeatureSupported } from './features.js';
|
|
6
6
|
export function registerToolConfigsRoutes(router) {
|
|
7
7
|
router.add('@qvac/put-tool-configs', trackActivity(async ({ settings }, { tx, log }) => {
|
|
8
8
|
if (!toolConfigsWithinBounds(settings.settings)) {
|
|
@@ -30,7 +30,8 @@ export function registerToolConfigsRoutes(router) {
|
|
|
30
30
|
});
|
|
31
31
|
return;
|
|
32
32
|
}
|
|
33
|
-
if (!settings.deletedAt &&
|
|
33
|
+
if (!settings.deletedAt &&
|
|
34
|
+
!(await isFeatureSupported(tx, FEATURES.TOOL_CONFIGS_WITHOUT_AGENT))) {
|
|
34
35
|
const agent = await tx.get(AGENTS, { id: settings.agentId });
|
|
35
36
|
if (!agent || agent.deletedAt) {
|
|
36
37
|
log?.debug('contract', 'put-tool-configs dropped: agent missing or deleted', {
|
|
@@ -26,6 +26,8 @@ export declare function leavesNoAdmin(context: Signed, memberId: Buffer): Promis
|
|
|
26
26
|
export declare function kicksAdminsLastDevice(context: Signed, deviceId: Buffer | null | undefined): Promise<boolean>;
|
|
27
27
|
export declare function readMemberKey(tx: HyperDB, memberId: Buffer): Promise<QvacMemberKey | null>;
|
|
28
28
|
export declare function uncountMember(tx: HyperDB, memberId: Buffer): Promise<void>;
|
|
29
|
+
export declare function upgradeWriter(tx: HyperDB, writerKey: Buffer): Promise<boolean>;
|
|
29
30
|
export declare function upgradeMembers(tx: HyperDB): Promise<Buffer[]>;
|
|
31
|
+
export declare function countedMembers(tx: HyperDB): Promise<Buffer[]>;
|
|
30
32
|
export declare function registerWorkspaceRoutes(router: Router): void;
|
|
31
33
|
export {};
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { liveDeviceForWriter, liveDevices, liveDevicesOf, trackActivity } from './activity.js';
|
|
2
2
|
import { meshKind } from './mesh-kind.js';
|
|
3
|
-
import {
|
|
3
|
+
import { bornUnderRules } from './mesh-rules.js';
|
|
4
|
+
import { FEATURES, isFeatureSupported } from './features.js';
|
|
4
5
|
import b4a from 'b4a';
|
|
5
6
|
import { removedBy } from './invites.js';
|
|
6
7
|
import { hostsLiveAgent, readPublicSeed } from './public-seeds.js';
|
|
7
8
|
import { SEALED_SEED_BYTES } from '../device-seal.js';
|
|
8
|
-
import { DEVICES, MEMBER_KEYS, MEMBER_ROLES, MEMBERS, MESH_FOUNDERS, MESH_KINDS, PUBLIC_SEEDS, WORKSPACE } from '../collections.js';
|
|
9
|
+
import { DEVICES, LIVE_DEVICES_BY_WRITER_KEY, MEMBER_KEYS, MEMBER_ROLES, MEMBERS, MESH_FOUNDERS, MESH_KINDS, PUBLIC_SEEDS, WORKSPACE } from '../collections.js';
|
|
9
10
|
export const WORKSPACE_ROW = 'workspace';
|
|
10
11
|
export const WORKSPACE_ROLES = new Set(['admin', 'member']);
|
|
11
12
|
export function readWorkspace(tx) {
|
|
@@ -132,7 +133,12 @@ async function bornWorkspace({ workspace }, context) {
|
|
|
132
133
|
log?.warn('contract', 'put-workspace dropped: the genesis founded no member to be its admin');
|
|
133
134
|
return;
|
|
134
135
|
}
|
|
135
|
-
const row = {
|
|
136
|
+
const row = {
|
|
137
|
+
...workspace,
|
|
138
|
+
id: WORKSPACE_ROW,
|
|
139
|
+
upgradeMemberIds: undefined,
|
|
140
|
+
upgradeWriterKeys: undefined
|
|
141
|
+
};
|
|
136
142
|
await tx.insert(WORKSPACE, row);
|
|
137
143
|
await tx.insert(MEMBER_ROLES, { memberId: founder.memberId, role: 'admin' });
|
|
138
144
|
log?.info('contract', 'workspace born');
|
|
@@ -148,8 +154,13 @@ async function editedWorkspace(existing, { workspace }, context) {
|
|
|
148
154
|
log?.debug('contract', 'put-workspace dropped: not newer than the stored row');
|
|
149
155
|
return;
|
|
150
156
|
}
|
|
151
|
-
const { upgradeMemberIds } = existing;
|
|
152
|
-
const row = {
|
|
157
|
+
const { upgradeMemberIds, upgradeWriterKeys } = existing;
|
|
158
|
+
const row = {
|
|
159
|
+
...workspace,
|
|
160
|
+
id: WORKSPACE_ROW,
|
|
161
|
+
upgradeMemberIds,
|
|
162
|
+
upgradeWriterKeys
|
|
163
|
+
};
|
|
153
164
|
await tx.insert(WORKSPACE, row);
|
|
154
165
|
return row;
|
|
155
166
|
}
|
|
@@ -248,7 +259,7 @@ async function putMemberKey(op, context) {
|
|
|
248
259
|
async function upgradeToWorkspace(_op, context) {
|
|
249
260
|
const { tx, log } = context;
|
|
250
261
|
const kind = await meshKind(tx);
|
|
251
|
-
const onRules =
|
|
262
|
+
const onRules = await isFeatureSupported(tx, FEATURES.WORKSPACE_UPGRADE);
|
|
252
263
|
if (!onRules || (kind !== 'shared' && kind !== 'workspace') || (await bornUnderRules(tx))) {
|
|
253
264
|
log?.debug('contract', 'upgrade-to-workspace dropped: not a shared mesh born before the rules');
|
|
254
265
|
return;
|
|
@@ -270,11 +281,25 @@ async function upgradeToWorkspace(_op, context) {
|
|
|
270
281
|
}
|
|
271
282
|
}
|
|
272
283
|
await tx.insert(MESH_KINDS, { id: 'mesh', kind: 'workspace' });
|
|
273
|
-
const row = {
|
|
284
|
+
const row = {
|
|
285
|
+
id: WORKSPACE_ROW,
|
|
286
|
+
updatedAt: 0,
|
|
287
|
+
upgradeMemberIds,
|
|
288
|
+
upgradeWriterKeys: await liveWriterKeys(tx)
|
|
289
|
+
};
|
|
274
290
|
await tx.insert(WORKSPACE, row);
|
|
275
291
|
log?.info('contract', 'shared mesh upgraded to a workspace');
|
|
276
292
|
return row;
|
|
277
293
|
}
|
|
294
|
+
async function liveWriterKeys(tx) {
|
|
295
|
+
const keys = [];
|
|
296
|
+
for await (const { writerKey } of tx.find(LIVE_DEVICES_BY_WRITER_KEY)) {
|
|
297
|
+
if (writerKey) {
|
|
298
|
+
keys.push(writerKey);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
return keys;
|
|
302
|
+
}
|
|
278
303
|
async function countUpgrader(tx, signer, { log }) {
|
|
279
304
|
const row = await tx.get(WORKSPACE, { id: WORKSPACE_ROW });
|
|
280
305
|
const counted = row?.upgradeMemberIds ?? [];
|
|
@@ -295,10 +320,18 @@ export async function uncountMember(tx, memberId) {
|
|
|
295
320
|
await tx.insert(WORKSPACE, { ...row, upgradeMemberIds });
|
|
296
321
|
}
|
|
297
322
|
}
|
|
323
|
+
export async function upgradeWriter(tx, writerKey) {
|
|
324
|
+
const row = await tx.get(WORKSPACE, { id: WORKSPACE_ROW });
|
|
325
|
+
return !!row?.upgradeWriterKeys?.some((key) => b4a.equals(key, writerKey));
|
|
326
|
+
}
|
|
298
327
|
export async function upgradeMembers(tx) {
|
|
299
328
|
const row = await tx.get(WORKSPACE, { id: WORKSPACE_ROW });
|
|
300
329
|
return row?.upgradeMemberIds ?? [];
|
|
301
330
|
}
|
|
331
|
+
export async function countedMembers(tx) {
|
|
332
|
+
const live = await liveMemberIds(tx);
|
|
333
|
+
return (await upgradeMembers(tx)).filter((id) => live.has(b4a.toHex(id)));
|
|
334
|
+
}
|
|
302
335
|
export function registerWorkspaceRoutes(router) {
|
|
303
336
|
router.add('@qvac/put-workspace', trackActivity(async (op, context) => {
|
|
304
337
|
const existing = await readWorkspace(context.tx);
|
|
@@ -14,7 +14,7 @@ import { dropHostRows, hostedDevice, releaseHostedDevice } from './agent-join.js
|
|
|
14
14
|
import { actsFor, dropRole, kicksAdminsLastDevice, leavesNoAdmin, uncountMember } from './workspace.js';
|
|
15
15
|
import { verifyDeviceProof } from '../identity.js';
|
|
16
16
|
import { sealed } from './seal.js';
|
|
17
|
-
import {
|
|
17
|
+
import { FEATURES, isFeatureSupported } from './features.js';
|
|
18
18
|
import b4a from 'b4a';
|
|
19
19
|
import { DEVICES, MEMBERS, MESH_INVITE_GRANTS, PROJECT_MEMBERS } from '../collections.js';
|
|
20
20
|
async function boundToAnother(tx, writerKey, id) {
|
|
@@ -191,7 +191,7 @@ async function ownLeave({ writerKey, deviceId, revokedAt }, { tx, node }) {
|
|
|
191
191
|
if (!b4a.equals(node.key, writerKey) || !deviceId) {
|
|
192
192
|
return false;
|
|
193
193
|
}
|
|
194
|
-
if ((await
|
|
194
|
+
if (!(await isFeatureSupported(tx, FEATURES.OWN_LEAVE_HOLDS))) {
|
|
195
195
|
return false;
|
|
196
196
|
}
|
|
197
197
|
const device = await tx.get(DEVICES, { id: deviceId });
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { CoreError } from '../../errors.js';
|
|
2
2
|
import { linkAttachment } from '../attachment-link.js';
|
|
3
|
+
import { mayEditChunk } from '../contract/chunks.js';
|
|
3
4
|
import { getChunk, readChunk, updateChunk } from '../chunk.js';
|
|
4
5
|
import { uploadFile } from './upload.js';
|
|
5
6
|
import b4a from 'b4a';
|
|
@@ -49,7 +50,9 @@ export function attachmentFiles(mesh, { maxBytes = MAX_ATTACHMENT_BYTES, deviceI
|
|
|
49
50
|
if (chunk.type !== 'attachment' || !chunk.attachment) {
|
|
50
51
|
throw new CoreError('NOT_AN_ATTACHMENT', [chunkId]);
|
|
51
52
|
}
|
|
52
|
-
if (deviceId &&
|
|
53
|
+
if (deviceId &&
|
|
54
|
+
!b4a.equals(chunk.originDeviceId, deviceId) &&
|
|
55
|
+
!(await mayEditChunk(mesh.view, chunk, mesh.localWriterKey))) {
|
|
53
56
|
throw new CoreError('ATTACHMENT_DELETE_NOT_ORIGIN');
|
|
54
57
|
}
|
|
55
58
|
const attachment = chunk.attachment;
|
|
@@ -79,6 +79,7 @@ export declare class Files extends EventEmitter {
|
|
|
79
79
|
knowledge(options?: Parameters<typeof knowledgeFiles>[1]): import("./knowledge.ts").KnowledgeFiles;
|
|
80
80
|
listKnowledge(chatId: string): Promise<QvacKnowledge[]>;
|
|
81
81
|
knowledgeBase(): Promise<AssistantKnowledgeBaseList>;
|
|
82
|
+
private _embedding;
|
|
82
83
|
record(id: string): Promise<QvacKnowledge | null>;
|
|
83
84
|
resolve(request: ChatFileKey): Promise<FileRecord | null>;
|
|
84
85
|
fileRead(request: Omit<AssistantFileReadRequest, 'meshId'>, options?: FileReadOptions): AsyncGenerator<AssistantFileReadFrame>;
|
|
@@ -125,7 +125,8 @@ export class Files extends EventEmitter {
|
|
|
125
125
|
error: failure?.detail,
|
|
126
126
|
loaded: !!this.state.get(knowledge.id)?.loaded,
|
|
127
127
|
reason: failure?.reason,
|
|
128
|
-
failedOn: device?.name
|
|
128
|
+
failedOn: device?.name,
|
|
129
|
+
progress: status?.status === 'ready' ? undefined : this._embedding(knowledge.id, rag)
|
|
129
130
|
});
|
|
130
131
|
}
|
|
131
132
|
return {
|
|
@@ -138,6 +139,15 @@ export class Files extends EventEmitter {
|
|
|
138
139
|
await snap.close();
|
|
139
140
|
}
|
|
140
141
|
}
|
|
142
|
+
_embedding(knowledgeId, attempts) {
|
|
143
|
+
const progress = this.state.get(knowledgeId)?.progress;
|
|
144
|
+
const now = Date.now();
|
|
145
|
+
const claimed = attempts.some((row) => isClaimLive(row, now) && b4a.equals(row.claimedBy, this._mesh.localWriterKey));
|
|
146
|
+
if (progress?.stage !== 'embedding' || !claimed) {
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
return chunkProgress(progress);
|
|
150
|
+
}
|
|
141
151
|
record(id) {
|
|
142
152
|
return getKnowledge(this._mesh, id);
|
|
143
153
|
}
|
|
@@ -684,11 +694,7 @@ export class Files extends EventEmitter {
|
|
|
684
694
|
if (local?.progress &&
|
|
685
695
|
processor.state !== 'not-applicable' &&
|
|
686
696
|
(!claim || b4a.equals(claim.claimedBy, this._mesh.localWriterKey))) {
|
|
687
|
-
processor.progress =
|
|
688
|
-
completed: local.progress.processedChunks,
|
|
689
|
-
total: local.progress.totalChunks || undefined,
|
|
690
|
-
unit: 'chunks'
|
|
691
|
-
};
|
|
697
|
+
processor.progress = chunkProgress(local.progress);
|
|
692
698
|
}
|
|
693
699
|
const readyForChat = applicable
|
|
694
700
|
? processor.state === 'ready'
|
|
@@ -797,6 +803,13 @@ export class Files extends EventEmitter {
|
|
|
797
803
|
await this._store.close();
|
|
798
804
|
}
|
|
799
805
|
}
|
|
806
|
+
function chunkProgress(progress) {
|
|
807
|
+
return {
|
|
808
|
+
completed: progress.processedChunks,
|
|
809
|
+
total: progress.totalChunks || undefined,
|
|
810
|
+
unit: 'chunks'
|
|
811
|
+
};
|
|
812
|
+
}
|
|
800
813
|
function settledFile(file) {
|
|
801
814
|
return file.readyForChat || file.state === 'failed';
|
|
802
815
|
}
|
|
@@ -246,7 +246,10 @@ export class FileProcessing {
|
|
|
246
246
|
_pump(lane) {
|
|
247
247
|
while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
|
|
248
248
|
const busy = [...lane.jobs];
|
|
249
|
-
|
|
249
|
+
const full = lane.stage === 'produce'
|
|
250
|
+
? busy.length > 1 || (busy.length === 1 && !busy[0].admitted)
|
|
251
|
+
: busy.some((job) => !job.admitted);
|
|
252
|
+
if (full) {
|
|
250
253
|
return;
|
|
251
254
|
}
|
|
252
255
|
const next = [...lane.queue].find(([queued]) => !busy.some((job) => job.id === queued));
|
|
@@ -76,9 +76,31 @@ async function pageChunks(input, pdfPages) {
|
|
|
76
76
|
mimeType: knowledge.mimeType,
|
|
77
77
|
content: ''
|
|
78
78
|
};
|
|
79
|
-
const chunks = pages.flatMap(({ page, text }) => chunkRagDocuments([{ ...document, content: text }], {}).map((chunk) => ({ chunk, page })));
|
|
79
|
+
const chunks = withoutShortPages(pages).flatMap(({ page, text }) => chunkRagDocuments([{ ...document, content: text }], {}).map((chunk) => ({ chunk, page })));
|
|
80
80
|
return chunks.map(({ chunk, page }, index) => numbered(chunk, { page, index }));
|
|
81
81
|
}
|
|
82
|
+
const SHORT_PAGE_WORDS = 12;
|
|
83
|
+
const UNSPACED = /[\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Thai}\p{sc=Lao}\p{sc=Khmer}\p{sc=Myanmar}]/gu;
|
|
84
|
+
const wordsIn = (text) => text.split(/\s+/).filter(Boolean).length + (text.match(UNSPACED)?.length ?? 0);
|
|
85
|
+
const oneParagraph = (text) => text.trim().replace(/\n\s*\n/g, '\n');
|
|
86
|
+
function withoutShortPages(pages) {
|
|
87
|
+
const out = [];
|
|
88
|
+
let held = '';
|
|
89
|
+
for (const { page, text } of pages) {
|
|
90
|
+
const short = wordsIn(text) < SHORT_PAGE_WORDS;
|
|
91
|
+
held = [held, short ? oneParagraph(text) : text].filter(Boolean).join('\n');
|
|
92
|
+
if (short && wordsIn(held) < SHORT_PAGE_WORDS) {
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
out.push({ page, text: held });
|
|
96
|
+
held = '';
|
|
97
|
+
}
|
|
98
|
+
const last = out.pop();
|
|
99
|
+
if (!last) {
|
|
100
|
+
return pages;
|
|
101
|
+
}
|
|
102
|
+
return [...out, { page: last.page, text: [last.text, held].filter(Boolean).join('\n') }];
|
|
103
|
+
}
|
|
82
104
|
function numbered(chunk, { page, index }) {
|
|
83
105
|
const parsed = JSON.parse(chunk.metadataJson);
|
|
84
106
|
const metadata = isRecord(parsed) ? parsed : {};
|
|
@@ -22,7 +22,10 @@ export declare class HydratePass {
|
|
|
22
22
|
private readonly _index;
|
|
23
23
|
private readonly _embedder;
|
|
24
24
|
private readonly _extractors;
|
|
25
|
+
private _rebuilding;
|
|
26
|
+
private readonly _rebuildWaiters;
|
|
25
27
|
constructor(opts: HydratePassOptions);
|
|
26
28
|
run(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
27
29
|
private _rebuild;
|
|
30
|
+
private _rebuildOne;
|
|
28
31
|
}
|
|
@@ -21,6 +21,8 @@ export class HydratePass {
|
|
|
21
21
|
_index;
|
|
22
22
|
_embedder;
|
|
23
23
|
_extractors;
|
|
24
|
+
_rebuilding = false;
|
|
25
|
+
_rebuildWaiters = new Map();
|
|
24
26
|
constructor(opts) {
|
|
25
27
|
this._model = opts.model;
|
|
26
28
|
this._index = opts.index;
|
|
@@ -165,13 +167,32 @@ export class HydratePass {
|
|
|
165
167
|
}
|
|
166
168
|
}
|
|
167
169
|
async _rebuild(document, artifact, signal) {
|
|
170
|
+
const embedder = this._embedder;
|
|
171
|
+
if (!embedder || (document.artifact(artifact.id).rebuildFailures ?? 0) > EMBED_RETRY_LIMIT) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
if (this._rebuilding) {
|
|
175
|
+
this._rebuildWaiters.set(document.record.id, () => document.schedule(Date.now()));
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
this._rebuilding = true;
|
|
179
|
+
document.admit();
|
|
180
|
+
try {
|
|
181
|
+
await this._rebuildOne(document, { artifact, embedder, signal });
|
|
182
|
+
}
|
|
183
|
+
finally {
|
|
184
|
+
this._rebuilding = false;
|
|
185
|
+
const waiters = [...this._rebuildWaiters.values()];
|
|
186
|
+
this._rebuildWaiters.clear();
|
|
187
|
+
for (const wake of waiters) {
|
|
188
|
+
wake();
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
async _rebuildOne(document, { artifact, embedder, signal }) {
|
|
168
193
|
const knowledge = document.record;
|
|
169
194
|
const local = document.artifact(artifact.id);
|
|
170
|
-
if (
|
|
171
|
-
(local.rebuildFailures ?? 0) > EMBED_RETRY_LIMIT ||
|
|
172
|
-
document.paused() ||
|
|
173
|
-
document.closed() ||
|
|
174
|
-
signal.aborted) {
|
|
195
|
+
if (document.paused() || document.closed() || signal.aborted) {
|
|
175
196
|
return;
|
|
176
197
|
}
|
|
177
198
|
const retryAt = document.output(knowledge.id, knowledge.blobId).blockedUntil();
|
|
@@ -200,7 +221,7 @@ export class HydratePass {
|
|
|
200
221
|
}
|
|
201
222
|
document.load({ state: 'running', reason: 'rebuilding local index' });
|
|
202
223
|
const payload = await buildArtifactPayload({
|
|
203
|
-
embedder
|
|
224
|
+
embedder,
|
|
204
225
|
extractors: this._extractors,
|
|
205
226
|
knowledge,
|
|
206
227
|
artifactId: artifact.id,
|
|
@@ -40,7 +40,7 @@ export declare const MESH_LOCAL_COLLECTIONS: readonly [{
|
|
|
40
40
|
readonly collection: "@local/run-log-steps";
|
|
41
41
|
readonly field: 'meshKey';
|
|
42
42
|
}];
|
|
43
|
-
export declare const MESH_CONFIG_FAMILIES: readonly ['workspace-upgrade', 'workspace-agents'];
|
|
43
|
+
export declare const MESH_CONFIG_FAMILIES: readonly ['workspace-upgrade', 'workspace-agents', 'workspace-edits', 'copied'];
|
|
44
44
|
export interface ResolvedSession {
|
|
45
45
|
readonly seed: Buffer;
|
|
46
46
|
readonly baseKey: Buffer | undefined;
|
|
@@ -114,6 +114,9 @@ export interface DropMeshOptions {
|
|
|
114
114
|
export interface RecordDepartureOptions {
|
|
115
115
|
readonly creator: boolean;
|
|
116
116
|
}
|
|
117
|
+
interface ArchiveMeshOptions {
|
|
118
|
+
readonly homeTo?: Buffer | null;
|
|
119
|
+
}
|
|
117
120
|
export interface AdoptAccountMeshOptions {
|
|
118
121
|
readonly account?: Buffer | null;
|
|
119
122
|
readonly keepName?: boolean;
|
|
@@ -167,6 +170,7 @@ declare class LocalStore {
|
|
|
167
170
|
readAccountRecovery(meshKey: Buffer): Promise<LocalAccountRecovery | null>;
|
|
168
171
|
readChatMove(meshKey: Buffer, chatId: string): Promise<LocalChatMove | null>;
|
|
169
172
|
listChatMoves(): Promise<LocalChatMove[]>;
|
|
173
|
+
listChatMovesFrom(meshKey: Buffer): Promise<LocalChatMove[]>;
|
|
170
174
|
readMeshUpdate(meshKey: Buffer): Promise<LocalMeshUpdate | null>;
|
|
171
175
|
updateMeshUpdate(meshKey: Buffer, change: (row: LocalMeshUpdate | null) => LocalMeshUpdate | null): Promise<void>;
|
|
172
176
|
meshNames(): Promise<LocalMeshName[]>;
|
|
@@ -188,7 +192,7 @@ declare class LocalStore {
|
|
|
188
192
|
departures(): Promise<LocalMeshDeparture[]>;
|
|
189
193
|
departureRole(departure: LocalMeshDeparture): Promise<MeshRole>;
|
|
190
194
|
deleteDeparture(key: Buffer): Promise<void>;
|
|
191
|
-
archiveMesh(key: Buffer): Promise<void>;
|
|
195
|
+
archiveMesh(key: Buffer, { homeTo }?: ArchiveMeshOptions): Promise<void>;
|
|
192
196
|
markRetired(keys: Buffer[], accountKey: Buffer): Promise<void>;
|
|
193
197
|
unretire(accountKey: Buffer): Promise<Buffer[]>;
|
|
194
198
|
markKicked(key: Buffer): Promise<void>;
|
|
@@ -24,7 +24,12 @@ export const MESH_LOCAL_COLLECTIONS = [
|
|
|
24
24
|
{ collection: LOCAL_RUN_LOGS, field: 'meshKey' },
|
|
25
25
|
{ collection: LOCAL_RUN_LOG_STEPS, field: 'meshKey' }
|
|
26
26
|
];
|
|
27
|
-
export const MESH_CONFIG_FAMILIES = [
|
|
27
|
+
export const MESH_CONFIG_FAMILIES = [
|
|
28
|
+
'workspace-upgrade',
|
|
29
|
+
'workspace-agents',
|
|
30
|
+
'workspace-edits',
|
|
31
|
+
'copied'
|
|
32
|
+
];
|
|
28
33
|
const IDENTITY_ID = 'identity';
|
|
29
34
|
const MEMBERSHIP_ID = 'membership';
|
|
30
35
|
const APP_CONFIG_ID = 'default';
|
|
@@ -418,6 +423,10 @@ class LocalStore {
|
|
|
418
423
|
listChatMoves() {
|
|
419
424
|
return this._db.find(LOCAL_CHAT_MOVES).toArray();
|
|
420
425
|
}
|
|
426
|
+
listChatMovesFrom(meshKey) {
|
|
427
|
+
const range = { gte: { meshKey }, lte: { meshKey } };
|
|
428
|
+
return this._db.find(LOCAL_CHAT_MOVES, range).toArray();
|
|
429
|
+
}
|
|
421
430
|
readMeshUpdate(meshKey) {
|
|
422
431
|
return this._db.get(LOCAL_MESH_UPDATES, { meshKey });
|
|
423
432
|
}
|
|
@@ -653,7 +662,7 @@ class LocalStore {
|
|
|
653
662
|
await tx.flush();
|
|
654
663
|
});
|
|
655
664
|
}
|
|
656
|
-
archiveMesh(key) {
|
|
665
|
+
archiveMesh(key, { homeTo = null } = {}) {
|
|
657
666
|
return this._locked(async () => {
|
|
658
667
|
const existing = await this._db.get(LOCAL_MESH_MEMBERSHIP, { key });
|
|
659
668
|
const { cleared, home } = await this._pointersAt(key);
|
|
@@ -666,7 +675,7 @@ class LocalStore {
|
|
|
666
675
|
});
|
|
667
676
|
}
|
|
668
677
|
for (const pointer of cleared) {
|
|
669
|
-
await tx.insert(LOCAL_ACTIVE_MESH, pointer);
|
|
678
|
+
await tx.insert(LOCAL_ACTIVE_MESH, { ...pointer, homeMeshKey: homeTo });
|
|
670
679
|
}
|
|
671
680
|
await this._dropChatOrder(tx, key);
|
|
672
681
|
await this._dropRows(tx, key, { collection: LOCAL_FILES, field: 'meshKey' });
|
|
@@ -45,12 +45,16 @@ export interface MeshOptions {
|
|
|
45
45
|
wakeup?: Wakeup;
|
|
46
46
|
attachmentUrl?: AttachmentUrl | null;
|
|
47
47
|
fileIndex?: MeshFileIndex | null;
|
|
48
|
+
history?: MeshHistory | null;
|
|
48
49
|
pendingUpdates?: PendingUpdates | null;
|
|
49
50
|
sidebarChats?: SidebarChats | null;
|
|
50
51
|
onerror?: ErrorSink;
|
|
51
52
|
slowWriteMs?: number;
|
|
52
53
|
slowDrainMs?: number;
|
|
53
54
|
}
|
|
55
|
+
export interface MeshHistory {
|
|
56
|
+
edited(chatIds: ReadonlySet<string> | null): Promise<void>;
|
|
57
|
+
}
|
|
54
58
|
export type SidebarChats = (limit: number) => Promise<ReadonlyArray<{
|
|
55
59
|
readonly chatId: string;
|
|
56
60
|
}>>;
|
|
@@ -130,6 +134,7 @@ export declare class Mesh extends ReadyResource {
|
|
|
130
134
|
private readonly _wakeup;
|
|
131
135
|
readonly attachmentUrl: AttachmentUrl | null;
|
|
132
136
|
readonly fileIndex: MeshFileIndex | null;
|
|
137
|
+
readonly history: MeshHistory | null;
|
|
133
138
|
private readonly _scope;
|
|
134
139
|
readonly router: Router;
|
|
135
140
|
private readonly _keyPair;
|
|
@@ -255,6 +260,7 @@ export declare class Mesh extends ReadyResource {
|
|
|
255
260
|
_autobeeIsTrusted(key: Buffer, reference: MeshView): Promise<boolean>;
|
|
256
261
|
_autobeeMostRecentTrusted(target: MeshView, reference: MeshView | null): Promise<AutobeeOplogRef | null>;
|
|
257
262
|
private _newestLinked;
|
|
263
|
+
private _trustedIndex;
|
|
258
264
|
private _newestTrustedOf;
|
|
259
265
|
private _trusts;
|
|
260
266
|
_autobeeOpen(bee: unknown): MeshView;
|
|
@@ -104,6 +104,7 @@ export class Mesh extends ReadyResource {
|
|
|
104
104
|
_wakeup;
|
|
105
105
|
attachmentUrl;
|
|
106
106
|
fileIndex;
|
|
107
|
+
history;
|
|
107
108
|
_scope;
|
|
108
109
|
router;
|
|
109
110
|
_keyPair;
|
|
@@ -304,6 +305,7 @@ export class Mesh extends ReadyResource {
|
|
|
304
305
|
this._wakeup = opts.wakeup;
|
|
305
306
|
this.attachmentUrl = opts.attachmentUrl ?? null;
|
|
306
307
|
this.fileIndex = opts.fileIndex ?? null;
|
|
308
|
+
this.history = opts.history ?? null;
|
|
307
309
|
this._sidebarChats = opts.sidebarChats ?? null;
|
|
308
310
|
this._updates = new ViewUpdates((update) => this._syncUpdate(update), {
|
|
309
311
|
pending: opts.pendingUpdates ?? memoryPending(),
|
|
@@ -532,9 +534,6 @@ export class Mesh extends ReadyResource {
|
|
|
532
534
|
this._background.stop();
|
|
533
535
|
this._ownWriterSession?.destroy();
|
|
534
536
|
await this._files?.close();
|
|
535
|
-
if (!this.autobee.closing) {
|
|
536
|
-
await this.autobee.update();
|
|
537
|
-
}
|
|
538
537
|
await Promise.all([this.autobee.close(), this.blobs.close()]);
|
|
539
538
|
await this.view.close();
|
|
540
539
|
this._blindPeering = null;
|
|
@@ -587,6 +586,7 @@ export class Mesh extends ReadyResource {
|
|
|
587
586
|
assertJoined(this.autobee, opts);
|
|
588
587
|
const write = this.autobee.append(encode(`@qvac/${name}`, versionedOp(name, payload)), opts);
|
|
589
588
|
await this._watchStall(write, { what: 'mesh write', op: name, ms: this._slowWriteMs });
|
|
589
|
+
assertWritten(this.autobee);
|
|
590
590
|
}
|
|
591
591
|
async _watchStall(pending, watch) {
|
|
592
592
|
const started = Date.now();
|
|
@@ -946,13 +946,15 @@ export class Mesh extends ReadyResource {
|
|
|
946
946
|
}
|
|
947
947
|
}
|
|
948
948
|
_newestLinked(db) {
|
|
949
|
-
|
|
950
|
-
|
|
949
|
+
return db.findOne(this._trustedIndex(), { reverse: true });
|
|
950
|
+
}
|
|
951
|
+
_trustedIndex() {
|
|
952
|
+
return this.ourMemberId ? LINKED_WRITERS_BY_MOST_RECENT : LIVE_WRITERS_BY_MOST_RECENT;
|
|
951
953
|
}
|
|
952
954
|
async _newestTrustedOf(candidate, ours) {
|
|
953
955
|
const held = this._heldOrServed(ours);
|
|
954
956
|
try {
|
|
955
|
-
for await (const writer of candidate.find(
|
|
957
|
+
for await (const writer of candidate.find(this._trustedIndex(), {
|
|
956
958
|
reverse: true
|
|
957
959
|
})) {
|
|
958
960
|
if (await isTrustedWriter(held, writer.writerKey, this)) {
|
|
@@ -987,24 +989,25 @@ export class Mesh extends ReadyResource {
|
|
|
987
989
|
const read = view.db.snapshot();
|
|
988
990
|
snap = read;
|
|
989
991
|
const tail = { reverse: true, limit: WARMUP_TAIL };
|
|
990
|
-
const [own, byId, name, rules,
|
|
992
|
+
const [own, byId, name, rules, live, linked, chats, groups] = await Promise.all([
|
|
991
993
|
liveDeviceForWriter(read, this.localWriterKey),
|
|
992
994
|
this._deviceId ? read.get(DEVICES, { id: this._deviceId }) : null,
|
|
993
995
|
read.get(MESH_NAMES, { id: MESH_NAME_ROW }),
|
|
994
996
|
read.get(MESH_RULES, { id: MESH_RULES_ROW }),
|
|
995
|
-
read.
|
|
996
|
-
read.
|
|
997
|
+
read.findOne(LIVE_WRITERS_BY_MOST_RECENT, { reverse: true }),
|
|
998
|
+
read.findOne(LINKED_WRITERS_BY_MOST_RECENT, { reverse: true }),
|
|
997
999
|
this._sidebarTop(read),
|
|
998
1000
|
read.find(ACTIVE_CHAT_GROUPS, tail).toArray()
|
|
999
1001
|
]);
|
|
1000
1002
|
const memberId = byId?.memberId ?? own?.memberId ?? null;
|
|
1003
|
+
const newest = [live, linked].filter((row) => row !== null);
|
|
1001
1004
|
const [member, counters, devices] = await Promise.all([
|
|
1002
1005
|
memberId ? read.get(MEMBERS, { id: memberId }) : null,
|
|
1003
1006
|
sidebarRows(read, chats, memberId),
|
|
1004
|
-
Promise.all(
|
|
1007
|
+
Promise.all(newest.map((writer) => liveDeviceForWriter(read, writer.writerKey)))
|
|
1005
1008
|
]);
|
|
1006
|
-
const rows = [own, byId, name, rules, member, ...
|
|
1007
|
-
const warmed = rows.length +
|
|
1009
|
+
const rows = [own, byId, name, rules, ...newest, member, ...devices, ...counters].filter((row) => row !== null);
|
|
1010
|
+
const warmed = rows.length + chats.length + groups.length;
|
|
1008
1011
|
this._log?.info(this._scope, 'warmed candidate view', {
|
|
1009
1012
|
warmed,
|
|
1010
1013
|
length: view.db.db.head()?.length ?? 0,
|
|
@@ -1096,6 +1099,7 @@ export class Mesh extends ReadyResource {
|
|
|
1096
1099
|
reached = await syncChanges(this, this.autobee.view, from);
|
|
1097
1100
|
}
|
|
1098
1101
|
else {
|
|
1102
|
+
await this.history?.edited(null);
|
|
1099
1103
|
this._arrive();
|
|
1100
1104
|
}
|
|
1101
1105
|
if (this._admitted && !this.autobee.writable) {
|
|
@@ -1251,6 +1255,12 @@ export class MeshBatch {
|
|
|
1251
1255
|
this._ops = [];
|
|
1252
1256
|
this._names = [];
|
|
1253
1257
|
await this._watch(names.join(','), this._autobee.append(ops, opts));
|
|
1258
|
+
assertWritten(this._autobee);
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
function assertWritten(autobee) {
|
|
1262
|
+
if (autobee.closing && autobee.appending) {
|
|
1263
|
+
throw new CoreError('MESH_CLOSED_BEFORE_WRITE');
|
|
1254
1264
|
}
|
|
1255
1265
|
}
|
|
1256
1266
|
function noop() { }
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import type { KeyPair } from 'hyperswarm';
|
|
2
|
+
import type { Logger } from '../../../../index.ts';
|
|
3
|
+
import type { QvacProject } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
4
|
+
import type { MeshRole } from '../../contract/roles.ts';
|
|
5
|
+
import { openLocalStore } from '../../local/index.ts';
|
|
6
|
+
import type { Mesh, MeshHistory, ProjectIdentity } from '../../mesh.ts';
|
|
7
|
+
import { type PublicRecord } from '../../public-mesh.ts';
|
|
8
|
+
import type { AgentPicks } from '../../shared-agents.ts';
|
|
9
|
+
export interface UpgradeHost {
|
|
10
|
+
readonly closing: Promise<void> | null;
|
|
11
|
+
readonly local: ReturnType<typeof openLocalStore>;
|
|
12
|
+
readonly log: Logger | null;
|
|
13
|
+
readonly ownWorkspaceId: string | null;
|
|
14
|
+
accountMesh(): Mesh | null;
|
|
15
|
+
agentPicks(): AgentPicks;
|
|
16
|
+
followProjects(workspace: Mesh): Promise<void>;
|
|
17
|
+
following(meshId: string): Promise<void> | undefined;
|
|
18
|
+
heldMesh(key: Buffer): Promise<Mesh | null>;
|
|
19
|
+
isAccount(mesh: Mesh): boolean;
|
|
20
|
+
isPooled(mesh: Mesh): boolean;
|
|
21
|
+
listProject(workspace: Mesh, project: Pick<Mesh, 'key' | 'seed'>, listing: Omit<QvacProject, 'id'>): Promise<void>;
|
|
22
|
+
meshIdOf(mesh: Mesh): string;
|
|
23
|
+
notifyStatus(): void;
|
|
24
|
+
openNewProject(workspace: Mesh, project: ProjectIdentity): Promise<Mesh>;
|
|
25
|
+
openPooledMesh(meshId: string): Promise<Mesh>;
|
|
26
|
+
pooled(meshId: string | null): Mesh | null;
|
|
27
|
+
publicMesh(workspaceId: string): Mesh | null;
|
|
28
|
+
publicRecord(account: Mesh, workspaceKey: Buffer): Promise<PublicRecord | null>;
|
|
29
|
+
publicWriter(mesh: Mesh): boolean;
|
|
30
|
+
purgeProject(key: Buffer): Promise<void>;
|
|
31
|
+
reopenInRole(mesh: Mesh, role: MeshRole): Promise<void>;
|
|
32
|
+
requireDeviceKeyPair(): KeyPair;
|
|
33
|
+
savePublicRecord(account: Mesh, workspaceKey: Buffer, record: PublicRecord): Promise<void>;
|
|
34
|
+
storedAccountKey(): Promise<Buffer | undefined>;
|
|
35
|
+
writableAccount(): Promise<Mesh | null>;
|
|
36
|
+
}
|
|
37
|
+
export declare class WorkspaceUpgrade {
|
|
38
|
+
private readonly _host;
|
|
39
|
+
private readonly _upgrading;
|
|
40
|
+
private readonly _groupPasses;
|
|
41
|
+
private readonly _groupPassesDue;
|
|
42
|
+
private _publicMigration;
|
|
43
|
+
private _copyAgain;
|
|
44
|
+
private _accountCopied;
|
|
45
|
+
constructor(host: UpgradeHost);
|
|
46
|
+
get accountCopy(): Promise<void> | null;
|
|
47
|
+
get accountUnread(): boolean;
|
|
48
|
+
followShared(mesh: Mesh): Promise<void>;
|
|
49
|
+
follow(workspace: Mesh): void;
|
|
50
|
+
private _passGroups;
|
|
51
|
+
private _rehome;
|
|
52
|
+
private _rehomeInto;
|
|
53
|
+
private _carryHome;
|
|
54
|
+
private _removeCarried;
|
|
55
|
+
private _groupBuckets;
|
|
56
|
+
private _markFoldDue;
|
|
57
|
+
private _passAgents;
|
|
58
|
+
private _heldProjects;
|
|
59
|
+
private _groupRecord;
|
|
60
|
+
private _scanBuckets;
|
|
61
|
+
private _passGroup;
|
|
62
|
+
private _followPours;
|
|
63
|
+
private _foundGroupProject;
|
|
64
|
+
private _mergeGroupInto;
|
|
65
|
+
private _winnerOf;
|
|
66
|
+
private _copyGroup;
|
|
67
|
+
private _copyBaseline;
|
|
68
|
+
private _addWorkspaceMembers;
|
|
69
|
+
private _settleListed;
|
|
70
|
+
private _listNewMembers;
|
|
71
|
+
private _listWorkspaceMembers;
|
|
72
|
+
boot(): Promise<void>;
|
|
73
|
+
private _markUpgradeChecked;
|
|
74
|
+
private _upgradable;
|
|
75
|
+
private _markJoinedWorkspace;
|
|
76
|
+
copyAccount(account: Mesh): void;
|
|
77
|
+
private _copyTargets;
|
|
78
|
+
private _copyAccountInto;
|
|
79
|
+
refill(into: Mesh): Promise<void>;
|
|
80
|
+
private _recordCopied;
|
|
81
|
+
private _markCopied;
|
|
82
|
+
history(mesh: () => Mesh): MeshHistory;
|
|
83
|
+
private _noteEdits;
|
|
84
|
+
private _noteLateWrites;
|
|
85
|
+
}
|