@qvac/core 1.0.0-rc.7 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +241 -3
- package/README.md +1 -0
- package/dist-lib/lib/api/attachments.d.ts +1 -1
- package/dist-lib/lib/api/attachments.js +11 -0
- package/dist-lib/lib/api/chunks.d.ts +4 -2
- package/dist-lib/lib/api/chunks.js +6 -3
- package/dist-lib/lib/api/device-routes.d.ts +6 -1
- package/dist-lib/lib/api/device-routes.js +20 -1
- package/dist-lib/lib/api/devices.js +6 -18
- package/dist-lib/lib/api/execution.d.ts +6 -2
- package/dist-lib/lib/api/execution.js +29 -3
- package/dist-lib/lib/api/host-run-log.d.ts +7 -1
- package/dist-lib/lib/api/host-run-log.js +3 -1
- package/dist-lib/lib/api/index.d.ts +3 -1
- package/dist-lib/lib/api/index.js +17 -6
- package/dist-lib/lib/api/members.js +13 -29
- package/dist-lib/lib/api/mesh.d.ts +2 -1
- package/dist-lib/lib/api/projects.d.ts +1 -1
- package/dist-lib/lib/api/projects.js +9 -5
- package/dist-lib/lib/api/rag-label.d.ts +4 -0
- package/dist-lib/lib/api/rag-label.js +21 -0
- package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
- package/dist-lib/lib/api/rag-lexical.js +1058 -0
- package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
- package/dist-lib/lib/api/rag-manifest.js +46 -0
- package/dist-lib/lib/api/rag-outline.d.ts +36 -0
- package/dist-lib/lib/api/rag-outline.js +426 -0
- package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
- package/dist-lib/lib/api/rag-plugin.js +70 -0
- package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
- package/dist-lib/lib/api/rag-synthesis.js +102 -0
- package/dist-lib/lib/api/rag-targets.d.ts +23 -0
- package/dist-lib/lib/api/rag-targets.js +215 -0
- package/dist-lib/lib/api/rag-window.d.ts +5 -0
- package/dist-lib/lib/api/rag-window.js +46 -0
- package/dist-lib/lib/api/rag.d.ts +3 -2
- package/dist-lib/lib/api/rag.js +421 -184
- package/dist-lib/lib/api/served.js +8 -14
- package/dist-lib/lib/api/workspace.d.ts +8 -1
- package/dist-lib/lib/api/workspace.js +27 -13
- package/dist-lib/lib/assistant/harness.d.ts +9 -1
- package/dist-lib/lib/assistant/history.d.ts +11 -2
- package/dist-lib/lib/assistant/history.js +74 -4
- package/dist-lib/lib/assistant/media-round.d.ts +61 -0
- package/dist-lib/lib/assistant/media-round.js +111 -0
- package/dist-lib/lib/assistant/message.d.ts +7 -0
- package/dist-lib/lib/assistant/model-round.d.ts +19 -1
- package/dist-lib/lib/assistant/model-round.js +163 -28
- package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
- package/dist-lib/lib/assistant/pdf-pages.js +67 -3
- package/dist-lib/lib/assistant/round-host.d.ts +22 -0
- package/dist-lib/lib/assistant/round-host.js +142 -8
- package/dist-lib/lib/assistant/upload.d.ts +12 -0
- package/dist-lib/lib/assistant/upload.js +36 -0
- package/dist-lib/lib/assistant-client.d.ts +1 -0
- package/dist-lib/lib/assistant-client.js +40 -19
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +48 -1
- package/dist-lib/lib/assistant.d.ts +18 -1
- package/dist-lib/lib/assistant.js +484 -95
- package/dist-lib/lib/client.d.ts +1 -0
- package/dist-lib/lib/client.js +4 -1
- package/dist-lib/lib/compaction.js +2 -2
- package/dist-lib/lib/core/agent.js +12 -45
- package/dist-lib/lib/core/approval-rules.js +1 -8
- package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
- package/dist-lib/lib/core/archived-mesh.js +12 -8
- package/dist-lib/lib/core/chat-copy.d.ts +18 -3
- package/dist-lib/lib/core/chat-copy.js +74 -25
- package/dist-lib/lib/core/chat-fork.js +15 -6
- package/dist-lib/lib/core/chat-group.js +16 -24
- package/dist-lib/lib/core/chat-move.d.ts +9 -0
- package/dist-lib/lib/core/chat-move.js +94 -3
- package/dist-lib/lib/core/chat-order.js +12 -18
- package/dist-lib/lib/core/chat.js +38 -31
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +18 -25
- package/dist-lib/lib/core/collections.d.ts +2 -0
- package/dist-lib/lib/core/collections.js +2 -0
- package/dist-lib/lib/core/contract/activity.d.ts +4 -0
- package/dist-lib/lib/core/contract/activity.js +34 -4
- package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
- package/dist-lib/lib/core/contract/chat-group.js +21 -1
- package/dist-lib/lib/core/contract/chat.d.ts +9 -0
- package/dist-lib/lib/core/contract/chat.js +127 -5
- package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
- package/dist-lib/lib/core/contract/chunks.js +37 -6
- package/dist-lib/lib/core/contract/features.d.ts +21 -0
- package/dist-lib/lib/core/contract/features.js +22 -0
- package/dist-lib/lib/core/contract/index.js +2 -0
- package/dist-lib/lib/core/contract/indexing.js +4 -2
- package/dist-lib/lib/core/contract/knowledge.js +5 -4
- package/dist-lib/lib/core/contract/members.js +16 -10
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
- package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
- package/dist-lib/lib/core/contract/projects.d.ts +5 -0
- package/dist-lib/lib/core/contract/projects.js +13 -5
- package/dist-lib/lib/core/contract/roles.js +32 -4
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
- package/dist-lib/lib/core/contract/run-requests.js +35 -2
- package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
- package/dist-lib/lib/core/contract/tool-configs.js +3 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
- package/dist-lib/lib/core/contract/workspace.js +40 -7
- package/dist-lib/lib/core/contract/writers.js +2 -2
- package/dist-lib/lib/core/departures.d.ts +3 -1
- package/dist-lib/lib/core/departures.js +3 -0
- package/dist-lib/lib/core/devices.js +1 -8
- package/dist-lib/lib/core/files/attachments.d.ts +2 -0
- package/dist-lib/lib/core/files/attachments.js +19 -1
- package/dist-lib/lib/core/files/index.d.ts +20 -4
- package/dist-lib/lib/core/files/index.js +132 -21
- package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
- package/dist-lib/lib/core/files/processing/document.js +7 -0
- package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
- package/dist-lib/lib/core/files/processing/index.js +144 -12
- package/dist-lib/lib/core/files/processing/protocol.js +13 -34
- package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
- package/dist-lib/lib/core/files/rag/build.js +123 -19
- package/dist-lib/lib/core/files/rag/failure.js +2 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
- package/dist-lib/lib/core/files/rag/index.js +13 -1
- package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
- package/dist-lib/lib/core/files/rag/load.js +27 -6
- package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
- package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
- package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/produce.js +104 -16
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
- package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
- package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
- package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
- package/dist-lib/lib/core/files/records/knowledge.js +7 -40
- package/dist-lib/lib/core/files/records/reclaim.js +5 -10
- package/dist-lib/lib/core/files/upload.js +68 -41
- package/dist-lib/lib/core/fs-drives.js +14 -26
- package/dist-lib/lib/core/invite.d.ts +1 -0
- package/dist-lib/lib/core/invite.js +4 -2
- package/dist-lib/lib/core/local/files.d.ts +1 -1
- package/dist-lib/lib/core/local/files.js +5 -5
- package/dist-lib/lib/core/local/index.d.ts +13 -6
- package/dist-lib/lib/core/local/index.js +103 -69
- package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
- package/dist-lib/lib/core/local/run-logs.js +5 -5
- package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
- package/dist-lib/lib/core/mesh-processor.js +87 -0
- package/dist-lib/lib/core/mesh.d.ts +48 -5
- package/dist-lib/lib/core/mesh.js +318 -129
- package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
- package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
- package/dist-lib/lib/core/migrations/writer-links.js +32 -0
- package/dist-lib/lib/core/project-keys.d.ts +6 -0
- package/dist-lib/lib/core/project-keys.js +11 -0
- package/dist-lib/lib/core/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/public-mesh.js +10 -8
- package/dist-lib/lib/core/run-requests.d.ts +3 -0
- package/dist-lib/lib/core/run-requests.js +21 -1
- package/dist-lib/lib/core/skill-availability.d.ts +33 -0
- package/dist-lib/lib/core/skill-availability.js +67 -0
- package/dist-lib/lib/core/sync/background.d.ts +0 -1
- package/dist-lib/lib/core/sync/background.js +2 -1
- package/dist-lib/lib/core/sync/changed.d.ts +4 -2
- package/dist-lib/lib/core/sync/changed.js +1 -1
- package/dist-lib/lib/core/sync/index.js +52 -10
- package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
- package/dist-lib/lib/core/sync/view-updates.js +82 -91
- package/dist-lib/lib/core/sync-status.js +97 -41
- package/dist-lib/lib/core/tool-activity.js +6 -12
- package/dist-lib/lib/core/tool-configs.js +17 -29
- package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
- package/dist-lib/lib/core/upgrade-summary.js +67 -0
- package/dist-lib/lib/core/view-read.d.ts +4 -0
- package/dist-lib/lib/core/view-read.js +23 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
- package/dist-lib/lib/core/workspace-upgrade.js +27 -16
- package/dist-lib/lib/core.d.ts +45 -42
- package/dist-lib/lib/core.js +675 -640
- package/dist-lib/lib/errors.d.ts +24 -0
- package/dist-lib/lib/errors.js +24 -0
- package/dist-lib/lib/llm-model-names.d.ts +1 -1
- package/dist-lib/lib/llm-model-names.js +1 -1
- package/dist-lib/lib/mesh-scoped-engine.js +4 -0
- package/dist-lib/lib/model-context.js +1 -2
- package/dist-lib/lib/model-download.d.ts +6 -1
- package/dist-lib/lib/model-download.js +16 -3
- package/dist-lib/lib/oauth/resolve.js +29 -8
- package/dist-lib/lib/oauth/token-exchange-core.js +11 -0
- package/dist-lib/lib/probe-rounds.d.ts +7 -2
- package/dist-lib/lib/probe-rounds.js +6 -1
- package/dist-lib/lib/run-transcript.d.ts +4 -1
- package/dist-lib/lib/run-transcript.js +15 -6
- package/dist-lib/schema/engine.js +83 -6
- package/dist-lib/schema/indexes.d.ts +3 -0
- package/dist-lib/schema/indexes.js +5 -0
- package/dist-lib/schema/local.js +11 -2
- package/dist-lib/schema/qvac.js +49 -5
- package/dist-lib/spec/bind.js +34 -0
- package/dist-lib/spec/calls.js +5 -0
- package/dist-lib/spec/capabilities.d.ts +17 -0
- package/dist-lib/spec/hrpc/hrpc.json +65 -0
- package/dist-lib/spec/hrpc/index.js +68 -5
- package/dist-lib/spec/hrpc/messages.js +260 -23
- package/dist-lib/spec/hyperschema/index.js +260 -23
- package/dist-lib/spec/hyperschema/schema.json +181 -0
- package/dist-lib/spec/hyperschema/types.d.ts +48 -0
- package/dist-lib/spec/local/hyperdb/messages.js +270 -224
- package/dist-lib/spec/local/hyperschema/index.js +73 -38
- package/dist-lib/spec/local/hyperschema/schema.json +31 -0
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
- package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
- package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
- package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
- package/package.json +82 -71
- package/tui/README.md +52 -0
- package/tui/app.mjs +69 -12
- package/tui/boot.mjs +27 -6
- package/tui/cli.mjs +93 -0
- package/tui/index.mjs +2 -87
- package/tui/lib/attachment-files.mjs +82 -0
- package/tui/lib/reopen-meshes.mjs +15 -0
- package/tui/lib/updater.mjs +86 -0
- package/tui/main.mjs +29 -57
- package/worker/assistant-sidecar.mjs +10 -3
- package/worker/stow-entry.mjs +3 -52
- package/dist-lib/lib/core/chat-project.d.ts +0 -2
- package/dist-lib/lib/core/chat-project.js +0 -15
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { trackActivity } from './activity.js';
|
|
2
|
-
import {
|
|
2
|
+
import { FEATURES, isFeatureSupported } from './features.js';
|
|
3
3
|
import { AGENTS, CHATS, MESH_BLIND_PEERS } from '../collections.js';
|
|
4
4
|
async function seedsChat({ role = 'account' }, tx) {
|
|
5
|
-
return role !== 'account' || (await
|
|
5
|
+
return role !== 'account' || !(await isFeatureSupported(tx, FEATURES.ACCOUNT_SEEDS_NO_CHAT));
|
|
6
6
|
}
|
|
7
7
|
export function registerSeedDefaultsRoutes(router, options = {}) {
|
|
8
8
|
router.add('@qvac/seed-defaults', trackActivity(async (op, { tx, log }) => {
|
|
@@ -2,7 +2,7 @@ import { toolConfigsWithinBounds } from '../../../schema/qvac.js';
|
|
|
2
2
|
import { toolConfigsCarrySecret } from '../tool-configs.js';
|
|
3
3
|
import { trackActivity } from './activity.js';
|
|
4
4
|
import { AGENTS, TOOL_CONFIGS } from '../collections.js';
|
|
5
|
-
import {
|
|
5
|
+
import { FEATURES, isFeatureSupported } from './features.js';
|
|
6
6
|
export function registerToolConfigsRoutes(router) {
|
|
7
7
|
router.add('@qvac/put-tool-configs', trackActivity(async ({ settings }, { tx, log }) => {
|
|
8
8
|
if (!toolConfigsWithinBounds(settings.settings)) {
|
|
@@ -30,7 +30,8 @@ export function registerToolConfigsRoutes(router) {
|
|
|
30
30
|
});
|
|
31
31
|
return;
|
|
32
32
|
}
|
|
33
|
-
if (!settings.deletedAt &&
|
|
33
|
+
if (!settings.deletedAt &&
|
|
34
|
+
!(await isFeatureSupported(tx, FEATURES.TOOL_CONFIGS_WITHOUT_AGENT))) {
|
|
34
35
|
const agent = await tx.get(AGENTS, { id: settings.agentId });
|
|
35
36
|
if (!agent || agent.deletedAt) {
|
|
36
37
|
log?.debug('contract', 'put-tool-configs dropped: agent missing or deleted', {
|
|
@@ -26,6 +26,8 @@ export declare function leavesNoAdmin(context: Signed, memberId: Buffer): Promis
|
|
|
26
26
|
export declare function kicksAdminsLastDevice(context: Signed, deviceId: Buffer | null | undefined): Promise<boolean>;
|
|
27
27
|
export declare function readMemberKey(tx: HyperDB, memberId: Buffer): Promise<QvacMemberKey | null>;
|
|
28
28
|
export declare function uncountMember(tx: HyperDB, memberId: Buffer): Promise<void>;
|
|
29
|
+
export declare function upgradeWriter(tx: HyperDB, writerKey: Buffer): Promise<boolean>;
|
|
29
30
|
export declare function upgradeMembers(tx: HyperDB): Promise<Buffer[]>;
|
|
31
|
+
export declare function countedMembers(tx: HyperDB): Promise<Buffer[]>;
|
|
30
32
|
export declare function registerWorkspaceRoutes(router: Router): void;
|
|
31
33
|
export {};
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { liveDeviceForWriter, liveDevices, liveDevicesOf, trackActivity } from './activity.js';
|
|
2
2
|
import { meshKind } from './mesh-kind.js';
|
|
3
|
-
import {
|
|
3
|
+
import { bornUnderRules } from './mesh-rules.js';
|
|
4
|
+
import { FEATURES, isFeatureSupported } from './features.js';
|
|
4
5
|
import b4a from 'b4a';
|
|
5
6
|
import { removedBy } from './invites.js';
|
|
6
7
|
import { hostsLiveAgent, readPublicSeed } from './public-seeds.js';
|
|
7
8
|
import { SEALED_SEED_BYTES } from '../device-seal.js';
|
|
8
|
-
import { DEVICES, MEMBER_KEYS, MEMBER_ROLES, MEMBERS, MESH_FOUNDERS, MESH_KINDS, PUBLIC_SEEDS, WORKSPACE } from '../collections.js';
|
|
9
|
+
import { DEVICES, LIVE_DEVICES_BY_WRITER_KEY, MEMBER_KEYS, MEMBER_ROLES, MEMBERS, MESH_FOUNDERS, MESH_KINDS, PUBLIC_SEEDS, WORKSPACE } from '../collections.js';
|
|
9
10
|
export const WORKSPACE_ROW = 'workspace';
|
|
10
11
|
export const WORKSPACE_ROLES = new Set(['admin', 'member']);
|
|
11
12
|
export function readWorkspace(tx) {
|
|
@@ -132,7 +133,12 @@ async function bornWorkspace({ workspace }, context) {
|
|
|
132
133
|
log?.warn('contract', 'put-workspace dropped: the genesis founded no member to be its admin');
|
|
133
134
|
return;
|
|
134
135
|
}
|
|
135
|
-
const row = {
|
|
136
|
+
const row = {
|
|
137
|
+
...workspace,
|
|
138
|
+
id: WORKSPACE_ROW,
|
|
139
|
+
upgradeMemberIds: undefined,
|
|
140
|
+
upgradeWriterKeys: undefined
|
|
141
|
+
};
|
|
136
142
|
await tx.insert(WORKSPACE, row);
|
|
137
143
|
await tx.insert(MEMBER_ROLES, { memberId: founder.memberId, role: 'admin' });
|
|
138
144
|
log?.info('contract', 'workspace born');
|
|
@@ -148,8 +154,13 @@ async function editedWorkspace(existing, { workspace }, context) {
|
|
|
148
154
|
log?.debug('contract', 'put-workspace dropped: not newer than the stored row');
|
|
149
155
|
return;
|
|
150
156
|
}
|
|
151
|
-
const { upgradeMemberIds } = existing;
|
|
152
|
-
const row = {
|
|
157
|
+
const { upgradeMemberIds, upgradeWriterKeys } = existing;
|
|
158
|
+
const row = {
|
|
159
|
+
...workspace,
|
|
160
|
+
id: WORKSPACE_ROW,
|
|
161
|
+
upgradeMemberIds,
|
|
162
|
+
upgradeWriterKeys
|
|
163
|
+
};
|
|
153
164
|
await tx.insert(WORKSPACE, row);
|
|
154
165
|
return row;
|
|
155
166
|
}
|
|
@@ -248,7 +259,7 @@ async function putMemberKey(op, context) {
|
|
|
248
259
|
async function upgradeToWorkspace(_op, context) {
|
|
249
260
|
const { tx, log } = context;
|
|
250
261
|
const kind = await meshKind(tx);
|
|
251
|
-
const onRules =
|
|
262
|
+
const onRules = await isFeatureSupported(tx, FEATURES.WORKSPACE_UPGRADE);
|
|
252
263
|
if (!onRules || (kind !== 'shared' && kind !== 'workspace') || (await bornUnderRules(tx))) {
|
|
253
264
|
log?.debug('contract', 'upgrade-to-workspace dropped: not a shared mesh born before the rules');
|
|
254
265
|
return;
|
|
@@ -270,11 +281,25 @@ async function upgradeToWorkspace(_op, context) {
|
|
|
270
281
|
}
|
|
271
282
|
}
|
|
272
283
|
await tx.insert(MESH_KINDS, { id: 'mesh', kind: 'workspace' });
|
|
273
|
-
const row = {
|
|
284
|
+
const row = {
|
|
285
|
+
id: WORKSPACE_ROW,
|
|
286
|
+
updatedAt: 0,
|
|
287
|
+
upgradeMemberIds,
|
|
288
|
+
upgradeWriterKeys: await liveWriterKeys(tx)
|
|
289
|
+
};
|
|
274
290
|
await tx.insert(WORKSPACE, row);
|
|
275
291
|
log?.info('contract', 'shared mesh upgraded to a workspace');
|
|
276
292
|
return row;
|
|
277
293
|
}
|
|
294
|
+
async function liveWriterKeys(tx) {
|
|
295
|
+
const keys = [];
|
|
296
|
+
for await (const { writerKey } of tx.find(LIVE_DEVICES_BY_WRITER_KEY)) {
|
|
297
|
+
if (writerKey) {
|
|
298
|
+
keys.push(writerKey);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
return keys;
|
|
302
|
+
}
|
|
278
303
|
async function countUpgrader(tx, signer, { log }) {
|
|
279
304
|
const row = await tx.get(WORKSPACE, { id: WORKSPACE_ROW });
|
|
280
305
|
const counted = row?.upgradeMemberIds ?? [];
|
|
@@ -295,10 +320,18 @@ export async function uncountMember(tx, memberId) {
|
|
|
295
320
|
await tx.insert(WORKSPACE, { ...row, upgradeMemberIds });
|
|
296
321
|
}
|
|
297
322
|
}
|
|
323
|
+
export async function upgradeWriter(tx, writerKey) {
|
|
324
|
+
const row = await tx.get(WORKSPACE, { id: WORKSPACE_ROW });
|
|
325
|
+
return !!row?.upgradeWriterKeys?.some((key) => b4a.equals(key, writerKey));
|
|
326
|
+
}
|
|
298
327
|
export async function upgradeMembers(tx) {
|
|
299
328
|
const row = await tx.get(WORKSPACE, { id: WORKSPACE_ROW });
|
|
300
329
|
return row?.upgradeMemberIds ?? [];
|
|
301
330
|
}
|
|
331
|
+
export async function countedMembers(tx) {
|
|
332
|
+
const live = await liveMemberIds(tx);
|
|
333
|
+
return (await upgradeMembers(tx)).filter((id) => live.has(b4a.toHex(id)));
|
|
334
|
+
}
|
|
302
335
|
export function registerWorkspaceRoutes(router) {
|
|
303
336
|
router.add('@qvac/put-workspace', trackActivity(async (op, context) => {
|
|
304
337
|
const existing = await readWorkspace(context.tx);
|
|
@@ -14,7 +14,7 @@ import { dropHostRows, hostedDevice, releaseHostedDevice } from './agent-join.js
|
|
|
14
14
|
import { actsFor, dropRole, kicksAdminsLastDevice, leavesNoAdmin, uncountMember } from './workspace.js';
|
|
15
15
|
import { verifyDeviceProof } from '../identity.js';
|
|
16
16
|
import { sealed } from './seal.js';
|
|
17
|
-
import {
|
|
17
|
+
import { FEATURES, isFeatureSupported } from './features.js';
|
|
18
18
|
import b4a from 'b4a';
|
|
19
19
|
import { DEVICES, MEMBERS, MESH_INVITE_GRANTS, PROJECT_MEMBERS } from '../collections.js';
|
|
20
20
|
async function boundToAnother(tx, writerKey, id) {
|
|
@@ -191,7 +191,7 @@ async function ownLeave({ writerKey, deviceId, revokedAt }, { tx, node }) {
|
|
|
191
191
|
if (!b4a.equals(node.key, writerKey) || !deviceId) {
|
|
192
192
|
return false;
|
|
193
193
|
}
|
|
194
|
-
if ((await
|
|
194
|
+
if (!(await isFeatureSupported(tx, FEATURES.OWN_LEAVE_HOLDS))) {
|
|
195
195
|
return false;
|
|
196
196
|
}
|
|
197
197
|
const device = await tx.get(DEVICES, { id: deviceId });
|
|
@@ -4,7 +4,7 @@ import type Wakeup from 'protomux-wakeup';
|
|
|
4
4
|
import type { LoggerInterface } from '@qvac/logging';
|
|
5
5
|
import type { LocalMeshDeparture } from '../../spec/local/hyperschema/types.d.ts';
|
|
6
6
|
import type BlindPeering from 'blind-peering';
|
|
7
|
-
import { Mesh } from './mesh.ts';
|
|
7
|
+
import { Mesh, type ErrorSink } from './mesh.ts';
|
|
8
8
|
import type { MeshRole } from './contract/roles.ts';
|
|
9
9
|
interface DepartureRows {
|
|
10
10
|
departures(): Promise<LocalMeshDeparture[]>;
|
|
@@ -16,6 +16,7 @@ export interface PendingDeparturesOptions {
|
|
|
16
16
|
isPooled: (key: Buffer) => boolean;
|
|
17
17
|
deviceId: () => Buffer | null;
|
|
18
18
|
logger?: LoggerInterface | null;
|
|
19
|
+
onerror?: ErrorSink;
|
|
19
20
|
blindPeerKeys?: ReadonlyArray<Buffer>;
|
|
20
21
|
blindPeering: () => BlindPeering | null;
|
|
21
22
|
wakeup?: Wakeup;
|
|
@@ -26,6 +27,7 @@ export declare class PendingDepartures {
|
|
|
26
27
|
private readonly _local;
|
|
27
28
|
private readonly _swarm;
|
|
28
29
|
private readonly _log;
|
|
30
|
+
private readonly _onerror;
|
|
29
31
|
private readonly _blindPeerKeys;
|
|
30
32
|
private readonly _wakeup;
|
|
31
33
|
private readonly _isPooled;
|
|
@@ -7,6 +7,7 @@ export class PendingDepartures {
|
|
|
7
7
|
_local;
|
|
8
8
|
_swarm;
|
|
9
9
|
_log;
|
|
10
|
+
_onerror;
|
|
10
11
|
_blindPeerKeys;
|
|
11
12
|
_wakeup;
|
|
12
13
|
_isPooled;
|
|
@@ -25,6 +26,7 @@ export class PendingDepartures {
|
|
|
25
26
|
this._deviceId = opts.deviceId;
|
|
26
27
|
this._delivered = opts.delivered ?? noop;
|
|
27
28
|
this._log = opts.logger ?? null;
|
|
29
|
+
this._onerror = opts.onerror;
|
|
28
30
|
this._blindPeerKeys = opts.blindPeerKeys ?? [];
|
|
29
31
|
this._wakeup = opts.wakeup;
|
|
30
32
|
this._blindPeering = opts.blindPeering;
|
|
@@ -52,6 +54,7 @@ export class PendingDepartures {
|
|
|
52
54
|
writerSeed: row.writerSeed ?? undefined,
|
|
53
55
|
deviceId: this._deviceId() ?? undefined,
|
|
54
56
|
logger: this._log,
|
|
57
|
+
onerror: this._onerror,
|
|
55
58
|
blindPeerKeys: this._blindPeerKeys,
|
|
56
59
|
wakeup: this._wakeup
|
|
57
60
|
});
|
|
@@ -8,14 +8,7 @@ export async function saveDeviceCapabilities(store, input, isAccount) {
|
|
|
8
8
|
export async function retractInventory(store, deviceId, isAccount) {
|
|
9
9
|
if (isAccount(store))
|
|
10
10
|
return;
|
|
11
|
-
const
|
|
12
|
-
let row;
|
|
13
|
-
try {
|
|
14
|
-
row = await snap.get(DEVICES, { id: deviceId });
|
|
15
|
-
}
|
|
16
|
-
finally {
|
|
17
|
-
await snap.close();
|
|
18
|
-
}
|
|
11
|
+
const row = await store.view.get(DEVICES, { id: deviceId });
|
|
19
12
|
if (!row) {
|
|
20
13
|
return;
|
|
21
14
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { AssistantPutBlobRequest, AssistantPutBlobResponse } from '../../../spec/hyperschema/types.d.ts';
|
|
1
2
|
import type { Mesh } from '../mesh.ts';
|
|
2
3
|
import { type FileHandlersOptions } from './upload.ts';
|
|
3
4
|
import type { CapabilityHandlers } from '../../../spec/capabilities.d.ts';
|
|
@@ -7,6 +8,7 @@ export interface BoundAttachments {
|
|
|
7
8
|
uploadAttachment: CapabilityHandlers['uploadAttachment'];
|
|
8
9
|
downloadAttachment: Bound<'downloadAttachment'>;
|
|
9
10
|
deleteAttachment: Bound<'deleteAttachment'>;
|
|
11
|
+
putBlob(frames: AsyncIterable<AssistantPutBlobRequest>): AsyncGenerator<AssistantPutBlobResponse>;
|
|
10
12
|
}
|
|
11
13
|
export declare function attachmentFiles(mesh: Mesh, { maxBytes, deviceId, inactivityMs }?: FileHandlersOptions): BoundAttachments;
|
|
12
14
|
export {};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { CoreError } from '../../errors.js';
|
|
2
2
|
import { linkAttachment } from '../attachment-link.js';
|
|
3
|
+
import { mayEditChunk } from '../contract/chunks.js';
|
|
3
4
|
import { getChunk, readChunk, updateChunk } from '../chunk.js';
|
|
4
5
|
import { uploadFile } from './upload.js';
|
|
5
6
|
import b4a from 'b4a';
|
|
@@ -23,6 +24,21 @@ export function attachmentFiles(mesh, { maxBytes = MAX_ATTACHMENT_BYTES, deviceI
|
|
|
23
24
|
downloadAttachment: async function* ({ blobId }) {
|
|
24
25
|
yield* mesh.files.read(blobId);
|
|
25
26
|
},
|
|
27
|
+
putBlob: async function* (frames) {
|
|
28
|
+
const pieces = [];
|
|
29
|
+
let bytesWritten = 0;
|
|
30
|
+
for await (const { data } of frames) {
|
|
31
|
+
if (!data)
|
|
32
|
+
continue;
|
|
33
|
+
bytesWritten += data.length;
|
|
34
|
+
if (bytesWritten > maxBytes) {
|
|
35
|
+
throw new CoreError('UPLOAD_TOO_LARGE', ['attachment', maxBytes]);
|
|
36
|
+
}
|
|
37
|
+
pieces.push(data);
|
|
38
|
+
}
|
|
39
|
+
const { reference } = await mesh.files.put(b4a.concat(pieces));
|
|
40
|
+
yield { bytesWritten, blobId: reference };
|
|
41
|
+
},
|
|
26
42
|
async deleteAttachment({ chunkId, at }) {
|
|
27
43
|
if (!at || at < 0) {
|
|
28
44
|
throw new Error(`delete-attachment needs a positive timestamp, got ${at}`);
|
|
@@ -34,7 +50,9 @@ export function attachmentFiles(mesh, { maxBytes = MAX_ATTACHMENT_BYTES, deviceI
|
|
|
34
50
|
if (chunk.type !== 'attachment' || !chunk.attachment) {
|
|
35
51
|
throw new CoreError('NOT_AN_ATTACHMENT', [chunkId]);
|
|
36
52
|
}
|
|
37
|
-
if (deviceId &&
|
|
53
|
+
if (deviceId &&
|
|
54
|
+
!b4a.equals(chunk.originDeviceId, deviceId) &&
|
|
55
|
+
!(await mayEditChunk(mesh.view, chunk, mesh.localWriterKey))) {
|
|
38
56
|
throw new CoreError('ATTACHMENT_DELETE_NOT_ORIGIN');
|
|
39
57
|
}
|
|
40
58
|
const attachment = chunk.attachment;
|
|
@@ -3,12 +3,14 @@ import EventEmitter from 'bare-events';
|
|
|
3
3
|
import type { AssistantFileStatus, AssistantFileStatusList, AssistantFileReadRequest, AssistantFileReadFrame, AssistantFilePreviewResponse, AssistantIndexingStatusList, AssistantKnowledgeBaseList } from '../../../spec/hyperschema/types.d.ts';
|
|
4
4
|
import type { QvacChunk, QvacKnowledge, QvacKnowledgeArtifact } from '../../../spec/mesh/hyperschema/types.d.ts';
|
|
5
5
|
import { attachmentFiles } from './attachments.ts';
|
|
6
|
+
import { type CachedText } from './rag/text-cache.ts';
|
|
7
|
+
import type { PreparedText } from './rag/build.ts';
|
|
6
8
|
import { addKnowledgeArtifact, type FileRecord } from './records/index.ts';
|
|
7
9
|
import { knowledgeFiles } from './knowledge.ts';
|
|
8
10
|
import { FileState, type LocalWork, type FileProcessor } from './processing/index.ts';
|
|
9
11
|
import type ScopeLock from 'scope-lock';
|
|
10
12
|
import { BlobStore } from './store/index.ts';
|
|
11
|
-
import { type IndexingDevice, type IndexingProgress, type IndexingEmbedder, type
|
|
13
|
+
import { type IndexingDevice, type IndexingProgress, type IndexingEmbedder, type KnowledgeExtractors, type VectorIndex } from './rag/index.ts';
|
|
12
14
|
import type { Mesh } from '../mesh.ts';
|
|
13
15
|
import type { ChatFileKey } from './records/source.ts';
|
|
14
16
|
export { fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, type VectorHit, type RagDocumentRef } from './rag/index.ts';
|
|
@@ -29,9 +31,10 @@ export interface StartProcessorOptions {
|
|
|
29
31
|
readonly device?: IndexingDevice;
|
|
30
32
|
}
|
|
31
33
|
export interface StartProcessingOptions extends StartProcessorOptions {
|
|
34
|
+
readonly textDir?: string;
|
|
32
35
|
readonly embedder: IndexingEmbedder;
|
|
33
36
|
readonly index: VectorIndex;
|
|
34
|
-
readonly extractors?:
|
|
37
|
+
readonly extractors?: KnowledgeExtractors;
|
|
35
38
|
readonly batches?: ScopeLock;
|
|
36
39
|
}
|
|
37
40
|
export interface FileLoadOptions {
|
|
@@ -53,6 +56,7 @@ export declare class Files extends EventEmitter {
|
|
|
53
56
|
private readonly _mesh;
|
|
54
57
|
private readonly _uploads;
|
|
55
58
|
readonly state: FileState;
|
|
59
|
+
private _text;
|
|
56
60
|
private readonly _states;
|
|
57
61
|
private readonly _processors;
|
|
58
62
|
private readonly _store;
|
|
@@ -79,6 +83,7 @@ export declare class Files extends EventEmitter {
|
|
|
79
83
|
knowledge(options?: Parameters<typeof knowledgeFiles>[1]): import("./knowledge.ts").KnowledgeFiles;
|
|
80
84
|
listKnowledge(chatId: string): Promise<QvacKnowledge[]>;
|
|
81
85
|
knowledgeBase(): Promise<AssistantKnowledgeBaseList>;
|
|
86
|
+
private _embedding;
|
|
82
87
|
record(id: string): Promise<QvacKnowledge | null>;
|
|
83
88
|
resolve(request: ChatFileKey): Promise<FileRecord | null>;
|
|
84
89
|
fileRead(request: Omit<AssistantFileReadRequest, 'meshId'>, options?: FileReadOptions): AsyncGenerator<AssistantFileReadFrame>;
|
|
@@ -115,7 +120,18 @@ export declare class Files extends EventEmitter {
|
|
|
115
120
|
clearLoaded(processorId?: string): void;
|
|
116
121
|
hydratedArtifact(id: string): string | null;
|
|
117
122
|
ready(chatId: string, ids: string[]): Promise<AssistantFileStatusList>;
|
|
123
|
+
private _demand;
|
|
124
|
+
hasText(id: string): boolean;
|
|
125
|
+
hasUsableText(id: string, blobId: string): Promise<boolean>;
|
|
126
|
+
keepText(id: string, blobId: string, prepared: PreparedText): Promise<void>;
|
|
127
|
+
textOf(id: string, blobId: string): Promise<CachedText | null>;
|
|
128
|
+
forgetText(id: string): Promise<void>;
|
|
129
|
+
textReadyHere(chatId: string, ids: string[], { untilMs }?: {
|
|
130
|
+
untilMs?: number | undefined;
|
|
131
|
+
}): Promise<void>;
|
|
132
|
+
extractingHere(file: AssistantFileStatus): boolean;
|
|
118
133
|
indexingHere(chatId: string, ids: string[]): Promise<void>;
|
|
134
|
+
requestVectors(ids: string[]): Promise<void>;
|
|
119
135
|
indexesHere(file: AssistantFileStatus): boolean;
|
|
120
136
|
private _until;
|
|
121
137
|
private _changes;
|
|
@@ -134,8 +150,8 @@ export declare class Files extends EventEmitter {
|
|
|
134
150
|
private _processorStatus;
|
|
135
151
|
close(): Promise<void>;
|
|
136
152
|
}
|
|
137
|
-
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
|
|
138
|
-
export type { IndexingDevice, IndexingEmbedder, RagExtractors, RagSource, VectorIndex } from './rag/index.ts';
|
|
153
|
+
export { MemoryVectorIndex, PluginVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, resolveTargetDocument, selectCoverage, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
|
|
154
|
+
export type { IndexingDevice, IndexingEmbedder, KnowledgeExtractors, PluginHit, RagExtractors, ReadPdfPages, RagSource, VectorIndex } from './rag/index.ts';
|
|
139
155
|
export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.ts';
|
|
140
156
|
export { CoreBlobServer, PENDING_BLOB_REFERENCE, isPendingBlobReference, parseBlobReferenceCoreKey } from './store/index.ts';
|
|
141
157
|
export type { MeshBlobKeys } from './store/index.ts';
|
|
@@ -2,6 +2,7 @@ import b4a from 'b4a';
|
|
|
2
2
|
import EventEmitter from 'bare-events';
|
|
3
3
|
import { Readable } from 'streamx';
|
|
4
4
|
import { attachmentFiles } from './attachments.js';
|
|
5
|
+
import { TextCache } from './rag/text-cache.js';
|
|
5
6
|
import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, reclaimPurgedFiles, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId, knowledgeCapBytes, usedKnowledgeBytes } from './records/index.js';
|
|
6
7
|
import { knowledgeFiles } from './knowledge.js';
|
|
7
8
|
import { randomId } from '../id.js';
|
|
@@ -19,6 +20,7 @@ import { parseFailure } from './rag/failure.js';
|
|
|
19
20
|
import { CoreError } from '../../errors.js';
|
|
20
21
|
import { errorMessage } from '../../guards.js';
|
|
21
22
|
import { ACTIVE_KNOWLEDGE_BASE, ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS, INDEXING_STATUS_BY_CHAT, INDEXING_STATUS_BY_KNOWLEDGE, KNOWLEDGE, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE } from '../collections.js';
|
|
23
|
+
const TEXT_WAIT_MS = 30_000;
|
|
22
24
|
export { fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
|
|
23
25
|
export { isKnowledgeRoutedAttachment } from './records/index.js';
|
|
24
26
|
export { assertRemovableAt } from './records/index.js';
|
|
@@ -28,6 +30,7 @@ export class Files extends EventEmitter {
|
|
|
28
30
|
_mesh;
|
|
29
31
|
_uploads = new Map();
|
|
30
32
|
state = new FileState();
|
|
33
|
+
_text = null;
|
|
31
34
|
_states = new Map([['rag', this.state]]);
|
|
32
35
|
_processors = new Map();
|
|
33
36
|
_store;
|
|
@@ -125,7 +128,8 @@ export class Files extends EventEmitter {
|
|
|
125
128
|
error: failure?.detail,
|
|
126
129
|
loaded: !!this.state.get(knowledge.id)?.loaded,
|
|
127
130
|
reason: failure?.reason,
|
|
128
|
-
failedOn: device?.name
|
|
131
|
+
failedOn: device?.name,
|
|
132
|
+
progress: status?.status === 'ready' ? undefined : this._embedding(knowledge.id, rag)
|
|
129
133
|
});
|
|
130
134
|
}
|
|
131
135
|
return {
|
|
@@ -138,6 +142,15 @@ export class Files extends EventEmitter {
|
|
|
138
142
|
await snap.close();
|
|
139
143
|
}
|
|
140
144
|
}
|
|
145
|
+
_embedding(knowledgeId, attempts) {
|
|
146
|
+
const progress = this.state.get(knowledgeId)?.progress;
|
|
147
|
+
const now = Date.now();
|
|
148
|
+
const claimed = attempts.some((row) => isClaimLive(row, now) && b4a.equals(row.claimedBy, this._mesh.localWriterKey));
|
|
149
|
+
if (progress?.stage !== 'embedding' || !claimed) {
|
|
150
|
+
return undefined;
|
|
151
|
+
}
|
|
152
|
+
return chunkProgress(progress);
|
|
153
|
+
}
|
|
141
154
|
record(id) {
|
|
142
155
|
return getKnowledge(this._mesh, id);
|
|
143
156
|
}
|
|
@@ -191,7 +204,12 @@ export class Files extends EventEmitter {
|
|
|
191
204
|
return;
|
|
192
205
|
}
|
|
193
206
|
this.processorState(processorId).retry(file.id);
|
|
194
|
-
|
|
207
|
+
if (processorId === 'rag' && file.chatId) {
|
|
208
|
+
await entry.work.enqueueVectors([file.id]);
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
await entry.work.enqueue([file.id]);
|
|
212
|
+
}
|
|
195
213
|
this._updated();
|
|
196
214
|
}
|
|
197
215
|
async _retryKnowledgeBase(fileId) {
|
|
@@ -292,10 +310,13 @@ export class Files extends EventEmitter {
|
|
|
292
310
|
return this._processors.get('rag')?.work;
|
|
293
311
|
}
|
|
294
312
|
startProcessing(options) {
|
|
313
|
+
this._text = options.textDir ? new TextCache(options.textDir) : null;
|
|
314
|
+
void this._text?.list();
|
|
295
315
|
this.setModel(options.embedder.model);
|
|
296
316
|
this.startProcessor(new RagProcessor({
|
|
297
317
|
...options,
|
|
298
318
|
model: options.embedder.model,
|
|
319
|
+
deferChatVectors: this._mesh.role !== 'project',
|
|
299
320
|
log: this._mesh.log
|
|
300
321
|
}), { deviceId: options.deviceId, device: options.device });
|
|
301
322
|
}
|
|
@@ -326,7 +347,7 @@ export class Files extends EventEmitter {
|
|
|
326
347
|
.map((row) => ({ ...row, chatId, claimLive: isClaimLive(row, Date.now()) }));
|
|
327
348
|
for (const file of (await this.status(chatId)).files) {
|
|
328
349
|
const processor = file.processors.find((entry) => entry.processorId === 'rag');
|
|
329
|
-
if (!processor ||
|
|
350
|
+
if (!processor || processor.state === 'not-applicable') {
|
|
330
351
|
continue;
|
|
331
352
|
}
|
|
332
353
|
if (processor.claimedBy) {
|
|
@@ -342,12 +363,12 @@ export class Files extends EventEmitter {
|
|
|
342
363
|
id: latest?.id ?? file.fileId,
|
|
343
364
|
knowledgeId: file.fileId,
|
|
344
365
|
chatId,
|
|
345
|
-
status: processor.state
|
|
366
|
+
status: indexingStatus(processor.state),
|
|
346
367
|
claimId: '',
|
|
347
368
|
claimedBy: b4a.alloc(0),
|
|
348
369
|
at: Math.max(latest?.at ?? 0, processor.retryAt ?? 0),
|
|
349
370
|
expiresAt: 0,
|
|
350
|
-
error: processor.reason ?? 'waiting for local index',
|
|
371
|
+
error: processor.state === 'ready' ? undefined : (processor.reason ?? 'waiting for local index'),
|
|
351
372
|
claimLive: false
|
|
352
373
|
});
|
|
353
374
|
}
|
|
@@ -388,6 +409,9 @@ export class Files extends EventEmitter {
|
|
|
388
409
|
else if (entry.loaded?.artifactId !== artifactId) {
|
|
389
410
|
entry.loaded = { artifactId };
|
|
390
411
|
}
|
|
412
|
+
if (entry.loaded) {
|
|
413
|
+
this.processorState('rag').document(id).prepared = undefined;
|
|
414
|
+
}
|
|
391
415
|
this._updated();
|
|
392
416
|
}
|
|
393
417
|
clearLoaded(processorId = 'rag') {
|
|
@@ -397,27 +421,104 @@ export class Files extends EventEmitter {
|
|
|
397
421
|
hydratedArtifact(id) {
|
|
398
422
|
return this.state.get(id)?.loaded?.artifactId ?? null;
|
|
399
423
|
}
|
|
400
|
-
ready(chatId, ids) {
|
|
424
|
+
async ready(chatId, ids) {
|
|
401
425
|
const wanted = new Set(ids);
|
|
402
|
-
this.
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
426
|
+
const release = this._demand(ids);
|
|
427
|
+
try {
|
|
428
|
+
return await this._until(chatId, (all) => {
|
|
429
|
+
const files = all.filter((file) => wanted.has(file.fileId));
|
|
430
|
+
return files.every(settledFile) ? files : null;
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
finally {
|
|
434
|
+
release();
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
_demand(ids) {
|
|
438
|
+
return this._processing?.demand(ids) ?? noop;
|
|
439
|
+
}
|
|
440
|
+
hasText(id) {
|
|
441
|
+
const prepared = this.processorState('rag').get(id)?.prepared;
|
|
442
|
+
return !!prepared || !!this._text?.has(id);
|
|
443
|
+
}
|
|
444
|
+
async hasUsableText(id, blobId) {
|
|
445
|
+
const known = this._text?.usable(id, blobId);
|
|
446
|
+
if (known !== undefined) {
|
|
447
|
+
return known;
|
|
448
|
+
}
|
|
449
|
+
const text = await this.textOf(id, blobId);
|
|
450
|
+
return !!text?.content.trim();
|
|
451
|
+
}
|
|
452
|
+
async keepText(id, blobId, prepared) {
|
|
453
|
+
if (!this._text) {
|
|
454
|
+
this.processorState('rag').document(id).prepared = { blobId, ...prepared };
|
|
455
|
+
return;
|
|
456
|
+
}
|
|
457
|
+
await this._text.write(id, { blobId, ...prepared });
|
|
458
|
+
}
|
|
459
|
+
async textOf(id, blobId) {
|
|
460
|
+
const prepared = this.processorState('rag').get(id)?.prepared;
|
|
461
|
+
if (prepared?.blobId === blobId) {
|
|
462
|
+
return prepared;
|
|
463
|
+
}
|
|
464
|
+
if (prepared) {
|
|
465
|
+
this.processorState('rag').document(id).prepared = undefined;
|
|
466
|
+
}
|
|
467
|
+
return (await this._text?.read(id, blobId)) ?? null;
|
|
468
|
+
}
|
|
469
|
+
async forgetText(id) {
|
|
470
|
+
await this._text?.remove(id);
|
|
471
|
+
}
|
|
472
|
+
async textReadyHere(chatId, ids, { untilMs = TEXT_WAIT_MS } = {}) {
|
|
473
|
+
const wanted = new Set(ids);
|
|
474
|
+
const release = this._demand(ids);
|
|
475
|
+
let expired = false;
|
|
476
|
+
const deadline = new DeadlineScheduler(() => {
|
|
477
|
+
expired = true;
|
|
478
|
+
this._updated();
|
|
406
479
|
});
|
|
480
|
+
deadline.schedule(Date.now() + untilMs);
|
|
481
|
+
try {
|
|
482
|
+
await this._until(chatId, (files) => !expired && files.some((file) => wanted.has(file.fileId) && this.extractingHere(file))
|
|
483
|
+
? null
|
|
484
|
+
: files);
|
|
485
|
+
}
|
|
486
|
+
finally {
|
|
487
|
+
deadline.clear();
|
|
488
|
+
release();
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
extractingHere(file) {
|
|
492
|
+
return this.indexesHere(file) && !this.hasText(file.fileId);
|
|
407
493
|
}
|
|
408
494
|
async indexingHere(chatId, ids) {
|
|
409
495
|
const wanted = new Set(ids);
|
|
410
|
-
this.
|
|
411
|
-
|
|
496
|
+
const release = this._demand(ids);
|
|
497
|
+
try {
|
|
498
|
+
await this._processing?.enqueueVectors(ids);
|
|
499
|
+
await this._until(chatId, (files) => files.some((file) => wanted.has(file.fileId) && this.indexesHere(file)) ? null : files);
|
|
500
|
+
}
|
|
501
|
+
finally {
|
|
502
|
+
release();
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
requestVectors(ids) {
|
|
506
|
+
const release = this._demand(ids);
|
|
507
|
+
return (this._processing?.enqueueVectors(ids) ?? Promise.resolve()).finally(release);
|
|
412
508
|
}
|
|
413
509
|
indexesHere(file) {
|
|
414
510
|
const rag = file.processors.find((entry) => entry.processorId === 'rag');
|
|
415
|
-
if (!this._processing?.active ||
|
|
511
|
+
if (!this._processing?.active || rag?.state === 'ready' || rag?.state === 'not-applicable') {
|
|
416
512
|
return false;
|
|
417
513
|
}
|
|
418
514
|
if (this._processing.awaitsBytes(file.fileId)) {
|
|
419
515
|
return false;
|
|
420
516
|
}
|
|
517
|
+
if (rag?.state === 'failed' &&
|
|
518
|
+
rag.retryAt === undefined &&
|
|
519
|
+
!this._processing.pending(file.fileId)) {
|
|
520
|
+
return false;
|
|
521
|
+
}
|
|
421
522
|
if (rag?.claimedBy) {
|
|
422
523
|
return b4a.equals(rag.claimedBy, this._mesh.localWriterKey);
|
|
423
524
|
}
|
|
@@ -684,14 +785,10 @@ export class Files extends EventEmitter {
|
|
|
684
785
|
if (local?.progress &&
|
|
685
786
|
processor.state !== 'not-applicable' &&
|
|
686
787
|
(!claim || b4a.equals(claim.claimedBy, this._mesh.localWriterKey))) {
|
|
687
|
-
processor.progress =
|
|
688
|
-
completed: local.progress.processedChunks,
|
|
689
|
-
total: local.progress.totalChunks || undefined,
|
|
690
|
-
unit: 'chunks'
|
|
691
|
-
};
|
|
788
|
+
processor.progress = chunkProgress(local.progress);
|
|
692
789
|
}
|
|
693
790
|
const readyForChat = applicable
|
|
694
|
-
? processor.state === 'ready'
|
|
791
|
+
? processor.state === 'ready' || (await this.hasUsableText(id, source.blobId))
|
|
695
792
|
: await this._store.available(source.blobId);
|
|
696
793
|
const processors = [
|
|
697
794
|
processor,
|
|
@@ -797,8 +894,15 @@ export class Files extends EventEmitter {
|
|
|
797
894
|
await this._store.close();
|
|
798
895
|
}
|
|
799
896
|
}
|
|
897
|
+
function chunkProgress(progress) {
|
|
898
|
+
return {
|
|
899
|
+
completed: progress.processedChunks,
|
|
900
|
+
total: progress.totalChunks || undefined,
|
|
901
|
+
unit: 'chunks'
|
|
902
|
+
};
|
|
903
|
+
}
|
|
800
904
|
function settledFile(file) {
|
|
801
|
-
return file.readyForChat || file.state === 'failed';
|
|
905
|
+
return file.readyForChat || file.state === 'failed' || file.state === 'not-applicable';
|
|
802
906
|
}
|
|
803
907
|
function yieldsToVerdict(work) {
|
|
804
908
|
return work.state === 'pending' || (work.state === 'failed' && work.retryAt === undefined);
|
|
@@ -824,6 +928,13 @@ function groupBy(items, key) {
|
|
|
824
928
|
}
|
|
825
929
|
return groups;
|
|
826
930
|
}
|
|
827
|
-
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.js';
|
|
931
|
+
export { MemoryVectorIndex, PluginVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, resolveTargetDocument, selectCoverage, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.js';
|
|
828
932
|
export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.js';
|
|
829
933
|
export { CoreBlobServer, PENDING_BLOB_REFERENCE, isPendingBlobReference, parseBlobReferenceCoreKey } from './store/index.js';
|
|
934
|
+
function noop() { }
|
|
935
|
+
function indexingStatus(state) {
|
|
936
|
+
if (state === 'ready') {
|
|
937
|
+
return 'ready';
|
|
938
|
+
}
|
|
939
|
+
return state === 'running' ? 'indexing' : 'failed';
|
|
940
|
+
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { LoggerInterface } from '@qvac/logging';
|
|
2
|
+
import type { PreparedText } from '../rag/build.ts';
|
|
3
|
+
import type { CachedText } from '../rag/text-cache.ts';
|
|
2
4
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
3
5
|
import { type IndexingProgress } from '../rag/index.ts';
|
|
4
6
|
import { IndexingClaims } from './claims.ts';
|
|
@@ -46,6 +48,8 @@ export declare class FileDocument {
|
|
|
46
48
|
private _fetched;
|
|
47
49
|
private readonly _progress?;
|
|
48
50
|
private readonly _processor;
|
|
51
|
+
keepText(prepared: PreparedText): Promise<void>;
|
|
52
|
+
keptText(artifactId?: string): Promise<CachedText | null>;
|
|
49
53
|
constructor(mesh: Mesh, options: FileDocumentOptions);
|
|
50
54
|
get log(): LoggerInterface | null;
|
|
51
55
|
get localCoreKey(): string;
|
|
@@ -19,6 +19,13 @@ export class FileDocument {
|
|
|
19
19
|
_fetched = false;
|
|
20
20
|
_progress;
|
|
21
21
|
_processor;
|
|
22
|
+
keepText(prepared) {
|
|
23
|
+
return this._mesh.files.keepText(this.record.id, this.record.blobId, prepared);
|
|
24
|
+
}
|
|
25
|
+
async keptText(artifactId) {
|
|
26
|
+
const text = await this._mesh.files.textOf(this.record.id, this.record.blobId);
|
|
27
|
+
return !artifactId || text?.artifactId === artifactId ? text : null;
|
|
28
|
+
}
|
|
22
29
|
constructor(mesh, options) {
|
|
23
30
|
const { record } = options;
|
|
24
31
|
this._mesh = mesh;
|