@qvac/core 1.0.0-rc.2 → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +158 -2
- package/dist-lib/index.d.ts +1 -0
- package/dist-lib/index.js +1 -0
- package/dist-lib/lib/api/chunk-delta.d.ts +7 -2
- package/dist-lib/lib/api/chunk-delta.js +89 -7
- package/dist-lib/lib/api/chunks.js +14 -8
- package/dist-lib/lib/api/config.d.ts +4 -1
- package/dist-lib/lib/api/config.js +45 -4
- package/dist-lib/lib/api/custom-models.js +3 -5
- package/dist-lib/lib/api/devices.js +3 -1
- package/dist-lib/lib/api/files.d.ts +1 -1
- package/dist-lib/lib/api/files.js +65 -8
- package/dist-lib/lib/api/instrument.js +12 -4
- package/dist-lib/lib/api/members.js +7 -13
- package/dist-lib/lib/api/mesh.d.ts +1 -3
- package/dist-lib/lib/api/mesh.js +2 -2
- package/dist-lib/lib/api/notifications.js +8 -4
- package/dist-lib/lib/api/projects.js +5 -1
- package/dist-lib/lib/api/rag.js +32 -19
- package/dist-lib/lib/api/served.js +2 -1
- package/dist-lib/lib/api/workspace.js +0 -1
- package/dist-lib/lib/assistant/harness.d.ts +8 -2
- package/dist-lib/lib/assistant/harness.js +17 -4
- package/dist-lib/lib/assistant/history.d.ts +1 -1
- package/dist-lib/lib/assistant/history.js +19 -13
- package/dist-lib/lib/assistant/message.d.ts +6 -1
- package/dist-lib/lib/assistant/message.js +22 -2
- package/dist-lib/lib/assistant/model-round.d.ts +5 -0
- package/dist-lib/lib/assistant/model-round.js +46 -18
- package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
- package/dist-lib/lib/assistant/pdf-budget.js +10 -11
- package/dist-lib/lib/assistant/pdf-pages.d.ts +25 -0
- package/dist-lib/lib/assistant/pdf-pages.js +75 -0
- package/dist-lib/lib/assistant/round-host.js +5 -3
- package/dist-lib/lib/assistant-client.d.ts +2 -1
- package/dist-lib/lib/assistant-client.js +24 -6
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +8 -5
- package/dist-lib/lib/assistant.d.ts +20 -7
- package/dist-lib/lib/assistant.js +468 -240
- package/dist-lib/lib/client.js +10 -1
- package/dist-lib/lib/compaction.d.ts +3 -1
- package/dist-lib/lib/compaction.js +18 -4
- package/dist-lib/lib/constants.d.ts +3 -0
- package/dist-lib/lib/constants.js +3 -0
- package/dist-lib/lib/core/chat-activity.js +13 -16
- package/dist-lib/lib/core/chat-copy.js +1 -1
- package/dist-lib/lib/core/chat-fork.js +2 -1
- package/dist-lib/lib/core/chat-move.d.ts +4 -0
- package/dist-lib/lib/core/chat-move.js +42 -11
- package/dist-lib/lib/core/chat.d.ts +0 -1
- package/dist-lib/lib/core/chat.js +1 -1
- package/dist-lib/lib/core/chunk.d.ts +14 -3
- package/dist-lib/lib/core/chunk.js +112 -22
- package/dist-lib/lib/core/collections.d.ts +12 -1
- package/dist-lib/lib/core/collections.js +12 -1
- package/dist-lib/lib/core/config-value.d.ts +6 -0
- package/dist-lib/lib/core/config-value.js +14 -0
- package/dist-lib/lib/core/contract/activity.d.ts +3 -2
- package/dist-lib/lib/core/contract/activity.js +39 -21
- package/dist-lib/lib/core/contract/agent-join.js +15 -19
- package/dist-lib/lib/core/contract/agents.js +8 -7
- package/dist-lib/lib/core/contract/chat.d.ts +3 -0
- package/dist-lib/lib/core/contract/chat.js +12 -1
- package/dist-lib/lib/core/contract/chunks.js +16 -6
- package/dist-lib/lib/core/contract/devices.js +6 -4
- package/dist-lib/lib/core/contract/index.js +2 -1
- package/dist-lib/lib/core/contract/invites.js +12 -31
- package/dist-lib/lib/core/contract/joined-meshes.js +2 -1
- package/dist-lib/lib/core/contract/mesh-rules.js +4 -0
- package/dist-lib/lib/core/contract/model-requests.js +7 -6
- package/dist-lib/lib/core/contract/project-device-keys.js +15 -6
- package/dist-lib/lib/core/contract/project-host-keys.js +5 -4
- package/dist-lib/lib/core/contract/projects.js +12 -13
- package/dist-lib/lib/core/contract/public-seeds.js +5 -5
- package/dist-lib/lib/core/contract/roles.js +2 -1
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -1
- package/dist-lib/lib/core/contract/run-requests.js +19 -13
- package/dist-lib/lib/core/contract/shipped-rows.d.ts +2 -0
- package/dist-lib/lib/core/contract/shipped-rows.js +22 -0
- package/dist-lib/lib/core/contract/workspace.js +4 -10
- package/dist-lib/lib/core/contract/writers.js +20 -6
- package/dist-lib/lib/core/devices.d.ts +2 -1
- package/dist-lib/lib/core/devices.js +4 -2
- package/dist-lib/lib/core/document-attachment.d.ts +6 -3
- package/dist-lib/lib/core/document-attachment.js +10 -5
- package/dist-lib/lib/core/files/attachments.js +3 -3
- package/dist-lib/lib/core/files/index.d.ts +8 -5
- package/dist-lib/lib/core/files/index.js +109 -18
- package/dist-lib/lib/core/files/processing/index.d.ts +1 -1
- package/dist-lib/lib/core/files/processing/index.js +8 -2
- package/dist-lib/lib/core/files/processing/source.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/source.js +3 -0
- package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/state.js +7 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/build.js +14 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/index.js +1 -1
- package/dist-lib/lib/core/files/rag/produce.js +1 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +7 -1
- package/dist-lib/lib/core/first-event.d.ts +6 -0
- package/dist-lib/lib/core/first-event.js +13 -0
- package/dist-lib/lib/core/invite.d.ts +2 -2
- package/dist-lib/lib/core/invite.js +5 -4
- package/dist-lib/lib/core/local/index.d.ts +10 -1
- package/dist-lib/lib/core/local/index.js +105 -10
- package/dist-lib/lib/core/mesh.d.ts +15 -4
- package/dist-lib/lib/core/mesh.js +128 -57
- package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
- package/dist-lib/lib/core/migrations/chat-chunks.js +47 -0
- package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
- package/dist-lib/lib/core/migrations/file-index.js +6 -2
- package/dist-lib/lib/core/migrations/index.d.ts +2 -1
- package/dist-lib/lib/core/migrations/index.js +66 -9
- package/dist-lib/lib/core/notification.js +96 -23
- package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
- package/dist-lib/lib/core/pdf/figure.js +64 -0
- package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
- package/dist-lib/lib/core/pdf/rasterize.js +72 -22
- package/dist-lib/lib/core/peer-duplex.js +57 -5
- package/dist-lib/lib/core/public-mesh.js +8 -2
- package/dist-lib/lib/core/run-requests.js +2 -3
- package/dist-lib/lib/core/senders.d.ts +7 -6
- package/dist-lib/lib/core/senders.js +23 -30
- package/dist-lib/lib/core/shared-agents.d.ts +1 -0
- package/dist-lib/lib/core/shared-agents.js +47 -19
- package/dist-lib/lib/core/sync/background.d.ts +3 -15
- package/dist-lib/lib/core/sync/background.js +25 -102
- package/dist-lib/lib/core/sync/devices.js +6 -3
- package/dist-lib/lib/core/sync/index.js +12 -2
- package/dist-lib/lib/core/sync-status.js +3 -3
- package/dist-lib/lib/core.d.ts +59 -10
- package/dist-lib/lib/core.js +706 -174
- package/dist-lib/lib/credential-pusher.d.ts +11 -7
- package/dist-lib/lib/credential-pusher.js +40 -8
- package/dist-lib/lib/device-tree.d.ts +18 -0
- package/dist-lib/lib/device-tree.js +32 -0
- package/dist-lib/lib/doctor/redact.js +1 -1
- package/dist-lib/lib/errors.d.ts +4 -0
- package/dist-lib/lib/errors.js +4 -0
- package/dist-lib/lib/fs-drive-bridge.d.ts +7 -0
- package/dist-lib/lib/fs-drive-bridge.js +308 -30
- package/dist-lib/lib/integrations/setup-routes.js +7 -5
- package/dist-lib/lib/knowledge-search.d.ts +1 -1
- package/dist-lib/lib/knowledge-search.js +5 -46
- package/dist-lib/lib/kv-cache-retention.d.ts +35 -0
- package/dist-lib/lib/kv-cache-retention.js +98 -0
- package/dist-lib/lib/latest-frame.d.ts +10 -0
- package/dist-lib/lib/latest-frame.js +48 -0
- package/dist-lib/lib/model-download.d.ts +6 -1
- package/dist-lib/lib/model-download.js +32 -17
- package/dist-lib/lib/oauth/resolve.js +13 -5
- package/dist-lib/lib/reconnecting-watch.js +19 -11
- package/dist-lib/lib/run-notices.d.ts +2 -0
- package/dist-lib/lib/run-notices.js +5 -0
- package/dist-lib/lib/run-replay.d.ts +13 -1
- package/dist-lib/lib/run-replay.js +58 -21
- package/dist-lib/schema/build.js +66 -57
- package/dist-lib/schema/content.js +2 -1
- package/dist-lib/schema/engine.js +37 -5
- package/dist-lib/schema/field-order.js +7 -1
- package/dist-lib/schema/indexes.d.ts +4 -0
- package/dist-lib/schema/indexes.js +18 -0
- package/dist-lib/schema/local.js +9 -2
- package/dist-lib/schema/qvac.js +112 -4
- package/dist-lib/spec/bind.js +3 -0
- package/dist-lib/spec/calls.js +1 -0
- package/dist-lib/spec/capabilities.d.ts +2 -0
- package/dist-lib/spec/hrpc/hrpc.json +13 -0
- package/dist-lib/spec/hrpc/index.js +15 -3
- package/dist-lib/spec/hrpc/messages.js +1475 -1392
- package/dist-lib/spec/hyperschema/index.js +1475 -1392
- package/dist-lib/spec/hyperschema/schema.json +86 -1
- package/dist-lib/spec/hyperschema/types.d.ts +21 -1
- package/dist-lib/spec/local/hyperdb/messages.js +110 -54
- package/dist-lib/spec/local/hyperschema/index.js +56 -28
- package/dist-lib/spec/local/hyperschema/schema.json +55 -20
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -1
- package/dist-lib/spec/mesh/hyperdb/db.json +255 -51
- package/dist-lib/spec/mesh/hyperdb/index.js +1063 -404
- package/dist-lib/spec/mesh/hyperdb/messages.js +1647 -1214
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +62 -56
- package/dist-lib/spec/mesh/hyperdispatch/index.js +156 -139
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +787 -629
- package/dist-lib/spec/mesh/hyperschema/index.js +787 -629
- package/dist-lib/spec/mesh/hyperschema/schema.json +283 -174
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +29 -1
- package/package.json +23 -17
- package/worker/assistant-sidecar.mjs +3 -0
- package/dist-lib/lib/core/files/processing/settle.d.ts +0 -13
- package/dist-lib/lib/core/files/processing/settle.js +0 -86
- package/dist-lib/lib/core/migrations/device-indexes.d.ts +0 -2
- package/dist-lib/lib/core/migrations/device-indexes.js +0 -7
|
@@ -79,7 +79,8 @@ export const ROUTE_ROLES = {
|
|
|
79
79
|
'@qvac/put-agent-user': WORKSPACE,
|
|
80
80
|
'@qvac/agent-join': CONTENT,
|
|
81
81
|
'@qvac/take-run-request': CONTENT,
|
|
82
|
-
'@qvac/put-skill-secret': ['account']
|
|
82
|
+
'@qvac/put-skill-secret': ['account'],
|
|
83
|
+
'@qvac/reindex-chat-chunks': CONTENT
|
|
83
84
|
};
|
|
84
85
|
export function inRole(role, route) {
|
|
85
86
|
return ROUTE_ROLES[route]?.includes(role) ?? false;
|
|
@@ -2,7 +2,7 @@ import type HyperDB from 'hyperdb';
|
|
|
2
2
|
import type { Router } from '../../../spec/mesh/hyperdispatch/index.js';
|
|
3
3
|
import type { QvacAppendChunkOperation, QvacChunk } from '../../../spec/mesh/hyperschema/types.d.ts';
|
|
4
4
|
import type { RouterContext } from './types.ts';
|
|
5
|
-
export declare const TERMINAL_RUN_STATES:
|
|
5
|
+
export declare const TERMINAL_RUN_STATES: ReadonlySet<string>;
|
|
6
6
|
export declare function runHasTerminalStatus(tx: HyperDB, chatId: string, runId: string): Promise<boolean>;
|
|
7
7
|
export declare function runnerRefusal(op: QvacAppendChunkOperation, { tx, node }: RouterContext): Promise<string | null>;
|
|
8
8
|
export declare function runTaker(tx: HyperDB, request: QvacChunk, writerKey: Buffer): Promise<Buffer | null>;
|
|
@@ -1,21 +1,27 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
2
|
import { liveDeviceForWriter, trackActivity } from './activity.js';
|
|
3
|
+
import { putChunk } from './chat.js';
|
|
3
4
|
import { ownerSigner } from './model-requests.js';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
|
|
5
|
+
import { WORKSPACES_RULES, rulesVersion } from './mesh-rules.js';
|
|
6
|
+
import { chunkTypeOrdinal, terminalRunStates } from '../../../schema/indexes.js';
|
|
7
|
+
import { CHUNKS, CHUNKS_BY_TYPE, DEVICES, TERMINAL_RUN_STATUSES } from '../collections.js';
|
|
8
|
+
export const TERMINAL_RUN_STATES = new Set(terminalRunStates);
|
|
7
9
|
export async function runHasTerminalStatus(tx, chatId, runId) {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
10
|
+
if ((await rulesVersion(tx)) < WORKSPACES_RULES) {
|
|
11
|
+
const type = chunkTypeOrdinal('run-status');
|
|
12
|
+
const statuses = tx.find(CHUNKS_BY_TYPE, {
|
|
13
|
+
gte: { chatId, type },
|
|
14
|
+
lte: { chatId, type }
|
|
15
|
+
});
|
|
16
|
+
for await (const chunk of statuses) {
|
|
17
|
+
if (chunk.runId === runId && TERMINAL_RUN_STATES.has(chunk.runStatus?.state ?? '')) {
|
|
18
|
+
return true;
|
|
19
|
+
}
|
|
16
20
|
}
|
|
21
|
+
return false;
|
|
17
22
|
}
|
|
18
|
-
|
|
23
|
+
const range = { gte: { chatId, runId }, lte: { chatId, runId } };
|
|
24
|
+
return (await tx.findOne(TERMINAL_RUN_STATUSES, range)) !== null;
|
|
19
25
|
}
|
|
20
26
|
export async function runnerRefusal(op, { tx, node }) {
|
|
21
27
|
if (!op.runnerDeviceId) {
|
|
@@ -69,7 +75,7 @@ export function registerRunRequestRoutes(router) {
|
|
|
69
75
|
runTakenAt: op.at,
|
|
70
76
|
updatedAt: Math.max(request.updatedAt ?? 0, op.at)
|
|
71
77
|
};
|
|
72
|
-
await tx
|
|
78
|
+
await putChunk(tx, taken);
|
|
73
79
|
return taken;
|
|
74
80
|
}));
|
|
75
81
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { chunkTypeOrdinal } from '../../../schema/indexes.js';
|
|
2
|
+
import { CHUNK_COUNTERS, CHUNKS, CHUNKS_BY_TYPE, DEVICES } from '../collections.js';
|
|
3
|
+
export async function reindexShippedRows(tx) {
|
|
4
|
+
const devices = await tx.find(DEVICES).toArray();
|
|
5
|
+
for (const device of devices) {
|
|
6
|
+
await tx.insert(DEVICES, { ...device, version: undefined });
|
|
7
|
+
}
|
|
8
|
+
const statuses = [];
|
|
9
|
+
for await (const chunk of runStatuses(tx)) {
|
|
10
|
+
statuses.push(chunk);
|
|
11
|
+
}
|
|
12
|
+
for (const chunk of statuses) {
|
|
13
|
+
await tx.insert(CHUNKS, { ...chunk, version: undefined });
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
async function* runStatuses(db) {
|
|
17
|
+
const type = chunkTypeOrdinal('run-status');
|
|
18
|
+
for await (const { chatId } of db.find(CHUNK_COUNTERS)) {
|
|
19
|
+
const range = { gte: { chatId, type }, lte: { chatId, type } };
|
|
20
|
+
yield* db.find(CHUNKS_BY_TYPE, range);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { liveDeviceForWriter, trackActivity } from './activity.js';
|
|
1
|
+
import { liveDeviceForWriter, liveDevices, liveDevicesOf, trackActivity } from './activity.js';
|
|
2
2
|
import { meshKind } from './mesh-kind.js';
|
|
3
3
|
import { WORKSPACES_RULES, bornUnderRules, rulesVersion } from './mesh-rules.js';
|
|
4
4
|
import b4a from 'b4a';
|
|
@@ -52,8 +52,8 @@ function ownerOf(device) {
|
|
|
52
52
|
}
|
|
53
53
|
export async function liveMemberIds(tx) {
|
|
54
54
|
const live = new Set();
|
|
55
|
-
for await (const device of tx
|
|
56
|
-
const owner =
|
|
55
|
+
for await (const device of liveDevices(tx)) {
|
|
56
|
+
const owner = ownerOf(device);
|
|
57
57
|
if (owner) {
|
|
58
58
|
live.add(b4a.toHex(owner));
|
|
59
59
|
}
|
|
@@ -100,13 +100,7 @@ export async function adminsLastDevice(tx, device) {
|
|
|
100
100
|
if (device.revoked || !owner || (await memberRole(tx, owner)) !== 'admin') {
|
|
101
101
|
return false;
|
|
102
102
|
}
|
|
103
|
-
|
|
104
|
-
const theirs = ownerOf(other);
|
|
105
|
-
if (!other.revoked && theirs && b4a.equals(theirs, owner) && !b4a.equals(other.id, device.id)) {
|
|
106
|
-
return false;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
return true;
|
|
103
|
+
return (await liveDevicesOf(tx, owner)).every((other) => b4a.equals(other.id, device.id));
|
|
110
104
|
}
|
|
111
105
|
export async function leavesNoAdmin(context, memberId) {
|
|
112
106
|
if (!(await isWorkspace(context.tx))) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import crypto from 'hypercore-crypto';
|
|
2
|
-
import { devicesByWriterKey, liveDeviceForWriter, markWriterRevoked, trackActivity } from './activity.js';
|
|
2
|
+
import { devicesByWriterKey, devicesOfMember, liveDeviceForWriter, markWriterRevoked, trackActivity } from './activity.js';
|
|
3
3
|
import { recordMemberRemoval } from './grant-order.js';
|
|
4
4
|
import { verifyMemberWriterClaim, verifyWriterClaim } from '../device-attestation.js';
|
|
5
5
|
import { meshKind, sharedKind } from './mesh-kind.js';
|
|
@@ -14,6 +14,7 @@ import { dropHostRows, hostedDevice, releaseHostedDevice } from './agent-join.js
|
|
|
14
14
|
import { actsFor, dropRole, kicksAdminsLastDevice, leavesNoAdmin, uncountMember } from './workspace.js';
|
|
15
15
|
import { verifyDeviceProof } from '../identity.js';
|
|
16
16
|
import { sealed } from './seal.js';
|
|
17
|
+
import { WORKSPACES_RULES, rulesVersion } from './mesh-rules.js';
|
|
17
18
|
import b4a from 'b4a';
|
|
18
19
|
import { DEVICES, MEMBERS, MESH_INVITE_GRANTS, PROJECT_MEMBERS } from '../collections.js';
|
|
19
20
|
async function boundToAnother(tx, writerKey, id) {
|
|
@@ -186,6 +187,19 @@ async function revokeWriter(payload, context) {
|
|
|
186
187
|
}
|
|
187
188
|
return revoked;
|
|
188
189
|
}
|
|
190
|
+
async function ownLeave({ writerKey, deviceId, revokedAt }, { tx, node }) {
|
|
191
|
+
if (!b4a.equals(node.key, writerKey) || !deviceId) {
|
|
192
|
+
return false;
|
|
193
|
+
}
|
|
194
|
+
if ((await rulesVersion(tx)) < WORKSPACES_RULES) {
|
|
195
|
+
return false;
|
|
196
|
+
}
|
|
197
|
+
const device = await tx.get(DEVICES, { id: deviceId });
|
|
198
|
+
if (!device?.writerKey || !b4a.equals(device.writerKey, writerKey)) {
|
|
199
|
+
return false;
|
|
200
|
+
}
|
|
201
|
+
return device.joinedAt <= (revokedAt ?? 0);
|
|
202
|
+
}
|
|
189
203
|
export function registerWriterRoutes(router) {
|
|
190
204
|
router.add('@qvac/add-writer', trackActivity(async (payload, context) => {
|
|
191
205
|
if (await meshKind(context.tx)) {
|
|
@@ -219,6 +233,10 @@ export function registerWriterRoutes(router) {
|
|
|
219
233
|
return payload.device;
|
|
220
234
|
}));
|
|
221
235
|
router.add('@qvac/remove-writer', trackActivity(async (payload, context) => {
|
|
236
|
+
if (context.node.optimistic && !(await ownLeave(payload, context))) {
|
|
237
|
+
context.log?.warn('contract', 'remove-writer dropped: its author is not a writer here');
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
222
240
|
const self = b4a.equals(context.node.key, payload.writerKey);
|
|
223
241
|
if (!self && !(await hasStanding(context))) {
|
|
224
242
|
context.log?.warn('contract', 'remove-writer dropped: the author speaks for no member here');
|
|
@@ -256,11 +274,7 @@ export function registerWriterRoutes(router) {
|
|
|
256
274
|
await context.tx.delete(PROJECT_MEMBERS, { memberId });
|
|
257
275
|
const removals = [];
|
|
258
276
|
const devices = [];
|
|
259
|
-
for
|
|
260
|
-
const claims = device.memberId ?? device.claimedMemberId;
|
|
261
|
-
if (!claims || !b4a.equals(claims, memberId)) {
|
|
262
|
-
continue;
|
|
263
|
-
}
|
|
277
|
+
for (const device of await devicesOfMember(context.tx, memberId)) {
|
|
264
278
|
devices.push(device.id);
|
|
265
279
|
if (!device.revoked && device.writerKey) {
|
|
266
280
|
removals.push({ writerKey: device.writerKey, deviceId: device.id, revokedAt });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { QvacToolReportEntry } from '../../spec/mesh/hyperschema/types.d.ts';
|
|
1
|
+
import type { QvacCustomModelReportEntry, QvacToolReportEntry } from '../../spec/mesh/hyperschema/types.d.ts';
|
|
2
2
|
import type { AppBuildMetadata } from './app-build.ts';
|
|
3
3
|
import type { Mesh } from './mesh.ts';
|
|
4
4
|
export interface SaveDeviceCapabilitiesInput {
|
|
@@ -9,6 +9,7 @@ export interface SaveDeviceCapabilitiesInput {
|
|
|
9
9
|
cachedModelNames: string[];
|
|
10
10
|
readySkillNames?: string[] | null;
|
|
11
11
|
toolCatalog?: QvacToolReportEntry[] | null;
|
|
12
|
+
customModels?: QvacCustomModelReportEntry[] | null;
|
|
12
13
|
}
|
|
13
14
|
export declare function saveDeviceCapabilities(store: Mesh, input: SaveDeviceCapabilitiesInput, isAccount: (mesh: Mesh) => boolean): Promise<void>;
|
|
14
15
|
export declare function retractInventory(store: Mesh, deviceId: Buffer, isAccount: (mesh: Mesh) => boolean): Promise<void>;
|
|
@@ -32,7 +32,8 @@ export async function retractInventory(store, deviceId, isAccount) {
|
|
|
32
32
|
row.installedSkillNames,
|
|
33
33
|
row.installedModelNames,
|
|
34
34
|
row.cachedModelNames,
|
|
35
|
-
row.toolCatalog
|
|
35
|
+
row.toolCatalog,
|
|
36
|
+
row.customModels
|
|
36
37
|
].some((list) => list?.length) ||
|
|
37
38
|
(row.readySkillNames?.length ?? 0) > (trimmed.readySkillNames?.length ?? 0);
|
|
38
39
|
if (leaked) {
|
|
@@ -48,7 +49,8 @@ async function hostedOnly(store, input) {
|
|
|
48
49
|
installedModelNames: [],
|
|
49
50
|
cachedModelNames: [],
|
|
50
51
|
readySkillNames: readinessForAgents(input.readySkillNames ?? [], hosted),
|
|
51
|
-
toolCatalog: []
|
|
52
|
+
toolCatalog: [],
|
|
53
|
+
customModels: []
|
|
52
54
|
};
|
|
53
55
|
}
|
|
54
56
|
export async function advertiseDeviceVersion(store, deviceId, app) {
|
|
@@ -25,11 +25,14 @@ export declare function documentUnreadReason({ knowledgeOn, failedFiles, attache
|
|
|
25
25
|
export declare function documentUnreadNotice(reason: DocumentUnreadReason, plural: boolean): string;
|
|
26
26
|
export interface PdfPagesCutState {
|
|
27
27
|
readonly fileName: string;
|
|
28
|
-
readonly
|
|
28
|
+
readonly cutAt: number;
|
|
29
29
|
readonly pageCount: number;
|
|
30
|
+
readonly shown: number;
|
|
31
|
+
readonly wholeText: boolean;
|
|
30
32
|
}
|
|
31
|
-
export declare function pdfPagesCutNotice({ fileName,
|
|
32
|
-
export declare
|
|
33
|
+
export declare function pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText }: PdfPagesCutState): string;
|
|
34
|
+
export declare const PDF_PAGES_SOURCE_LINE = "Page images from the PDFs in this chat are a source too: when an image shows what the excerpts do not, answer from the image.";
|
|
35
|
+
export declare function pdfPagesShownNotice(fileName: string, labels: ReadonlyArray<string>): string;
|
|
33
36
|
export interface AnnotatedMessage {
|
|
34
37
|
content: string;
|
|
35
38
|
documents?: DocumentAttachment[];
|
|
@@ -33,11 +33,16 @@ export function documentUnreadNotice(reason, plural) {
|
|
|
33
33
|
}[reason];
|
|
34
34
|
return `[The attached document${plural ? 's' : ''} could not be read for this reply: ${why}.]`;
|
|
35
35
|
}
|
|
36
|
-
export function pdfPagesCutNotice({ fileName,
|
|
37
|
-
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
|
|
36
|
+
export function pdfPagesCutNotice({ fileName, cutAt, pageCount, shown, wholeText }) {
|
|
37
|
+
const missing = shown
|
|
38
|
+
? `Some pictures in "${fileName}" (${pageCount} pages), the first on page ${cutAt + 1}, were not shown to you as images`
|
|
39
|
+
: `No picture in "${fileName}" (${pageCount} pages) was shown to you as an image`;
|
|
40
|
+
const text = wholeText ? '; its text is all above' : '';
|
|
41
|
+
return `[${missing}: they did not fit in the context window, and no tool can show them${text}. If asked about a picture you cannot see, say it could not be shown. Do not guess its content.]`;
|
|
42
|
+
}
|
|
43
|
+
export const PDF_PAGES_SOURCE_LINE = 'Page images from the PDFs in this chat are a source too: when an image shows what the excerpts do not, answer from the image.';
|
|
44
|
+
export function pdfPagesShownNotice(fileName, labels) {
|
|
45
|
+
return `[The images from "${fileName}" are, in order: ${labels.join('; ')}.]`;
|
|
41
46
|
}
|
|
42
47
|
export function annotateDocumentAttachments(messages) {
|
|
43
48
|
for (const turn of messages.filter((message) => message.documents?.length)) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CoreError } from '../../errors.js';
|
|
2
2
|
import { linkAttachment } from '../attachment-link.js';
|
|
3
|
-
import { getChunk, updateChunk } from '../chunk.js';
|
|
3
|
+
import { getChunk, readChunk, updateChunk } from '../chunk.js';
|
|
4
4
|
import { uploadFile } from './upload.js';
|
|
5
5
|
import b4a from 'b4a';
|
|
6
6
|
export const MAX_ATTACHMENT_BYTES = 100 * 1024 * 1024;
|
|
@@ -39,7 +39,7 @@ export function attachmentFiles(mesh, { maxBytes = MAX_ATTACHMENT_BYTES, deviceI
|
|
|
39
39
|
}
|
|
40
40
|
const attachment = chunk.attachment;
|
|
41
41
|
if (attachment.deletedAt) {
|
|
42
|
-
return chunk;
|
|
42
|
+
return readChunk(chunk);
|
|
43
43
|
}
|
|
44
44
|
const updated = await updateChunk(mesh, {
|
|
45
45
|
id: chunkId,
|
|
@@ -48,7 +48,7 @@ export function attachmentFiles(mesh, { maxBytes = MAX_ATTACHMENT_BYTES, deviceI
|
|
|
48
48
|
if (!updated.attachment?.deletedAt) {
|
|
49
49
|
throw new CoreError('ATTACHMENT_DELETE_DROPPED');
|
|
50
50
|
}
|
|
51
|
-
return updated;
|
|
51
|
+
return readChunk(updated);
|
|
52
52
|
}
|
|
53
53
|
};
|
|
54
54
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import EventEmitter from 'bare-events';
|
|
3
|
-
import type { AssistantFileStatusList, AssistantFileReadRequest, AssistantFileReadFrame, AssistantFilePreviewResponse, AssistantIndexingStatusList, AssistantKnowledgeBaseList } from '../../../spec/hyperschema/types.d.ts';
|
|
3
|
+
import type { AssistantFileStatus, AssistantFileStatusList, AssistantFileReadRequest, AssistantFileReadFrame, AssistantFilePreviewResponse, AssistantIndexingStatusList, AssistantKnowledgeBaseList } from '../../../spec/hyperschema/types.d.ts';
|
|
4
4
|
import type { QvacChunk, QvacKnowledge, QvacKnowledgeArtifact } from '../../../spec/mesh/hyperschema/types.d.ts';
|
|
5
5
|
import { attachmentFiles } from './attachments.ts';
|
|
6
6
|
import { addKnowledgeArtifact, type FileRecord } from './records/index.ts';
|
|
@@ -11,8 +11,7 @@ import { BlobStore } from './store/index.ts';
|
|
|
11
11
|
import { type IndexingDevice, type IndexingProgress, type IndexingEmbedder, type RagExtractors, type VectorIndex } from './rag/index.ts';
|
|
12
12
|
import type { Mesh } from '../mesh.ts';
|
|
13
13
|
import type { ChatFileKey } from './records/source.ts';
|
|
14
|
-
|
|
15
|
-
export { fullTextBudgetChars, fullTextHits, joinedIndex, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, type VectorHit, type RagDocumentRef } from './rag/index.ts';
|
|
14
|
+
export { fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, type VectorHit, type RagDocumentRef } from './rag/index.ts';
|
|
16
15
|
export { isKnowledgeRoutedAttachment } from './records/index.ts';
|
|
17
16
|
export { assertRemovableAt } from './records/index.ts';
|
|
18
17
|
export { assertArtifactOutput } from './records/index.ts';
|
|
@@ -115,7 +114,11 @@ export declare class Files extends EventEmitter {
|
|
|
115
114
|
load(id: string, state: LocalWork, options?: FileLoadOptions): void;
|
|
116
115
|
clearLoaded(processorId?: string): void;
|
|
117
116
|
hydratedArtifact(id: string): string | null;
|
|
118
|
-
|
|
117
|
+
ready(chatId: string, ids: string[]): Promise<AssistantFileStatusList>;
|
|
118
|
+
indexingHere(chatId: string): Promise<void>;
|
|
119
|
+
indexesHere(file: AssistantFileStatus): boolean;
|
|
120
|
+
private _until;
|
|
121
|
+
private _changes;
|
|
119
122
|
reconnect(): void;
|
|
120
123
|
report: (update: Parameters<IndexingProgress>[0], processorId?: string) => void;
|
|
121
124
|
progress(chatId: string): import("./processing/state.ts").ChatProgressEntry[];
|
|
@@ -131,7 +134,7 @@ export declare class Files extends EventEmitter {
|
|
|
131
134
|
private _processorStatus;
|
|
132
135
|
close(): Promise<void>;
|
|
133
136
|
}
|
|
134
|
-
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
|
|
137
|
+
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
|
|
135
138
|
export type { IndexingDevice, IndexingEmbedder, RagExtractors, RagSource, VectorIndex } from './rag/index.ts';
|
|
136
139
|
export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.ts';
|
|
137
140
|
export { CoreBlobServer, PENDING_BLOB_REFERENCE, isPendingBlobReference, parseBlobReferenceCoreKey } from './store/index.ts';
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
2
|
import EventEmitter from 'bare-events';
|
|
3
|
+
import { Readable } from 'streamx';
|
|
3
4
|
import { attachmentFiles } from './attachments.js';
|
|
4
5
|
import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, reclaimPurgedFiles, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId, knowledgeCapBytes, usedKnowledgeBytes } from './records/index.js';
|
|
5
6
|
import { knowledgeFiles } from './knowledge.js';
|
|
6
7
|
import { randomId } from '../id.js';
|
|
7
8
|
import { findChunksOfType, getChunk, listChunks } from '../chunk.js';
|
|
8
|
-
import {
|
|
9
|
+
import { isClaimLive, listIndexingByChat, listIndexingByKnowledge, FileState, FileProcessing } from './processing/index.js';
|
|
9
10
|
import { DeadlineScheduler } from '../../deadline.js';
|
|
10
11
|
import { liveDeviceForWriter } from '../contract/activity.js';
|
|
11
12
|
import { runDefinition } from '../shared-agents.js';
|
|
13
|
+
import { toolsetEnabled } from '@qvac/harness/lib/tools/toolsets.ts';
|
|
12
14
|
import { BlobStore, parseBlobReference } from './store/index.js';
|
|
13
15
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
14
16
|
import { RagProcessor } from './rag/index.js';
|
|
@@ -17,7 +19,7 @@ import { parseFailure } from './rag/failure.js';
|
|
|
17
19
|
import { CoreError } from '../../errors.js';
|
|
18
20
|
import { errorMessage } from '../../guards.js';
|
|
19
21
|
import { ACTIVE_KNOWLEDGE_BASE, ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS, INDEXING_STATUS_BY_CHAT, INDEXING_STATUS_BY_KNOWLEDGE, KNOWLEDGE, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE } from '../collections.js';
|
|
20
|
-
export { fullTextBudgetChars, fullTextHits, joinedIndex, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
|
|
22
|
+
export { fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
|
|
21
23
|
export { isKnowledgeRoutedAttachment } from './records/index.js';
|
|
22
24
|
export { assertRemovableAt } from './records/index.js';
|
|
23
25
|
export { assertArtifactOutput } from './records/index.js';
|
|
@@ -395,18 +397,81 @@ export class Files extends EventEmitter {
|
|
|
395
397
|
hydratedArtifact(id) {
|
|
396
398
|
return this.state.get(id)?.loaded?.artifactId ?? null;
|
|
397
399
|
}
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
400
|
+
ready(chatId, ids) {
|
|
401
|
+
const wanted = new Set(ids);
|
|
402
|
+
this._processing?.prioritize(ids);
|
|
403
|
+
return this._until(chatId, (all) => {
|
|
404
|
+
const files = all.filter((file) => wanted.has(file.fileId));
|
|
405
|
+
return files.every(settledFile) ? files : null;
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
async indexingHere(chatId) {
|
|
409
|
+
await this._until(chatId, (files) => files.some((file) => this.indexesHere(file)) ? null : files);
|
|
410
|
+
}
|
|
411
|
+
indexesHere(file) {
|
|
412
|
+
const rag = file.processors.find((entry) => entry.processorId === 'rag');
|
|
413
|
+
if (!this._processing?.active || settledFile(file) || rag?.state === 'not-applicable') {
|
|
414
|
+
return false;
|
|
415
|
+
}
|
|
416
|
+
if (this._processing.awaitsBytes(file.fileId)) {
|
|
417
|
+
return false;
|
|
418
|
+
}
|
|
419
|
+
if (rag?.claimedBy) {
|
|
420
|
+
return b4a.equals(rag.claimedBy, this._mesh.localWriterKey);
|
|
421
|
+
}
|
|
422
|
+
return this._processing.pending(file.fileId);
|
|
423
|
+
}
|
|
424
|
+
async _until(chatId, settle) {
|
|
425
|
+
let all = [];
|
|
426
|
+
for await (const _ of this._changes()) {
|
|
427
|
+
all = (await this.status(chatId)).files;
|
|
428
|
+
const files = settle(all);
|
|
429
|
+
if (files || this.closed) {
|
|
430
|
+
return { files: files ?? all };
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
return { files: all };
|
|
434
|
+
}
|
|
435
|
+
_changes() {
|
|
436
|
+
let due = true;
|
|
437
|
+
let pending = null;
|
|
438
|
+
const deliver = () => {
|
|
439
|
+
const cb = pending;
|
|
440
|
+
if (!due || !cb) {
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
due = false;
|
|
444
|
+
pending = null;
|
|
445
|
+
changes.push(true);
|
|
446
|
+
cb(null);
|
|
447
|
+
};
|
|
448
|
+
const onUpdate = () => {
|
|
449
|
+
due = true;
|
|
450
|
+
deliver();
|
|
451
|
+
};
|
|
452
|
+
const onClose = () => {
|
|
453
|
+
changes.push(true);
|
|
454
|
+
changes.push(null);
|
|
455
|
+
pending?.(null);
|
|
456
|
+
};
|
|
457
|
+
const changes = new Readable({
|
|
458
|
+
read(cb) {
|
|
459
|
+
pending = cb;
|
|
460
|
+
deliver();
|
|
461
|
+
},
|
|
462
|
+
predestroy() {
|
|
463
|
+
pending?.(null);
|
|
464
|
+
pending = null;
|
|
465
|
+
},
|
|
466
|
+
destroy: (cb) => {
|
|
467
|
+
this.unwatch(onUpdate);
|
|
468
|
+
this.off('close', onClose);
|
|
469
|
+
cb(null);
|
|
408
470
|
}
|
|
409
471
|
});
|
|
472
|
+
this.watch(onUpdate);
|
|
473
|
+
this.once('close', onClose);
|
|
474
|
+
return changes;
|
|
410
475
|
}
|
|
411
476
|
reconnect() {
|
|
412
477
|
for (const { work } of this._processors.values()) {
|
|
@@ -529,9 +594,11 @@ export class Files extends EventEmitter {
|
|
|
529
594
|
return { files: [] };
|
|
530
595
|
}
|
|
531
596
|
const agent = await runDefinition(this._mesh, await snap.get(AGENTS, { id: chat.agentId }));
|
|
597
|
+
const knowledgeOn = toolsetEnabled('knowledge', agent?.disabledToolsets);
|
|
532
598
|
const knowledge = await snap.find(ACTIVE_KNOWLEDGE_BY_CHAT, range).toArray();
|
|
533
599
|
const chunks = await findChunksOfType(snap, { chatId, type: 'attachment' });
|
|
534
600
|
const statuses = await snap.find(INDEXING_STATUS_BY_CHAT, range).toArray();
|
|
601
|
+
const statusesByKnowledge = groupBy(statuses, (row) => row.knowledgeId);
|
|
535
602
|
const sources = new Map(knowledge.map((row) => [row.id, row]));
|
|
536
603
|
const ragIds = new Set(sources.keys());
|
|
537
604
|
for (const chunk of chunks) {
|
|
@@ -565,12 +632,12 @@ export class Files extends EventEmitter {
|
|
|
565
632
|
lte: { knowledgeId: id }
|
|
566
633
|
})
|
|
567
634
|
.toArray();
|
|
568
|
-
const applicable = ragIds.has(id);
|
|
635
|
+
const applicable = knowledgeOn && ragIds.has(id);
|
|
569
636
|
const local = this.state.get(id);
|
|
570
637
|
const output = rows.find((row) => !isProcessorOutput(row.embedderVersion) &&
|
|
571
638
|
(!this._model || row.embedderVersion.startsWith(`${this._model}|`)));
|
|
572
639
|
const published = !!output;
|
|
573
|
-
const attempts =
|
|
640
|
+
const attempts = (statusesByKnowledge.get(id) ?? []).filter((row) => !isProcessorArtifactId(row.id));
|
|
574
641
|
const claim = attempts.find((row) => isClaimLive(row, Date.now()));
|
|
575
642
|
const failures = attempts.some((row) => row.status === 'ready')
|
|
576
643
|
? []
|
|
@@ -585,6 +652,7 @@ export class Files extends EventEmitter {
|
|
|
585
652
|
processor.outputId = local?.loaded?.artifactId ?? output?.id;
|
|
586
653
|
if (!applicable) {
|
|
587
654
|
processor.state = 'not-applicable';
|
|
655
|
+
processor.reason = ragIds.has(id) ? 'knowledge off' : undefined;
|
|
588
656
|
}
|
|
589
657
|
else if (local?.loaded) {
|
|
590
658
|
processor.state = 'ready';
|
|
@@ -625,7 +693,12 @@ export class Files extends EventEmitter {
|
|
|
625
693
|
: await this._store.available(source.blobId);
|
|
626
694
|
const processors = [
|
|
627
695
|
processor,
|
|
628
|
-
...(await this._extraProcessorStatuses(id, {
|
|
696
|
+
...(await this._extraProcessorStatuses(id, {
|
|
697
|
+
chunks,
|
|
698
|
+
knowledge,
|
|
699
|
+
rows,
|
|
700
|
+
statusesByKnowledge
|
|
701
|
+
}))
|
|
629
702
|
];
|
|
630
703
|
const uploadProgress = this._uploads.get(id)?.status.uploadProgress;
|
|
631
704
|
result.set(id, {
|
|
@@ -655,7 +728,7 @@ export class Files extends EventEmitter {
|
|
|
655
728
|
processor.agentId = agent.id;
|
|
656
729
|
}
|
|
657
730
|
}
|
|
658
|
-
async _extraProcessorStatuses(id, { chunks, knowledge, rows,
|
|
731
|
+
async _extraProcessorStatuses(id, { chunks, knowledge, rows, statusesByKnowledge }) {
|
|
659
732
|
const processors = [];
|
|
660
733
|
for (const { processor: descriptor } of this._processors.values()) {
|
|
661
734
|
if (descriptor.id === 'rag')
|
|
@@ -667,7 +740,10 @@ export class Files extends EventEmitter {
|
|
|
667
740
|
const input = record ? { ...record, kind: descriptor.input } : null;
|
|
668
741
|
processors.push(!input || !descriptor.applies(input)
|
|
669
742
|
? { processorId: descriptor.id, state: 'not-applicable', published: false }
|
|
670
|
-
: await this._processorStatus(descriptor, input, {
|
|
743
|
+
: await this._processorStatus(descriptor, input, {
|
|
744
|
+
rows,
|
|
745
|
+
attempts: statusesByKnowledge.get(id) ?? []
|
|
746
|
+
}));
|
|
671
747
|
}
|
|
672
748
|
return processors;
|
|
673
749
|
}
|
|
@@ -719,6 +795,9 @@ export class Files extends EventEmitter {
|
|
|
719
795
|
await this._store.close();
|
|
720
796
|
}
|
|
721
797
|
}
|
|
798
|
+
function settledFile(file) {
|
|
799
|
+
return file.readyForChat || file.state === 'failed';
|
|
800
|
+
}
|
|
722
801
|
function yieldsToVerdict(work) {
|
|
723
802
|
return work.state === 'pending' || (work.state === 'failed' && work.retryAt === undefined);
|
|
724
803
|
}
|
|
@@ -731,6 +810,18 @@ function fileState(processor, opts) {
|
|
|
731
810
|
}
|
|
732
811
|
return { state: 'pending', reason: 'source unavailable on this device' };
|
|
733
812
|
}
|
|
734
|
-
|
|
813
|
+
function groupBy(items, key) {
|
|
814
|
+
const groups = new Map();
|
|
815
|
+
for (const item of items) {
|
|
816
|
+
const id = key(item);
|
|
817
|
+
let group = groups.get(id);
|
|
818
|
+
if (!group) {
|
|
819
|
+
groups.set(id, (group = []));
|
|
820
|
+
}
|
|
821
|
+
group.push(item);
|
|
822
|
+
}
|
|
823
|
+
return groups;
|
|
824
|
+
}
|
|
825
|
+
export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, fullTextAttachmentId, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.js';
|
|
735
826
|
export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.js';
|
|
736
827
|
export { CoreBlobServer, PENDING_BLOB_REFERENCE, isPendingBlobReference, parseBlobReferenceCoreKey } from './store/index.js';
|
|
@@ -67,6 +67,7 @@ export declare class FileProcessing {
|
|
|
67
67
|
private _enqueue;
|
|
68
68
|
get active(): boolean;
|
|
69
69
|
pending(id: string): boolean;
|
|
70
|
+
awaitsBytes(id: string): boolean;
|
|
70
71
|
prioritize(ids: string[]): void;
|
|
71
72
|
private _shutdown;
|
|
72
73
|
private _stop;
|
|
@@ -85,4 +86,3 @@ export type { FileDocument } from './document.ts';
|
|
|
85
86
|
export { isClaimLive, listIndexingByChat, listIndexingByKnowledge } from './protocol.ts';
|
|
86
87
|
export { FileState } from './state.ts';
|
|
87
88
|
export type { IndexingProgressEntry, LocalWork } from './state.ts';
|
|
88
|
-
export { settleFiles } from './settle.ts';
|
|
@@ -130,12 +130,19 @@ export class FileProcessing {
|
|
|
130
130
|
this._source.pending((key) => key === id) ||
|
|
131
131
|
this._output.pending((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id)));
|
|
132
132
|
}
|
|
133
|
+
awaitsBytes(id) {
|
|
134
|
+
return (this._source.awaitsBytes((key) => key === id) ||
|
|
135
|
+
this._output.awaitsBytes((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id));
|
|
136
|
+
}
|
|
133
137
|
prioritize(ids) {
|
|
134
138
|
for (const lane of this._lanes) {
|
|
135
139
|
const queued = new Map(lane.queue);
|
|
136
140
|
lane.queue.clear();
|
|
137
141
|
for (const id of ids) {
|
|
138
|
-
|
|
142
|
+
const wakes = queued.get(id);
|
|
143
|
+
if (wakes) {
|
|
144
|
+
lane.queue.set(id, wakes);
|
|
145
|
+
}
|
|
139
146
|
}
|
|
140
147
|
for (const [id, wakes] of queued) {
|
|
141
148
|
if (!lane.queue.has(id)) {
|
|
@@ -349,4 +356,3 @@ export class FileProcessing {
|
|
|
349
356
|
}
|
|
350
357
|
export { isClaimLive, listIndexingByChat, listIndexingByKnowledge } from './protocol.js';
|
|
351
358
|
export { FileState } from './state.js';
|
|
352
|
-
export { settleFiles } from './settle.js';
|
|
@@ -35,5 +35,6 @@ export declare class IndexingSource {
|
|
|
35
35
|
read(id: string, blobRef: string, signal?: AbortSignal): Promise<SourceRead>;
|
|
36
36
|
cancel(matches?: (id: string) => boolean): Promise<void>;
|
|
37
37
|
pending(matches: (id: string) => boolean): boolean;
|
|
38
|
+
awaitsBytes(matches: (id: string) => boolean): boolean;
|
|
38
39
|
private _fetch;
|
|
39
40
|
}
|
|
@@ -45,6 +45,7 @@ export declare class FileRetries {
|
|
|
45
45
|
blockedUntil(id: string, now: number): number | null;
|
|
46
46
|
miss(id: string, now: number, retryMs: number): number;
|
|
47
47
|
markMissing(id: string): void;
|
|
48
|
+
missing(matches: (id: string) => boolean): boolean;
|
|
48
49
|
markFound(id: string): void;
|
|
49
50
|
clear(id?: string): void;
|
|
50
51
|
reconnect(onRetry?: (id: string, deadline: number) => void): number | null;
|
|
@@ -15,6 +15,13 @@ export class FileRetries {
|
|
|
15
15
|
markMissing(id) {
|
|
16
16
|
this._missing.add(id);
|
|
17
17
|
}
|
|
18
|
+
missing(matches) {
|
|
19
|
+
for (const id of this._missing) {
|
|
20
|
+
if (matches(id))
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
18
25
|
markFound(id) {
|
|
19
26
|
this._missing.delete(id);
|
|
20
27
|
}
|
|
@@ -20,6 +20,7 @@ export interface ArtifactBuildInput {
|
|
|
20
20
|
bytes: Buffer;
|
|
21
21
|
signal: AbortSignal;
|
|
22
22
|
onProgress?: BuildProgress;
|
|
23
|
+
onExtracting?: (pages: number, totalPages: number) => Promise<void>;
|
|
23
24
|
onExtracted?: () => void | Promise<void>;
|
|
24
25
|
extractors?: RagExtractors;
|
|
25
26
|
}
|