@qvac/core 1.0.0-rc.7 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +241 -3
- package/README.md +1 -0
- package/dist-lib/lib/api/attachments.d.ts +1 -1
- package/dist-lib/lib/api/attachments.js +11 -0
- package/dist-lib/lib/api/chunks.d.ts +4 -2
- package/dist-lib/lib/api/chunks.js +6 -3
- package/dist-lib/lib/api/device-routes.d.ts +6 -1
- package/dist-lib/lib/api/device-routes.js +20 -1
- package/dist-lib/lib/api/devices.js +6 -18
- package/dist-lib/lib/api/execution.d.ts +6 -2
- package/dist-lib/lib/api/execution.js +29 -3
- package/dist-lib/lib/api/host-run-log.d.ts +7 -1
- package/dist-lib/lib/api/host-run-log.js +3 -1
- package/dist-lib/lib/api/index.d.ts +3 -1
- package/dist-lib/lib/api/index.js +17 -6
- package/dist-lib/lib/api/members.js +13 -29
- package/dist-lib/lib/api/mesh.d.ts +2 -1
- package/dist-lib/lib/api/projects.d.ts +1 -1
- package/dist-lib/lib/api/projects.js +9 -5
- package/dist-lib/lib/api/rag-label.d.ts +4 -0
- package/dist-lib/lib/api/rag-label.js +21 -0
- package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
- package/dist-lib/lib/api/rag-lexical.js +1058 -0
- package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
- package/dist-lib/lib/api/rag-manifest.js +46 -0
- package/dist-lib/lib/api/rag-outline.d.ts +36 -0
- package/dist-lib/lib/api/rag-outline.js +426 -0
- package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
- package/dist-lib/lib/api/rag-plugin.js +70 -0
- package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
- package/dist-lib/lib/api/rag-synthesis.js +102 -0
- package/dist-lib/lib/api/rag-targets.d.ts +23 -0
- package/dist-lib/lib/api/rag-targets.js +215 -0
- package/dist-lib/lib/api/rag-window.d.ts +5 -0
- package/dist-lib/lib/api/rag-window.js +46 -0
- package/dist-lib/lib/api/rag.d.ts +3 -2
- package/dist-lib/lib/api/rag.js +421 -184
- package/dist-lib/lib/api/served.js +8 -14
- package/dist-lib/lib/api/workspace.d.ts +8 -1
- package/dist-lib/lib/api/workspace.js +27 -13
- package/dist-lib/lib/assistant/harness.d.ts +9 -1
- package/dist-lib/lib/assistant/history.d.ts +11 -2
- package/dist-lib/lib/assistant/history.js +74 -4
- package/dist-lib/lib/assistant/media-round.d.ts +61 -0
- package/dist-lib/lib/assistant/media-round.js +111 -0
- package/dist-lib/lib/assistant/message.d.ts +7 -0
- package/dist-lib/lib/assistant/model-round.d.ts +19 -1
- package/dist-lib/lib/assistant/model-round.js +163 -28
- package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
- package/dist-lib/lib/assistant/pdf-pages.js +67 -3
- package/dist-lib/lib/assistant/round-host.d.ts +22 -0
- package/dist-lib/lib/assistant/round-host.js +142 -8
- package/dist-lib/lib/assistant/upload.d.ts +12 -0
- package/dist-lib/lib/assistant/upload.js +36 -0
- package/dist-lib/lib/assistant-client.d.ts +1 -0
- package/dist-lib/lib/assistant-client.js +40 -19
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +48 -1
- package/dist-lib/lib/assistant.d.ts +18 -1
- package/dist-lib/lib/assistant.js +484 -95
- package/dist-lib/lib/client.d.ts +1 -0
- package/dist-lib/lib/client.js +4 -1
- package/dist-lib/lib/compaction.js +2 -2
- package/dist-lib/lib/core/agent.js +12 -45
- package/dist-lib/lib/core/approval-rules.js +1 -8
- package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
- package/dist-lib/lib/core/archived-mesh.js +12 -8
- package/dist-lib/lib/core/chat-copy.d.ts +18 -3
- package/dist-lib/lib/core/chat-copy.js +74 -25
- package/dist-lib/lib/core/chat-fork.js +15 -6
- package/dist-lib/lib/core/chat-group.js +16 -24
- package/dist-lib/lib/core/chat-move.d.ts +9 -0
- package/dist-lib/lib/core/chat-move.js +94 -3
- package/dist-lib/lib/core/chat-order.js +12 -18
- package/dist-lib/lib/core/chat.js +38 -31
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +18 -25
- package/dist-lib/lib/core/collections.d.ts +2 -0
- package/dist-lib/lib/core/collections.js +2 -0
- package/dist-lib/lib/core/contract/activity.d.ts +4 -0
- package/dist-lib/lib/core/contract/activity.js +34 -4
- package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
- package/dist-lib/lib/core/contract/chat-group.js +21 -1
- package/dist-lib/lib/core/contract/chat.d.ts +9 -0
- package/dist-lib/lib/core/contract/chat.js +127 -5
- package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
- package/dist-lib/lib/core/contract/chunks.js +37 -6
- package/dist-lib/lib/core/contract/features.d.ts +21 -0
- package/dist-lib/lib/core/contract/features.js +22 -0
- package/dist-lib/lib/core/contract/index.js +2 -0
- package/dist-lib/lib/core/contract/indexing.js +4 -2
- package/dist-lib/lib/core/contract/knowledge.js +5 -4
- package/dist-lib/lib/core/contract/members.js +16 -10
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
- package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
- package/dist-lib/lib/core/contract/projects.d.ts +5 -0
- package/dist-lib/lib/core/contract/projects.js +13 -5
- package/dist-lib/lib/core/contract/roles.js +32 -4
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
- package/dist-lib/lib/core/contract/run-requests.js +35 -2
- package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
- package/dist-lib/lib/core/contract/tool-configs.js +3 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
- package/dist-lib/lib/core/contract/workspace.js +40 -7
- package/dist-lib/lib/core/contract/writers.js +2 -2
- package/dist-lib/lib/core/departures.d.ts +3 -1
- package/dist-lib/lib/core/departures.js +3 -0
- package/dist-lib/lib/core/devices.js +1 -8
- package/dist-lib/lib/core/files/attachments.d.ts +2 -0
- package/dist-lib/lib/core/files/attachments.js +19 -1
- package/dist-lib/lib/core/files/index.d.ts +20 -4
- package/dist-lib/lib/core/files/index.js +132 -21
- package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
- package/dist-lib/lib/core/files/processing/document.js +7 -0
- package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
- package/dist-lib/lib/core/files/processing/index.js +144 -12
- package/dist-lib/lib/core/files/processing/protocol.js +13 -34
- package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
- package/dist-lib/lib/core/files/rag/build.js +123 -19
- package/dist-lib/lib/core/files/rag/failure.js +2 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
- package/dist-lib/lib/core/files/rag/index.js +13 -1
- package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
- package/dist-lib/lib/core/files/rag/load.js +27 -6
- package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
- package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
- package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/produce.js +104 -16
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
- package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
- package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
- package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
- package/dist-lib/lib/core/files/records/knowledge.js +7 -40
- package/dist-lib/lib/core/files/records/reclaim.js +5 -10
- package/dist-lib/lib/core/files/upload.js +68 -41
- package/dist-lib/lib/core/fs-drives.js +14 -26
- package/dist-lib/lib/core/invite.d.ts +1 -0
- package/dist-lib/lib/core/invite.js +4 -2
- package/dist-lib/lib/core/local/files.d.ts +1 -1
- package/dist-lib/lib/core/local/files.js +5 -5
- package/dist-lib/lib/core/local/index.d.ts +13 -6
- package/dist-lib/lib/core/local/index.js +103 -69
- package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
- package/dist-lib/lib/core/local/run-logs.js +5 -5
- package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
- package/dist-lib/lib/core/mesh-processor.js +87 -0
- package/dist-lib/lib/core/mesh.d.ts +48 -5
- package/dist-lib/lib/core/mesh.js +318 -129
- package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
- package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
- package/dist-lib/lib/core/migrations/writer-links.js +32 -0
- package/dist-lib/lib/core/project-keys.d.ts +6 -0
- package/dist-lib/lib/core/project-keys.js +11 -0
- package/dist-lib/lib/core/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/public-mesh.js +10 -8
- package/dist-lib/lib/core/run-requests.d.ts +3 -0
- package/dist-lib/lib/core/run-requests.js +21 -1
- package/dist-lib/lib/core/skill-availability.d.ts +33 -0
- package/dist-lib/lib/core/skill-availability.js +67 -0
- package/dist-lib/lib/core/sync/background.d.ts +0 -1
- package/dist-lib/lib/core/sync/background.js +2 -1
- package/dist-lib/lib/core/sync/changed.d.ts +4 -2
- package/dist-lib/lib/core/sync/changed.js +1 -1
- package/dist-lib/lib/core/sync/index.js +52 -10
- package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
- package/dist-lib/lib/core/sync/view-updates.js +82 -91
- package/dist-lib/lib/core/sync-status.js +97 -41
- package/dist-lib/lib/core/tool-activity.js +6 -12
- package/dist-lib/lib/core/tool-configs.js +17 -29
- package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
- package/dist-lib/lib/core/upgrade-summary.js +67 -0
- package/dist-lib/lib/core/view-read.d.ts +4 -0
- package/dist-lib/lib/core/view-read.js +23 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
- package/dist-lib/lib/core/workspace-upgrade.js +27 -16
- package/dist-lib/lib/core.d.ts +45 -42
- package/dist-lib/lib/core.js +675 -640
- package/dist-lib/lib/errors.d.ts +24 -0
- package/dist-lib/lib/errors.js +24 -0
- package/dist-lib/lib/llm-model-names.d.ts +1 -1
- package/dist-lib/lib/llm-model-names.js +1 -1
- package/dist-lib/lib/mesh-scoped-engine.js +4 -0
- package/dist-lib/lib/model-context.js +1 -2
- package/dist-lib/lib/model-download.d.ts +6 -1
- package/dist-lib/lib/model-download.js +16 -3
- package/dist-lib/lib/oauth/resolve.js +29 -8
- package/dist-lib/lib/oauth/token-exchange-core.js +11 -0
- package/dist-lib/lib/probe-rounds.d.ts +7 -2
- package/dist-lib/lib/probe-rounds.js +6 -1
- package/dist-lib/lib/run-transcript.d.ts +4 -1
- package/dist-lib/lib/run-transcript.js +15 -6
- package/dist-lib/schema/engine.js +83 -6
- package/dist-lib/schema/indexes.d.ts +3 -0
- package/dist-lib/schema/indexes.js +5 -0
- package/dist-lib/schema/local.js +11 -2
- package/dist-lib/schema/qvac.js +49 -5
- package/dist-lib/spec/bind.js +34 -0
- package/dist-lib/spec/calls.js +5 -0
- package/dist-lib/spec/capabilities.d.ts +17 -0
- package/dist-lib/spec/hrpc/hrpc.json +65 -0
- package/dist-lib/spec/hrpc/index.js +68 -5
- package/dist-lib/spec/hrpc/messages.js +260 -23
- package/dist-lib/spec/hyperschema/index.js +260 -23
- package/dist-lib/spec/hyperschema/schema.json +181 -0
- package/dist-lib/spec/hyperschema/types.d.ts +48 -0
- package/dist-lib/spec/local/hyperdb/messages.js +270 -224
- package/dist-lib/spec/local/hyperschema/index.js +73 -38
- package/dist-lib/spec/local/hyperschema/schema.json +31 -0
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
- package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
- package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
- package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
- package/package.json +82 -71
- package/tui/README.md +52 -0
- package/tui/app.mjs +69 -12
- package/tui/boot.mjs +27 -6
- package/tui/cli.mjs +93 -0
- package/tui/index.mjs +2 -87
- package/tui/lib/attachment-files.mjs +82 -0
- package/tui/lib/reopen-meshes.mjs +15 -0
- package/tui/lib/updater.mjs +86 -0
- package/tui/main.mjs +29 -57
- package/worker/assistant-sidecar.mjs +10 -3
- package/worker/stow-entry.mjs +3 -52
- package/dist-lib/lib/core/chat-project.d.ts +0 -2
- package/dist-lib/lib/core/chat-project.js +0 -15
package/dist-lib/lib/api/rag.js
CHANGED
|
@@ -2,10 +2,16 @@ import { toolsetEnabled } from '@qvac/harness/lib/tools/toolsets.ts';
|
|
|
2
2
|
import { getAgent } from '../core/agent.js';
|
|
3
3
|
import { runDefinition } from '../core/shared-agents.js';
|
|
4
4
|
import { getChat } from '../core/chat.js';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
5
|
+
import { isKnowledgeRoutedAttachment, fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, referencedDocuments, PluginVectorIndex } from '../core/files/index.js';
|
|
6
|
+
import { requireMesh } from './current-mesh.js';
|
|
7
|
+
import { searchPlugin } from './rag-plugin.js';
|
|
8
|
+
import { rankLexical } from './rag-lexical.js';
|
|
9
|
+
import { manifestBlock, readable, UNREAD } from './rag-manifest.js';
|
|
10
|
+
import { classifyRequest, outlineBlock } from './rag-synthesis.js';
|
|
11
|
+
import { DEFAULT_CHUNK_BUDGET_TOKENS } from 'rag-extract';
|
|
8
12
|
import { errorMessage } from '../guards.js';
|
|
13
|
+
import { parseFailure } from '../core/files/rag/failure.js';
|
|
14
|
+
import { listIndexingByKnowledge } from '../core/files/processing/index.js';
|
|
9
15
|
import b4a from 'b4a';
|
|
10
16
|
export const MAX_RAG_SEARCH_LIMIT = 20;
|
|
11
17
|
const DEFAULT_RAG_SEARCH_LIMIT = 5;
|
|
@@ -14,218 +20,341 @@ function clampLimit(limit) {
|
|
|
14
20
|
return DEFAULT_RAG_SEARCH_LIMIT;
|
|
15
21
|
return Math.min(Math.max(1, Math.floor(limit)), MAX_RAG_SEARCH_LIMIT);
|
|
16
22
|
}
|
|
17
|
-
export function createRagApi(pool, { index, embedder, effectiveCtxSize = null }) {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
};
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
23
|
+
export function createRagApi(pool, { index, embedder, effectiveCtxSize = null, extractors }) {
|
|
24
|
+
const ragSearch = async ({ chatId, query, mode, limit, attachmentIds, meshId, immediate, previousQuery, excerptsOnly }, { vectors = false } = {}) => {
|
|
25
|
+
let pendingFiles = [];
|
|
26
|
+
let projectFilesNotReady = 0;
|
|
27
|
+
let chatFilesIndexingHere = 0;
|
|
28
|
+
let manifest;
|
|
29
|
+
const respond = (overrides) => {
|
|
30
|
+
const result = response({
|
|
31
|
+
pendingFiles,
|
|
32
|
+
projectFilesNotReady,
|
|
33
|
+
chatFilesIndexingHere,
|
|
34
|
+
...overrides,
|
|
35
|
+
outline: manifest ?? overrides.outline
|
|
36
|
+
});
|
|
37
|
+
return { ...result, chatFilesNotReady: result.pendingFiles?.length ?? 0 };
|
|
38
|
+
};
|
|
39
|
+
const mesh = await requireMesh(pool, meshId);
|
|
40
|
+
const chat = await getChat(mesh, chatId);
|
|
41
|
+
const toolMode = mode === 'tool';
|
|
42
|
+
if (!chat) {
|
|
43
|
+
return respond({ enabled: false });
|
|
44
|
+
}
|
|
45
|
+
const request = classifyRequest(query);
|
|
46
|
+
const agent = await runDefinition(mesh, await getAgent(mesh, chat.agentId));
|
|
47
|
+
const scopeIds = new Set(toolMode ? [] : (attachmentIds ?? []));
|
|
48
|
+
const off = new Set(chat.knowledgeSourcesOff ?? []);
|
|
49
|
+
const kept = (id) => !off.has('chat') || scopeIds.has(id);
|
|
50
|
+
const chatKnowledge = async () => (await mesh.files.listKnowledge(chatId)).filter((row) => kept(row.id));
|
|
51
|
+
const chatFiles = async () => (await mesh.files.status(chatId)).files.filter((file) => kept(file.fileId));
|
|
52
|
+
const indexingHere = (files) => files.filter((file) => scopeIds.has(file.fileId) &&
|
|
53
|
+
(vectors ? mesh.files.indexesHere(file) : mesh.files.extractingHere(file))).length;
|
|
54
|
+
if (!immediate && scopeIds.size > 0) {
|
|
55
|
+
const ids = [...scopeIds];
|
|
56
|
+
await (vectors ? mesh.files.indexingHere(chatId, ids) : mesh.files.textReadyHere(chatId, ids));
|
|
57
|
+
}
|
|
58
|
+
else if (immediate && vectors && scopeIds.size > 0) {
|
|
59
|
+
await mesh.files.requestVectors([...scopeIds]);
|
|
60
|
+
}
|
|
61
|
+
const chatRows = await chatKnowledge();
|
|
62
|
+
if (toolMode && chatRows.length > 0) {
|
|
63
|
+
const ids = chatRows.map((row) => row.id);
|
|
64
|
+
if (immediate) {
|
|
65
|
+
await mesh.files.requestVectors(ids);
|
|
37
66
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const off = new Set(chat.knowledgeSourcesOff ?? []);
|
|
41
|
-
const kept = (id) => !off.has('chat') || scopeIds.has(id);
|
|
42
|
-
const chatKnowledge = async () => (await mesh.files.listKnowledge(chatId)).filter((row) => kept(row.id));
|
|
43
|
-
const chatFiles = async () => (await mesh.files.status(chatId)).files.filter((file) => kept(file.fileId));
|
|
44
|
-
const indexingHere = (files) => files.filter((file) => scopeIds.has(file.fileId) && mesh.files.indexesHere(file)).length;
|
|
45
|
-
if (!immediate && scopeIds.size > 0) {
|
|
46
|
-
await mesh.files.indexingHere(chatId, [...scopeIds]);
|
|
67
|
+
else {
|
|
68
|
+
await mesh.files.indexingHere(chatId, ids);
|
|
47
69
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
70
|
+
}
|
|
71
|
+
const initialFiles = await chatFiles();
|
|
72
|
+
pendingFiles = pendingFileNames(initialFiles);
|
|
73
|
+
chatFilesIndexingHere = indexingHere(initialFiles);
|
|
74
|
+
const scopeRows = chatRows.filter((row) => scopeIds.has(row.id));
|
|
75
|
+
const project = off.has('project') || mesh.role !== 'project' ? null : mesh;
|
|
76
|
+
const kbEntries = project ? ((await project.files.knowledgeBase()).entries ?? []) : [];
|
|
77
|
+
const kbRows = kbEntries.filter((entry) => entry.loaded).map((entry) => entry.knowledge);
|
|
78
|
+
projectFilesNotReady = kbEntries.filter((entry) => !entry.loaded && entry.status !== 'failed').length;
|
|
79
|
+
let window;
|
|
80
|
+
const ctxSize = () => (window ??= effectiveCtxSize ? effectiveCtxSize(agent) : Promise.resolve(0));
|
|
81
|
+
const attached = scopeIds.size > 0 && scopeRows.length === scopeIds.size ? scopeRows : [];
|
|
82
|
+
const named = new Set(referencedDocuments(query, chatRows.map((row) => ({ documentId: row.id, title: row.fileName }))).map((document) => document.documentId));
|
|
83
|
+
const focused = named.size > 0 ? chatRows.filter((row) => named.has(row.id)) : chatRows;
|
|
84
|
+
let subject = [];
|
|
85
|
+
if (scopeIds.size > 0) {
|
|
86
|
+
subject = attached;
|
|
87
|
+
}
|
|
88
|
+
else if (kbRows.length === 0) {
|
|
89
|
+
subject = focused;
|
|
90
|
+
}
|
|
91
|
+
if (!toolMode && request.wholeDocument && subject.length > 1) {
|
|
92
|
+
manifest = await listFiles(mesh, { files: initialFiles, rows: subject });
|
|
93
|
+
}
|
|
94
|
+
const packet = async (rows, pick) => {
|
|
95
|
+
const unread = await unreadFiles(mesh, {
|
|
96
|
+
files: initialFiles,
|
|
97
|
+
rows,
|
|
98
|
+
ids: pick.pendingIds
|
|
99
|
+
});
|
|
100
|
+
const read = rows.filter((row) => !pick.pendingIds.includes(row.id));
|
|
101
|
+
const gaps = unread.states.map(({ name, status }) => `${name}: ${status}`);
|
|
102
|
+
return respond({
|
|
103
|
+
pendingFiles: [
|
|
104
|
+
...new Set([
|
|
105
|
+
...pendingFileNames(initialFiles, new Set(read.map((row) => row.id))),
|
|
106
|
+
...unread.pending
|
|
107
|
+
])
|
|
108
|
+
],
|
|
109
|
+
searchedCount: pick.hits.length,
|
|
110
|
+
indexEmpty: false,
|
|
111
|
+
hasKnowledge: true,
|
|
112
|
+
availableFiles: [...new Set(read.map((row) => row.fileName))],
|
|
113
|
+
uncoveredItems: listed([...pick.uncovered, ...gaps]),
|
|
114
|
+
failedFiles: unread.failed,
|
|
115
|
+
outline: outlineBlock(pick),
|
|
116
|
+
hits: pick.hits.map(toWireHit)
|
|
117
|
+
});
|
|
118
|
+
};
|
|
119
|
+
const fromText = async (rows, { previousQuery: before, maxChars, whole = false }) => {
|
|
120
|
+
const allowed = agentAllowsRagSearch(agent);
|
|
121
|
+
const picks = !vectors && allowed;
|
|
122
|
+
const size = await ctxSize();
|
|
123
|
+
const budgetTokens = retrievalBudgetTokens(size) ?? DEFAULT_CHUNK_BUDGET_TOKENS;
|
|
124
|
+
const maxBudgetTokens = size > 0 ? Math.min(MAX_LIST_CONTEXT_TOKENS, Math.floor(size / 4)) : budgetTokens;
|
|
125
|
+
const wholeChars = Math.min(fullTextBudgetChars(size), WHOLE_CHARS);
|
|
126
|
+
const readWhole = (head) => !whole || excerptsOnly
|
|
54
127
|
? null
|
|
55
|
-
:
|
|
56
|
-
const
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
128
|
+
: fullTextHits(mesh, rows, { budgetChars: wholeChars, head, extractors: extractors?.() });
|
|
129
|
+
const wholeFirst = picks && request.synthesis;
|
|
130
|
+
const early = wholeFirst ? await readWhole(false) : null;
|
|
131
|
+
const texts = picks && !early ? await textsOf(mesh, rows, maxChars) : null;
|
|
132
|
+
const pick = texts &&
|
|
133
|
+
pickFromText(rows, texts, {
|
|
134
|
+
query,
|
|
135
|
+
budgetTokens,
|
|
136
|
+
maxBudgetTokens,
|
|
137
|
+
synthesis: request.synthesis,
|
|
138
|
+
previousQuery: before
|
|
64
139
|
});
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
return respond({
|
|
68
|
-
pendingFiles: pendingFileNames(initialFiles, read),
|
|
69
|
-
chatFilesIndexingHere: 0,
|
|
70
|
-
searchedCount: fullText.length,
|
|
71
|
-
indexEmpty: false,
|
|
72
|
-
hasKnowledge: true,
|
|
73
|
-
availableFiles: [...new Set(scopeRows.map((row) => row.fileName))],
|
|
74
|
-
hits: fullText
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
if (!toolMode && !agentAllowsRagSearch(agent)) {
|
|
79
|
-
return respond({ enabled: false });
|
|
80
|
-
}
|
|
81
|
-
if (chatRows.length === 0 && kbRows.length === 0) {
|
|
82
|
-
return respond({});
|
|
140
|
+
if (pick && pick.uncovered.length === 0) {
|
|
141
|
+
return packet(rows, pick);
|
|
83
142
|
}
|
|
84
|
-
const
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
143
|
+
const fullText = wholeFirst ? early : await readWhole(!allowed);
|
|
144
|
+
if (fullText) {
|
|
145
|
+
manifest = undefined;
|
|
146
|
+
return respond({
|
|
147
|
+
pendingFiles: pendingFileNames(initialFiles, new Set(rows.map((row) => row.id))),
|
|
148
|
+
chatFilesIndexingHere: 0,
|
|
149
|
+
searchedCount: fullText.length,
|
|
150
|
+
indexEmpty: false,
|
|
151
|
+
hasKnowledge: true,
|
|
152
|
+
availableFiles: [...new Set(rows.map((row) => row.fileName))],
|
|
153
|
+
hits: fullText
|
|
154
|
+
});
|
|
90
155
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
: chatIndex;
|
|
98
|
-
const artifactByRow = new Map([
|
|
99
|
-
...chatRows.map((row) => [row, mesh.files.hydratedArtifact(row.id)]),
|
|
100
|
-
...kbRows.map((row) => [row, project?.files.hydratedArtifact(row.id) ?? null])
|
|
101
|
-
]);
|
|
102
|
-
const hydratedDocuments = [];
|
|
103
|
-
for (const [row, artifactId] of artifactByRow) {
|
|
104
|
-
if (artifactId) {
|
|
105
|
-
hydratedDocuments.push({ documentId: artifactId, title: row.fileName });
|
|
106
|
-
}
|
|
156
|
+
return pick ? packet(rows, pick) : null;
|
|
157
|
+
};
|
|
158
|
+
if (attached.length > 0) {
|
|
159
|
+
const answer = await fromText(attached, { whole: true });
|
|
160
|
+
if (answer) {
|
|
161
|
+
return answer;
|
|
107
162
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
163
|
+
}
|
|
164
|
+
if (!toolMode && !agentAllowsRagSearch(agent)) {
|
|
165
|
+
return respond({ enabled: false });
|
|
166
|
+
}
|
|
167
|
+
if (chatRows.length === 0 && kbRows.length === 0) {
|
|
168
|
+
return respond({});
|
|
169
|
+
}
|
|
170
|
+
const followUp = !toolMode && !vectors && scopeIds.size === 0 && kbRows.length === 0;
|
|
171
|
+
if (followUp) {
|
|
172
|
+
const answer = await fromText(focused, {
|
|
173
|
+
maxChars: TEXT_CHARS,
|
|
174
|
+
previousQuery: named.size === 0 ? (previousQuery ?? undefined) : undefined
|
|
175
|
+
});
|
|
176
|
+
if (answer) {
|
|
177
|
+
return answer;
|
|
119
178
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
179
|
+
}
|
|
180
|
+
if (!toolMode && scopeIds.size === 0 && chatRows.length > 0) {
|
|
181
|
+
await mesh.files.requestVectors(chatRows.map((row) => row.id));
|
|
182
|
+
}
|
|
183
|
+
const rows = [...chatRows, ...kbRows];
|
|
184
|
+
const hasKnowledge = rows.length > 0;
|
|
185
|
+
const chatIndex = index(meshId);
|
|
186
|
+
const embed = embedder(meshId);
|
|
187
|
+
if (!chatIndex || !embed) {
|
|
188
|
+
return respond({ hasKnowledge, errorMessage: 'knowledge index unavailable on this device' });
|
|
189
|
+
}
|
|
190
|
+
const projectIndex = project && kbRows.length ? index(b4a.toHex(project.key)) : null;
|
|
191
|
+
const vectorIndex = projectIndex
|
|
192
|
+
? joinedIndex([chatIndex, projectIndex], {
|
|
193
|
+
chatId,
|
|
194
|
+
knowledgeBaseIds: new Set(kbRows.map((row) => row.id))
|
|
195
|
+
})
|
|
196
|
+
: chatIndex;
|
|
197
|
+
const artifactByRow = new Map([
|
|
198
|
+
...chatRows.map((row) => [row, mesh.files.hydratedArtifact(row.id)]),
|
|
199
|
+
...kbRows.map((row) => [row, project?.files.hydratedArtifact(row.id) ?? null])
|
|
200
|
+
]);
|
|
201
|
+
const hydratedDocuments = [];
|
|
202
|
+
for (const [row, artifactId] of artifactByRow) {
|
|
203
|
+
if (artifactId) {
|
|
204
|
+
hydratedDocuments.push({ documentId: artifactId, title: row.fileName });
|
|
132
205
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
206
|
+
}
|
|
207
|
+
const indexEmpty = hydratedDocuments.length === 0;
|
|
208
|
+
const availableFiles = [...new Set(hydratedDocuments.map((document) => document.title))];
|
|
209
|
+
const failedRows = new Set();
|
|
210
|
+
const fileStates = await chatFiles();
|
|
211
|
+
const statusById = new Map(fileStates.map((file) => [file.fileId, file]));
|
|
212
|
+
for (const row of rows) {
|
|
213
|
+
if (artifactByRow.get(row))
|
|
214
|
+
continue;
|
|
215
|
+
const rag = statusById
|
|
216
|
+
.get(row.id)
|
|
217
|
+
?.processors.find((processor) => processor.processorId === 'rag');
|
|
218
|
+
if (rag?.state === 'failed') {
|
|
219
|
+
failedRows.add(row);
|
|
141
220
|
}
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
221
|
+
}
|
|
222
|
+
pendingFiles = pendingFileNames(fileStates, new Set(rows.filter((row) => artifactByRow.get(row)).map((row) => row.id)));
|
|
223
|
+
const failedFiles = scopeRows.filter((row) => failedRows.has(row)).map((row) => row.fileName);
|
|
224
|
+
const scopeDocumentIds = [];
|
|
225
|
+
let scopePending = false;
|
|
226
|
+
for (const row of scopeRows) {
|
|
227
|
+
const artifactId = artifactByRow.get(row);
|
|
228
|
+
if (artifactId) {
|
|
229
|
+
scopeDocumentIds.push(artifactId);
|
|
149
230
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
title: row.fileName,
|
|
153
|
-
pending: !artifactByRow.get(row) && !failedRows.has(row)
|
|
154
|
-
})));
|
|
155
|
-
if (pendingReferenced) {
|
|
156
|
-
return respond({ pendingReferenced, indexEmpty, hasKnowledge, availableFiles, failedFiles });
|
|
231
|
+
else if (!failedRows.has(row)) {
|
|
232
|
+
scopePending = true;
|
|
157
233
|
}
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
234
|
+
}
|
|
235
|
+
const chunkedOnly = scopeRows.some((row) => !artifactByRow.get(row) && !failedRows.has(row) && mesh.files.hasText(row.id));
|
|
236
|
+
if (!vectors && scopeDocumentIds.length === 0 && chunkedOnly) {
|
|
237
|
+
return ragSearch({
|
|
238
|
+
chatId,
|
|
239
|
+
query,
|
|
240
|
+
mode,
|
|
241
|
+
limit,
|
|
242
|
+
attachmentIds,
|
|
243
|
+
meshId,
|
|
244
|
+
immediate,
|
|
245
|
+
previousQuery,
|
|
246
|
+
excerptsOnly
|
|
247
|
+
}, { vectors: true });
|
|
248
|
+
}
|
|
249
|
+
if (scopePending && scopeDocumentIds.length === 0) {
|
|
250
|
+
return respond({
|
|
251
|
+
pendingReferenced: true,
|
|
252
|
+
indexEmpty,
|
|
253
|
+
hasKnowledge,
|
|
254
|
+
availableFiles,
|
|
255
|
+
failedFiles
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
if (scopeIds.size > 0 && scopeDocumentIds.length === 0) {
|
|
259
|
+
return respond({
|
|
260
|
+
indexEmpty,
|
|
261
|
+
hasKnowledge,
|
|
262
|
+
availableFiles,
|
|
263
|
+
failedFiles
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
const pendingReferenced = scopeDocumentIds.length === 0 &&
|
|
267
|
+
referencesPendingFile(query, rows.map((row) => ({
|
|
268
|
+
title: row.fileName,
|
|
269
|
+
pending: !artifactByRow.get(row) && !failedRows.has(row)
|
|
270
|
+
})));
|
|
271
|
+
if (pendingReferenced) {
|
|
272
|
+
return respond({ pendingReferenced, indexEmpty, hasKnowledge, availableFiles, failedFiles });
|
|
273
|
+
}
|
|
274
|
+
const failedReferenced = scopeRows.length === 0 &&
|
|
275
|
+
failedRows.size > 0 &&
|
|
276
|
+
referencesPendingFile(query, rows.map((row) => ({ title: row.fileName, pending: failedRows.has(row) })));
|
|
277
|
+
if (failedReferenced) {
|
|
278
|
+
return respond({
|
|
279
|
+
indexEmpty,
|
|
280
|
+
hasKnowledge,
|
|
281
|
+
availableFiles,
|
|
282
|
+
failedFiles: rows.filter((row) => failedRows.has(row)).map((row) => row.fileName)
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
const allFailed = indexEmpty && rows.length > 0 && rows.every((row) => failedRows.has(row));
|
|
286
|
+
if (allFailed) {
|
|
287
|
+
return respond({
|
|
288
|
+
indexEmpty,
|
|
289
|
+
hasKnowledge,
|
|
290
|
+
availableFiles,
|
|
291
|
+
failedFiles: rows.map((row) => row.fileName)
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
if (chatIndex instanceof PluginVectorIndex) {
|
|
295
|
+
const ctxSize = effectiveCtxSize ? await effectiveCtxSize(agent) : 0;
|
|
296
|
+
const documents = [...artifactByRow].flatMap(([row, artifactId]) => {
|
|
297
|
+
const owner = kbRows.includes(row) ? projectIndex : chatIndex;
|
|
298
|
+
return artifactId && owner instanceof PluginVectorIndex
|
|
299
|
+
? [{ documentId: artifactId, title: row.fileName, index: owner }]
|
|
300
|
+
: [];
|
|
301
|
+
});
|
|
302
|
+
const scopes = [
|
|
303
|
+
...(chatRows.length > 0 ? [chatIndex.scope(chatId)] : []),
|
|
304
|
+
...(kbRows.length > 0 && projectIndex instanceof PluginVectorIndex
|
|
305
|
+
? [projectIndex.scope(null)]
|
|
306
|
+
: [])
|
|
307
|
+
];
|
|
308
|
+
try {
|
|
309
|
+
const found = await searchPlugin({
|
|
310
|
+
query,
|
|
311
|
+
scopes,
|
|
312
|
+
documents,
|
|
313
|
+
scopeDocumentIds,
|
|
314
|
+
budgetTokens: retrievalBudgetTokens(ctxSize),
|
|
315
|
+
previousQuery: previousQuery ?? undefined,
|
|
316
|
+
limit: toolMode ? clampLimit(limit) : undefined
|
|
167
317
|
});
|
|
318
|
+
const hits = found.hits.map(toWireHit);
|
|
319
|
+
const searched = { searchedCount: hits.length, indexEmpty, hasKnowledge, availableFiles };
|
|
320
|
+
return toolMode
|
|
321
|
+
? respond({ ...searched, hits })
|
|
322
|
+
: respond({
|
|
323
|
+
...searched,
|
|
324
|
+
targetDocumentId: found.targetDocumentId,
|
|
325
|
+
failedFiles,
|
|
326
|
+
uncoveredItems: listed(found.uncovered),
|
|
327
|
+
outline: listed(found.outline),
|
|
328
|
+
hits
|
|
329
|
+
});
|
|
168
330
|
}
|
|
169
|
-
|
|
170
|
-
if (allFailed) {
|
|
331
|
+
catch (err) {
|
|
171
332
|
return respond({
|
|
172
333
|
indexEmpty,
|
|
173
334
|
hasKnowledge,
|
|
174
335
|
availableFiles,
|
|
175
|
-
|
|
336
|
+
errorMessage: errorMessage(err)
|
|
176
337
|
});
|
|
177
338
|
}
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
const hits = await searchRagChunks({
|
|
181
|
-
query,
|
|
182
|
-
chatId,
|
|
183
|
-
limit: clampLimit(limit),
|
|
184
|
-
liveSourceIds: rows.map((row) => row.id)
|
|
185
|
-
}, {
|
|
186
|
-
index: vectorIndex,
|
|
187
|
-
embed: (texts) => embed.embed({ texts }),
|
|
188
|
-
debug: (message) => mesh.log?.debug('rag', message)
|
|
189
|
-
});
|
|
190
|
-
return respond({
|
|
191
|
-
searchedCount: hits.length,
|
|
192
|
-
indexEmpty,
|
|
193
|
-
hasKnowledge,
|
|
194
|
-
availableFiles,
|
|
195
|
-
hits: hits.map(toWireHit)
|
|
196
|
-
});
|
|
197
|
-
}
|
|
198
|
-
catch (err) {
|
|
199
|
-
return respond({
|
|
200
|
-
indexEmpty,
|
|
201
|
-
hasKnowledge,
|
|
202
|
-
availableFiles,
|
|
203
|
-
errorMessage: errorMessage(err)
|
|
204
|
-
});
|
|
205
|
-
}
|
|
206
|
-
}
|
|
339
|
+
}
|
|
340
|
+
if (toolMode) {
|
|
207
341
|
try {
|
|
208
|
-
const
|
|
209
|
-
const result = await retrieveRagKnowledge({
|
|
342
|
+
const hits = await searchRagChunks({
|
|
210
343
|
query,
|
|
211
344
|
chatId,
|
|
212
|
-
|
|
213
|
-
liveSourceIds: rows.map((row) => row.id)
|
|
214
|
-
scopeDocumentIds: scopeDocumentIds.length > 0 ? scopeDocumentIds : undefined,
|
|
215
|
-
chunkBudgetTokens: retrievalBudgetTokens(ctxSize)
|
|
345
|
+
limit: clampLimit(limit),
|
|
346
|
+
liveSourceIds: rows.map((row) => row.id)
|
|
216
347
|
}, {
|
|
217
348
|
index: vectorIndex,
|
|
218
349
|
embed: (texts) => embed.embed({ texts }),
|
|
219
350
|
debug: (message) => mesh.log?.debug('rag', message)
|
|
220
351
|
});
|
|
221
352
|
return respond({
|
|
222
|
-
searchedCount:
|
|
353
|
+
searchedCount: hits.length,
|
|
223
354
|
indexEmpty,
|
|
224
355
|
hasKnowledge,
|
|
225
|
-
targetDocumentId: result.targetDocumentId,
|
|
226
356
|
availableFiles,
|
|
227
|
-
|
|
228
|
-
hits: result.hits.map(toWireHit)
|
|
357
|
+
hits: hits.map(toWireHit)
|
|
229
358
|
});
|
|
230
359
|
}
|
|
231
360
|
catch (err) {
|
|
@@ -233,13 +362,121 @@ export function createRagApi(pool, { index, embedder, effectiveCtxSize = null })
|
|
|
233
362
|
indexEmpty,
|
|
234
363
|
hasKnowledge,
|
|
235
364
|
availableFiles,
|
|
236
|
-
failedFiles,
|
|
237
365
|
errorMessage: errorMessage(err)
|
|
238
366
|
});
|
|
239
367
|
}
|
|
240
368
|
}
|
|
369
|
+
try {
|
|
370
|
+
const ctxSize = effectiveCtxSize ? await effectiveCtxSize(agent) : 0;
|
|
371
|
+
const result = await retrieveRagKnowledge({
|
|
372
|
+
query,
|
|
373
|
+
chatId,
|
|
374
|
+
documents: hydratedDocuments,
|
|
375
|
+
liveSourceIds: rows.map((row) => row.id),
|
|
376
|
+
scopeDocumentIds: scopeDocumentIds.length > 0 ? scopeDocumentIds : undefined,
|
|
377
|
+
chunkBudgetTokens: retrievalBudgetTokens(ctxSize)
|
|
378
|
+
}, {
|
|
379
|
+
index: vectorIndex,
|
|
380
|
+
embed: (texts) => embed.embed({ texts }),
|
|
381
|
+
debug: (message) => mesh.log?.debug('rag', message)
|
|
382
|
+
});
|
|
383
|
+
return respond({
|
|
384
|
+
searchedCount: result.searchedCount,
|
|
385
|
+
indexEmpty,
|
|
386
|
+
hasKnowledge,
|
|
387
|
+
targetDocumentId: result.targetDocumentId,
|
|
388
|
+
availableFiles,
|
|
389
|
+
failedFiles,
|
|
390
|
+
hits: result.hits.map(toWireHit)
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
catch (err) {
|
|
394
|
+
return respond({
|
|
395
|
+
indexEmpty,
|
|
396
|
+
hasKnowledge,
|
|
397
|
+
availableFiles,
|
|
398
|
+
failedFiles,
|
|
399
|
+
errorMessage: errorMessage(err)
|
|
400
|
+
});
|
|
401
|
+
}
|
|
241
402
|
};
|
|
403
|
+
return { ragSearch: (request) => ragSearch(request) };
|
|
404
|
+
}
|
|
405
|
+
async function textsOf(mesh, rows, maxChars = Infinity) {
|
|
406
|
+
const texts = [];
|
|
407
|
+
let chars = 0;
|
|
408
|
+
for (const row of rows) {
|
|
409
|
+
const text = await mesh.files.textOf(row.id, row.blobId);
|
|
410
|
+
chars += text?.content.length ?? 0;
|
|
411
|
+
if (chars > maxChars) {
|
|
412
|
+
return null;
|
|
413
|
+
}
|
|
414
|
+
texts.push(text);
|
|
415
|
+
}
|
|
416
|
+
return texts;
|
|
417
|
+
}
|
|
418
|
+
function pickFromText(rows, texts, { query, budgetTokens, maxBudgetTokens, synthesis, previousQuery }) {
|
|
419
|
+
const pendingIds = rows.filter((_, at) => !texts[at]).map((row) => row.id);
|
|
420
|
+
if (pendingIds.length === rows.length) {
|
|
421
|
+
return null;
|
|
422
|
+
}
|
|
423
|
+
const chunks = texts.flatMap((text) => text?.chunks ?? []);
|
|
424
|
+
const pick = rankLexical(chunks.map((chunk) => ({ ...chunk, score: 0 })), query, { budgetTokens, maxBudgetTokens, synthesis, previousQuery });
|
|
425
|
+
return pick && { ...pick, pendingIds };
|
|
426
|
+
}
|
|
427
|
+
async function unreadFiles(mesh, { files, rows, ids }) {
|
|
428
|
+
const unread = { pending: [], failed: [], states: [] };
|
|
429
|
+
for (const row of rows.filter((candidate) => ids.includes(candidate.id))) {
|
|
430
|
+
const file = files.find((candidate) => candidate.fileId === row.id);
|
|
431
|
+
const state = (status) => unread.states.push({ id: row.id, name: row.fileName, status });
|
|
432
|
+
if (file && pendingFileNames([file]).length > 0) {
|
|
433
|
+
unread.pending.push(row.fileName);
|
|
434
|
+
state(UNREAD.reading);
|
|
435
|
+
}
|
|
436
|
+
else if (file?.state === 'failed') {
|
|
437
|
+
unread.failed.push(row.fileName);
|
|
438
|
+
const rag = file.processors.find((processor) => processor.processorId === 'rag');
|
|
439
|
+
state(file.reason === 'no readable text' ||
|
|
440
|
+
rag?.reason === 'no readable text' ||
|
|
441
|
+
(await noText(mesh, row.id))
|
|
442
|
+
? UNREAD.noText
|
|
443
|
+
: UNREAD.unreadable);
|
|
444
|
+
}
|
|
445
|
+
else {
|
|
446
|
+
unread.pending.push(row.fileName);
|
|
447
|
+
state(UNREAD.remote);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
return unread;
|
|
451
|
+
}
|
|
452
|
+
async function listFiles(mesh, { files, rows }) {
|
|
453
|
+
const details = new Map();
|
|
454
|
+
for (const row of rows) {
|
|
455
|
+
const text = await mesh.files.textOf(row.id, row.blobId);
|
|
456
|
+
if (text) {
|
|
457
|
+
details.set(row.id, readable(text));
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
const unread = await unreadFiles(mesh, {
|
|
461
|
+
files,
|
|
462
|
+
rows,
|
|
463
|
+
ids: rows.filter((row) => !details.has(row.id)).map((row) => row.id)
|
|
464
|
+
});
|
|
465
|
+
for (const { id, status } of unread.states) {
|
|
466
|
+
details.set(id, status);
|
|
467
|
+
}
|
|
468
|
+
return manifestBlock(rows.map((row) => ({ title: row.fileName, detail: details.get(row.id) ?? '' })));
|
|
469
|
+
}
|
|
470
|
+
async function noText(mesh, knowledgeId) {
|
|
471
|
+
const latest = (await listIndexingByKnowledge(mesh, { knowledgeId })).reduce((current, row) => (!current || row.at > current.at ? row : current), null);
|
|
472
|
+
return latest?.status === 'failed' && parseFailure(latest.error).reason === 'no-text';
|
|
473
|
+
}
|
|
474
|
+
function listed(items) {
|
|
475
|
+
return items?.length ? [...items] : undefined;
|
|
242
476
|
}
|
|
477
|
+
const TEXT_CHARS = 4_000_000;
|
|
478
|
+
const WHOLE_CHARS = 32_000;
|
|
479
|
+
const MAX_LIST_CONTEXT_TOKENS = 3584;
|
|
243
480
|
function toWireHit(hit) {
|
|
244
481
|
return { id: hit.id, content: hit.content, metadataJson: hit.metadataJson, score: hit.score };
|
|
245
482
|
}
|