@qvac/core 1.0.0-rc.8 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +243 -3
- package/README.md +1 -0
- package/dist-lib/lib/api/attachments.d.ts +1 -1
- package/dist-lib/lib/api/attachments.js +11 -0
- package/dist-lib/lib/api/chunks.d.ts +4 -2
- package/dist-lib/lib/api/chunks.js +6 -3
- package/dist-lib/lib/api/devices.js +6 -18
- package/dist-lib/lib/api/execution.d.ts +6 -2
- package/dist-lib/lib/api/execution.js +29 -3
- package/dist-lib/lib/api/host-run-log.d.ts +7 -1
- package/dist-lib/lib/api/host-run-log.js +3 -1
- package/dist-lib/lib/api/index.d.ts +3 -1
- package/dist-lib/lib/api/index.js +17 -6
- package/dist-lib/lib/api/members.js +13 -29
- package/dist-lib/lib/api/mesh.d.ts +2 -1
- package/dist-lib/lib/api/projects.d.ts +1 -1
- package/dist-lib/lib/api/projects.js +9 -5
- package/dist-lib/lib/api/rag-label.d.ts +4 -0
- package/dist-lib/lib/api/rag-label.js +21 -0
- package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
- package/dist-lib/lib/api/rag-lexical.js +1058 -0
- package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
- package/dist-lib/lib/api/rag-manifest.js +46 -0
- package/dist-lib/lib/api/rag-outline.d.ts +36 -0
- package/dist-lib/lib/api/rag-outline.js +426 -0
- package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
- package/dist-lib/lib/api/rag-plugin.js +70 -0
- package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
- package/dist-lib/lib/api/rag-synthesis.js +102 -0
- package/dist-lib/lib/api/rag-targets.d.ts +23 -0
- package/dist-lib/lib/api/rag-targets.js +215 -0
- package/dist-lib/lib/api/rag-window.d.ts +5 -0
- package/dist-lib/lib/api/rag-window.js +46 -0
- package/dist-lib/lib/api/rag.d.ts +3 -2
- package/dist-lib/lib/api/rag.js +421 -181
- package/dist-lib/lib/api/served.js +8 -14
- package/dist-lib/lib/api/workspace.d.ts +8 -1
- package/dist-lib/lib/api/workspace.js +27 -13
- package/dist-lib/lib/assistant/harness.d.ts +9 -1
- package/dist-lib/lib/assistant/history.d.ts +11 -2
- package/dist-lib/lib/assistant/history.js +74 -4
- package/dist-lib/lib/assistant/media-round.d.ts +61 -0
- package/dist-lib/lib/assistant/media-round.js +111 -0
- package/dist-lib/lib/assistant/message.d.ts +7 -0
- package/dist-lib/lib/assistant/model-round.d.ts +12 -0
- package/dist-lib/lib/assistant/model-round.js +130 -17
- package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
- package/dist-lib/lib/assistant/pdf-pages.js +67 -3
- package/dist-lib/lib/assistant/round-host.d.ts +22 -0
- package/dist-lib/lib/assistant/round-host.js +142 -8
- package/dist-lib/lib/assistant/upload.d.ts +12 -0
- package/dist-lib/lib/assistant/upload.js +36 -0
- package/dist-lib/lib/assistant-client.d.ts +1 -0
- package/dist-lib/lib/assistant-client.js +40 -19
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +41 -1
- package/dist-lib/lib/assistant.d.ts +15 -0
- package/dist-lib/lib/assistant.js +474 -89
- package/dist-lib/lib/client.d.ts +1 -0
- package/dist-lib/lib/client.js +4 -1
- package/dist-lib/lib/compaction.js +2 -2
- package/dist-lib/lib/core/agent.js +12 -45
- package/dist-lib/lib/core/approval-rules.js +1 -8
- package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
- package/dist-lib/lib/core/archived-mesh.js +12 -8
- package/dist-lib/lib/core/chat-copy.d.ts +18 -3
- package/dist-lib/lib/core/chat-copy.js +74 -25
- package/dist-lib/lib/core/chat-fork.js +15 -6
- package/dist-lib/lib/core/chat-group.js +16 -24
- package/dist-lib/lib/core/chat-move.d.ts +9 -0
- package/dist-lib/lib/core/chat-move.js +94 -3
- package/dist-lib/lib/core/chat-order.js +12 -18
- package/dist-lib/lib/core/chat.js +38 -31
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +18 -25
- package/dist-lib/lib/core/collections.d.ts +2 -0
- package/dist-lib/lib/core/collections.js +2 -0
- package/dist-lib/lib/core/contract/activity.d.ts +4 -0
- package/dist-lib/lib/core/contract/activity.js +34 -4
- package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
- package/dist-lib/lib/core/contract/chat-group.js +21 -1
- package/dist-lib/lib/core/contract/chat.d.ts +9 -0
- package/dist-lib/lib/core/contract/chat.js +127 -5
- package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
- package/dist-lib/lib/core/contract/chunks.js +37 -6
- package/dist-lib/lib/core/contract/features.d.ts +21 -0
- package/dist-lib/lib/core/contract/features.js +22 -0
- package/dist-lib/lib/core/contract/index.js +2 -0
- package/dist-lib/lib/core/contract/indexing.js +4 -2
- package/dist-lib/lib/core/contract/knowledge.js +5 -4
- package/dist-lib/lib/core/contract/members.js +16 -10
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
- package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
- package/dist-lib/lib/core/contract/projects.d.ts +5 -0
- package/dist-lib/lib/core/contract/projects.js +13 -5
- package/dist-lib/lib/core/contract/roles.js +32 -4
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
- package/dist-lib/lib/core/contract/run-requests.js +35 -2
- package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
- package/dist-lib/lib/core/contract/tool-configs.js +3 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
- package/dist-lib/lib/core/contract/workspace.js +40 -7
- package/dist-lib/lib/core/contract/writers.js +2 -2
- package/dist-lib/lib/core/departures.d.ts +3 -1
- package/dist-lib/lib/core/departures.js +3 -0
- package/dist-lib/lib/core/devices.js +1 -8
- package/dist-lib/lib/core/files/attachments.d.ts +2 -0
- package/dist-lib/lib/core/files/attachments.js +19 -1
- package/dist-lib/lib/core/files/index.d.ts +20 -4
- package/dist-lib/lib/core/files/index.js +132 -21
- package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
- package/dist-lib/lib/core/files/processing/document.js +7 -0
- package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
- package/dist-lib/lib/core/files/processing/index.js +144 -12
- package/dist-lib/lib/core/files/processing/protocol.js +13 -34
- package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
- package/dist-lib/lib/core/files/rag/build.js +123 -19
- package/dist-lib/lib/core/files/rag/failure.js +2 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
- package/dist-lib/lib/core/files/rag/index.js +13 -1
- package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
- package/dist-lib/lib/core/files/rag/load.js +27 -6
- package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
- package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
- package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/produce.js +104 -16
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
- package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
- package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
- package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
- package/dist-lib/lib/core/files/records/knowledge.js +7 -40
- package/dist-lib/lib/core/files/records/reclaim.js +5 -10
- package/dist-lib/lib/core/files/upload.js +68 -41
- package/dist-lib/lib/core/fs-drives.js +14 -26
- package/dist-lib/lib/core/invite.d.ts +1 -0
- package/dist-lib/lib/core/invite.js +4 -2
- package/dist-lib/lib/core/local/files.d.ts +1 -1
- package/dist-lib/lib/core/local/files.js +5 -5
- package/dist-lib/lib/core/local/index.d.ts +13 -6
- package/dist-lib/lib/core/local/index.js +103 -69
- package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
- package/dist-lib/lib/core/local/run-logs.js +5 -5
- package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
- package/dist-lib/lib/core/mesh-processor.js +87 -0
- package/dist-lib/lib/core/mesh.d.ts +48 -5
- package/dist-lib/lib/core/mesh.js +318 -129
- package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
- package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
- package/dist-lib/lib/core/migrations/writer-links.js +32 -0
- package/dist-lib/lib/core/project-keys.d.ts +6 -0
- package/dist-lib/lib/core/project-keys.js +11 -0
- package/dist-lib/lib/core/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/public-mesh.js +10 -8
- package/dist-lib/lib/core/run-requests.d.ts +3 -0
- package/dist-lib/lib/core/run-requests.js +21 -1
- package/dist-lib/lib/core/skill-availability.d.ts +33 -0
- package/dist-lib/lib/core/skill-availability.js +67 -0
- package/dist-lib/lib/core/sync/background.d.ts +0 -1
- package/dist-lib/lib/core/sync/background.js +2 -1
- package/dist-lib/lib/core/sync/changed.d.ts +4 -2
- package/dist-lib/lib/core/sync/changed.js +1 -1
- package/dist-lib/lib/core/sync/index.js +52 -10
- package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
- package/dist-lib/lib/core/sync/view-updates.js +82 -91
- package/dist-lib/lib/core/sync-status.js +97 -41
- package/dist-lib/lib/core/tool-activity.js +6 -12
- package/dist-lib/lib/core/tool-configs.js +17 -29
- package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
- package/dist-lib/lib/core/upgrade-summary.js +67 -0
- package/dist-lib/lib/core/view-read.d.ts +4 -0
- package/dist-lib/lib/core/view-read.js +23 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
- package/dist-lib/lib/core/workspace-upgrade.js +27 -16
- package/dist-lib/lib/core.d.ts +39 -42
- package/dist-lib/lib/core.js +661 -639
- package/dist-lib/lib/errors.d.ts +20 -0
- package/dist-lib/lib/errors.js +20 -0
- package/dist-lib/lib/llm-model-names.d.ts +1 -1
- package/dist-lib/lib/llm-model-names.js +1 -1
- package/dist-lib/lib/mesh-scoped-engine.js +4 -0
- package/dist-lib/lib/model-context.js +1 -2
- package/dist-lib/lib/probe-rounds.d.ts +7 -2
- package/dist-lib/lib/probe-rounds.js +6 -1
- package/dist-lib/lib/run-transcript.d.ts +4 -1
- package/dist-lib/lib/run-transcript.js +15 -6
- package/dist-lib/schema/engine.js +83 -6
- package/dist-lib/schema/indexes.d.ts +3 -0
- package/dist-lib/schema/indexes.js +5 -0
- package/dist-lib/schema/local.js +11 -2
- package/dist-lib/schema/qvac.js +49 -5
- package/dist-lib/spec/bind.js +34 -0
- package/dist-lib/spec/calls.js +5 -0
- package/dist-lib/spec/capabilities.d.ts +17 -0
- package/dist-lib/spec/hrpc/hrpc.json +65 -0
- package/dist-lib/spec/hrpc/index.js +68 -5
- package/dist-lib/spec/hrpc/messages.js +260 -23
- package/dist-lib/spec/hyperschema/index.js +260 -23
- package/dist-lib/spec/hyperschema/schema.json +181 -0
- package/dist-lib/spec/hyperschema/types.d.ts +48 -0
- package/dist-lib/spec/local/hyperdb/messages.js +270 -224
- package/dist-lib/spec/local/hyperschema/index.js +73 -38
- package/dist-lib/spec/local/hyperschema/schema.json +31 -0
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
- package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
- package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
- package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
- package/package.json +75 -70
- package/tui/README.md +2 -1
- package/tui/app.mjs +28 -7
- package/tui/boot.mjs +5 -1
- package/tui/lib/attachment-files.mjs +82 -0
- package/tui/lib/reopen-meshes.mjs +15 -0
- package/tui/main.mjs +4 -43
- package/worker/assistant-sidecar.mjs +10 -3
- package/worker/stow-entry.mjs +3 -52
package/dist-lib/lib/api/rag.js
CHANGED
|
@@ -2,9 +2,16 @@ import { toolsetEnabled } from '@qvac/harness/lib/tools/toolsets.ts';
|
|
|
2
2
|
import { getAgent } from '../core/agent.js';
|
|
3
3
|
import { runDefinition } from '../core/shared-agents.js';
|
|
4
4
|
import { getChat } from '../core/chat.js';
|
|
5
|
-
import { isKnowledgeRoutedAttachment, fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from '../core/files/index.js';
|
|
5
|
+
import { isKnowledgeRoutedAttachment, fullTextBudgetChars, fullTextHits, joinedIndex, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, referencedDocuments, PluginVectorIndex } from '../core/files/index.js';
|
|
6
6
|
import { requireMesh } from './current-mesh.js';
|
|
7
|
+
import { searchPlugin } from './rag-plugin.js';
|
|
8
|
+
import { rankLexical } from './rag-lexical.js';
|
|
9
|
+
import { manifestBlock, readable, UNREAD } from './rag-manifest.js';
|
|
10
|
+
import { classifyRequest, outlineBlock } from './rag-synthesis.js';
|
|
11
|
+
import { DEFAULT_CHUNK_BUDGET_TOKENS } from 'rag-extract';
|
|
7
12
|
import { errorMessage } from '../guards.js';
|
|
13
|
+
import { parseFailure } from '../core/files/rag/failure.js';
|
|
14
|
+
import { listIndexingByKnowledge } from '../core/files/processing/index.js';
|
|
8
15
|
import b4a from 'b4a';
|
|
9
16
|
export const MAX_RAG_SEARCH_LIMIT = 20;
|
|
10
17
|
const DEFAULT_RAG_SEARCH_LIMIT = 5;
|
|
@@ -13,216 +20,341 @@ function clampLimit(limit) {
|
|
|
13
20
|
return DEFAULT_RAG_SEARCH_LIMIT;
|
|
14
21
|
return Math.min(Math.max(1, Math.floor(limit)), MAX_RAG_SEARCH_LIMIT);
|
|
15
22
|
}
|
|
16
|
-
export function createRagApi(pool, { index, embedder, effectiveCtxSize = null }) {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
23
|
+
export function createRagApi(pool, { index, embedder, effectiveCtxSize = null, extractors }) {
|
|
24
|
+
const ragSearch = async ({ chatId, query, mode, limit, attachmentIds, meshId, immediate, previousQuery, excerptsOnly }, { vectors = false } = {}) => {
|
|
25
|
+
let pendingFiles = [];
|
|
26
|
+
let projectFilesNotReady = 0;
|
|
27
|
+
let chatFilesIndexingHere = 0;
|
|
28
|
+
let manifest;
|
|
29
|
+
const respond = (overrides) => {
|
|
30
|
+
const result = response({
|
|
31
|
+
pendingFiles,
|
|
32
|
+
projectFilesNotReady,
|
|
33
|
+
chatFilesIndexingHere,
|
|
34
|
+
...overrides,
|
|
35
|
+
outline: manifest ?? overrides.outline
|
|
36
|
+
});
|
|
37
|
+
return { ...result, chatFilesNotReady: result.pendingFiles?.length ?? 0 };
|
|
38
|
+
};
|
|
39
|
+
const mesh = await requireMesh(pool, meshId);
|
|
40
|
+
const chat = await getChat(mesh, chatId);
|
|
41
|
+
const toolMode = mode === 'tool';
|
|
42
|
+
if (!chat) {
|
|
43
|
+
return respond({ enabled: false });
|
|
44
|
+
}
|
|
45
|
+
const request = classifyRequest(query);
|
|
46
|
+
const agent = await runDefinition(mesh, await getAgent(mesh, chat.agentId));
|
|
47
|
+
const scopeIds = new Set(toolMode ? [] : (attachmentIds ?? []));
|
|
48
|
+
const off = new Set(chat.knowledgeSourcesOff ?? []);
|
|
49
|
+
const kept = (id) => !off.has('chat') || scopeIds.has(id);
|
|
50
|
+
const chatKnowledge = async () => (await mesh.files.listKnowledge(chatId)).filter((row) => kept(row.id));
|
|
51
|
+
const chatFiles = async () => (await mesh.files.status(chatId)).files.filter((file) => kept(file.fileId));
|
|
52
|
+
const indexingHere = (files) => files.filter((file) => scopeIds.has(file.fileId) &&
|
|
53
|
+
(vectors ? mesh.files.indexesHere(file) : mesh.files.extractingHere(file))).length;
|
|
54
|
+
if (!immediate && scopeIds.size > 0) {
|
|
55
|
+
const ids = [...scopeIds];
|
|
56
|
+
await (vectors ? mesh.files.indexingHere(chatId, ids) : mesh.files.textReadyHere(chatId, ids));
|
|
57
|
+
}
|
|
58
|
+
else if (immediate && vectors && scopeIds.size > 0) {
|
|
59
|
+
await mesh.files.requestVectors([...scopeIds]);
|
|
60
|
+
}
|
|
61
|
+
const chatRows = await chatKnowledge();
|
|
62
|
+
if (toolMode && chatRows.length > 0) {
|
|
63
|
+
const ids = chatRows.map((row) => row.id);
|
|
64
|
+
if (immediate) {
|
|
65
|
+
await mesh.files.requestVectors(ids);
|
|
36
66
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
const off = new Set(chat.knowledgeSourcesOff ?? []);
|
|
40
|
-
const kept = (id) => !off.has('chat') || scopeIds.has(id);
|
|
41
|
-
const chatKnowledge = async () => (await mesh.files.listKnowledge(chatId)).filter((row) => kept(row.id));
|
|
42
|
-
const chatFiles = async () => (await mesh.files.status(chatId)).files.filter((file) => kept(file.fileId));
|
|
43
|
-
const indexingHere = (files) => files.filter((file) => scopeIds.has(file.fileId) && mesh.files.indexesHere(file)).length;
|
|
44
|
-
if (!immediate && scopeIds.size > 0) {
|
|
45
|
-
await mesh.files.indexingHere(chatId, [...scopeIds]);
|
|
67
|
+
else {
|
|
68
|
+
await mesh.files.indexingHere(chatId, ids);
|
|
46
69
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
70
|
+
}
|
|
71
|
+
const initialFiles = await chatFiles();
|
|
72
|
+
pendingFiles = pendingFileNames(initialFiles);
|
|
73
|
+
chatFilesIndexingHere = indexingHere(initialFiles);
|
|
74
|
+
const scopeRows = chatRows.filter((row) => scopeIds.has(row.id));
|
|
75
|
+
const project = off.has('project') || mesh.role !== 'project' ? null : mesh;
|
|
76
|
+
const kbEntries = project ? ((await project.files.knowledgeBase()).entries ?? []) : [];
|
|
77
|
+
const kbRows = kbEntries.filter((entry) => entry.loaded).map((entry) => entry.knowledge);
|
|
78
|
+
projectFilesNotReady = kbEntries.filter((entry) => !entry.loaded && entry.status !== 'failed').length;
|
|
79
|
+
let window;
|
|
80
|
+
const ctxSize = () => (window ??= effectiveCtxSize ? effectiveCtxSize(agent) : Promise.resolve(0));
|
|
81
|
+
const attached = scopeIds.size > 0 && scopeRows.length === scopeIds.size ? scopeRows : [];
|
|
82
|
+
const named = new Set(referencedDocuments(query, chatRows.map((row) => ({ documentId: row.id, title: row.fileName }))).map((document) => document.documentId));
|
|
83
|
+
const focused = named.size > 0 ? chatRows.filter((row) => named.has(row.id)) : chatRows;
|
|
84
|
+
let subject = [];
|
|
85
|
+
if (scopeIds.size > 0) {
|
|
86
|
+
subject = attached;
|
|
87
|
+
}
|
|
88
|
+
else if (kbRows.length === 0) {
|
|
89
|
+
subject = focused;
|
|
90
|
+
}
|
|
91
|
+
if (!toolMode && request.wholeDocument && subject.length > 1) {
|
|
92
|
+
manifest = await listFiles(mesh, { files: initialFiles, rows: subject });
|
|
93
|
+
}
|
|
94
|
+
const packet = async (rows, pick) => {
|
|
95
|
+
const unread = await unreadFiles(mesh, {
|
|
96
|
+
files: initialFiles,
|
|
97
|
+
rows,
|
|
98
|
+
ids: pick.pendingIds
|
|
99
|
+
});
|
|
100
|
+
const read = rows.filter((row) => !pick.pendingIds.includes(row.id));
|
|
101
|
+
const gaps = unread.states.map(({ name, status }) => `${name}: ${status}`);
|
|
102
|
+
return respond({
|
|
103
|
+
pendingFiles: [
|
|
104
|
+
...new Set([
|
|
105
|
+
...pendingFileNames(initialFiles, new Set(read.map((row) => row.id))),
|
|
106
|
+
...unread.pending
|
|
107
|
+
])
|
|
108
|
+
],
|
|
109
|
+
searchedCount: pick.hits.length,
|
|
110
|
+
indexEmpty: false,
|
|
111
|
+
hasKnowledge: true,
|
|
112
|
+
availableFiles: [...new Set(read.map((row) => row.fileName))],
|
|
113
|
+
uncoveredItems: listed([...pick.uncovered, ...gaps]),
|
|
114
|
+
failedFiles: unread.failed,
|
|
115
|
+
outline: outlineBlock(pick),
|
|
116
|
+
hits: pick.hits.map(toWireHit)
|
|
117
|
+
});
|
|
118
|
+
};
|
|
119
|
+
const fromText = async (rows, { previousQuery: before, maxChars, whole = false }) => {
|
|
120
|
+
const allowed = agentAllowsRagSearch(agent);
|
|
121
|
+
const picks = !vectors && allowed;
|
|
122
|
+
const size = await ctxSize();
|
|
123
|
+
const budgetTokens = retrievalBudgetTokens(size) ?? DEFAULT_CHUNK_BUDGET_TOKENS;
|
|
124
|
+
const maxBudgetTokens = size > 0 ? Math.min(MAX_LIST_CONTEXT_TOKENS, Math.floor(size / 4)) : budgetTokens;
|
|
125
|
+
const wholeChars = Math.min(fullTextBudgetChars(size), WHOLE_CHARS);
|
|
126
|
+
const readWhole = (head) => !whole || excerptsOnly
|
|
127
|
+
? null
|
|
128
|
+
: fullTextHits(mesh, rows, { budgetChars: wholeChars, head, extractors: extractors?.() });
|
|
129
|
+
const wholeFirst = picks && request.synthesis;
|
|
130
|
+
const early = wholeFirst ? await readWhole(false) : null;
|
|
131
|
+
const texts = picks && !early ? await textsOf(mesh, rows, maxChars) : null;
|
|
132
|
+
const pick = texts &&
|
|
133
|
+
pickFromText(rows, texts, {
|
|
134
|
+
query,
|
|
135
|
+
budgetTokens,
|
|
136
|
+
maxBudgetTokens,
|
|
137
|
+
synthesis: request.synthesis,
|
|
138
|
+
previousQuery: before
|
|
61
139
|
});
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
return respond({
|
|
65
|
-
pendingFiles: pendingFileNames(initialFiles, read),
|
|
66
|
-
chatFilesIndexingHere: 0,
|
|
67
|
-
searchedCount: fullText.length,
|
|
68
|
-
indexEmpty: false,
|
|
69
|
-
hasKnowledge: true,
|
|
70
|
-
availableFiles: [...new Set(scopeRows.map((row) => row.fileName))],
|
|
71
|
-
hits: fullText
|
|
72
|
-
});
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
if (!toolMode && !agentAllowsRagSearch(agent)) {
|
|
76
|
-
return respond({ enabled: false });
|
|
77
|
-
}
|
|
78
|
-
if (chatRows.length === 0 && kbRows.length === 0) {
|
|
79
|
-
return respond({});
|
|
140
|
+
if (pick && pick.uncovered.length === 0) {
|
|
141
|
+
return packet(rows, pick);
|
|
80
142
|
}
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
143
|
+
const fullText = wholeFirst ? early : await readWhole(!allowed);
|
|
144
|
+
if (fullText) {
|
|
145
|
+
manifest = undefined;
|
|
146
|
+
return respond({
|
|
147
|
+
pendingFiles: pendingFileNames(initialFiles, new Set(rows.map((row) => row.id))),
|
|
148
|
+
chatFilesIndexingHere: 0,
|
|
149
|
+
searchedCount: fullText.length,
|
|
150
|
+
indexEmpty: false,
|
|
151
|
+
hasKnowledge: true,
|
|
152
|
+
availableFiles: [...new Set(rows.map((row) => row.fileName))],
|
|
153
|
+
hits: fullText
|
|
154
|
+
});
|
|
87
155
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
: chatIndex;
|
|
95
|
-
const artifactByRow = new Map([
|
|
96
|
-
...chatRows.map((row) => [row, mesh.files.hydratedArtifact(row.id)]),
|
|
97
|
-
...kbRows.map((row) => [row, project?.files.hydratedArtifact(row.id) ?? null])
|
|
98
|
-
]);
|
|
99
|
-
const hydratedDocuments = [];
|
|
100
|
-
for (const [row, artifactId] of artifactByRow) {
|
|
101
|
-
if (artifactId) {
|
|
102
|
-
hydratedDocuments.push({ documentId: artifactId, title: row.fileName });
|
|
103
|
-
}
|
|
156
|
+
return pick ? packet(rows, pick) : null;
|
|
157
|
+
};
|
|
158
|
+
if (attached.length > 0) {
|
|
159
|
+
const answer = await fromText(attached, { whole: true });
|
|
160
|
+
if (answer) {
|
|
161
|
+
return answer;
|
|
104
162
|
}
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
163
|
+
}
|
|
164
|
+
if (!toolMode && !agentAllowsRagSearch(agent)) {
|
|
165
|
+
return respond({ enabled: false });
|
|
166
|
+
}
|
|
167
|
+
if (chatRows.length === 0 && kbRows.length === 0) {
|
|
168
|
+
return respond({});
|
|
169
|
+
}
|
|
170
|
+
const followUp = !toolMode && !vectors && scopeIds.size === 0 && kbRows.length === 0;
|
|
171
|
+
if (followUp) {
|
|
172
|
+
const answer = await fromText(focused, {
|
|
173
|
+
maxChars: TEXT_CHARS,
|
|
174
|
+
previousQuery: named.size === 0 ? (previousQuery ?? undefined) : undefined
|
|
175
|
+
});
|
|
176
|
+
if (answer) {
|
|
177
|
+
return answer;
|
|
116
178
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
179
|
+
}
|
|
180
|
+
if (!toolMode && scopeIds.size === 0 && chatRows.length > 0) {
|
|
181
|
+
await mesh.files.requestVectors(chatRows.map((row) => row.id));
|
|
182
|
+
}
|
|
183
|
+
const rows = [...chatRows, ...kbRows];
|
|
184
|
+
const hasKnowledge = rows.length > 0;
|
|
185
|
+
const chatIndex = index(meshId);
|
|
186
|
+
const embed = embedder(meshId);
|
|
187
|
+
if (!chatIndex || !embed) {
|
|
188
|
+
return respond({ hasKnowledge, errorMessage: 'knowledge index unavailable on this device' });
|
|
189
|
+
}
|
|
190
|
+
const projectIndex = project && kbRows.length ? index(b4a.toHex(project.key)) : null;
|
|
191
|
+
const vectorIndex = projectIndex
|
|
192
|
+
? joinedIndex([chatIndex, projectIndex], {
|
|
193
|
+
chatId,
|
|
194
|
+
knowledgeBaseIds: new Set(kbRows.map((row) => row.id))
|
|
195
|
+
})
|
|
196
|
+
: chatIndex;
|
|
197
|
+
const artifactByRow = new Map([
|
|
198
|
+
...chatRows.map((row) => [row, mesh.files.hydratedArtifact(row.id)]),
|
|
199
|
+
...kbRows.map((row) => [row, project?.files.hydratedArtifact(row.id) ?? null])
|
|
200
|
+
]);
|
|
201
|
+
const hydratedDocuments = [];
|
|
202
|
+
for (const [row, artifactId] of artifactByRow) {
|
|
203
|
+
if (artifactId) {
|
|
204
|
+
hydratedDocuments.push({ documentId: artifactId, title: row.fileName });
|
|
129
205
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
206
|
+
}
|
|
207
|
+
const indexEmpty = hydratedDocuments.length === 0;
|
|
208
|
+
const availableFiles = [...new Set(hydratedDocuments.map((document) => document.title))];
|
|
209
|
+
const failedRows = new Set();
|
|
210
|
+
const fileStates = await chatFiles();
|
|
211
|
+
const statusById = new Map(fileStates.map((file) => [file.fileId, file]));
|
|
212
|
+
for (const row of rows) {
|
|
213
|
+
if (artifactByRow.get(row))
|
|
214
|
+
continue;
|
|
215
|
+
const rag = statusById
|
|
216
|
+
.get(row.id)
|
|
217
|
+
?.processors.find((processor) => processor.processorId === 'rag');
|
|
218
|
+
if (rag?.state === 'failed') {
|
|
219
|
+
failedRows.add(row);
|
|
138
220
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
221
|
+
}
|
|
222
|
+
pendingFiles = pendingFileNames(fileStates, new Set(rows.filter((row) => artifactByRow.get(row)).map((row) => row.id)));
|
|
223
|
+
const failedFiles = scopeRows.filter((row) => failedRows.has(row)).map((row) => row.fileName);
|
|
224
|
+
const scopeDocumentIds = [];
|
|
225
|
+
let scopePending = false;
|
|
226
|
+
for (const row of scopeRows) {
|
|
227
|
+
const artifactId = artifactByRow.get(row);
|
|
228
|
+
if (artifactId) {
|
|
229
|
+
scopeDocumentIds.push(artifactId);
|
|
146
230
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
title: row.fileName,
|
|
150
|
-
pending: !artifactByRow.get(row) && !failedRows.has(row)
|
|
151
|
-
})));
|
|
152
|
-
if (pendingReferenced) {
|
|
153
|
-
return respond({ pendingReferenced, indexEmpty, hasKnowledge, availableFiles, failedFiles });
|
|
231
|
+
else if (!failedRows.has(row)) {
|
|
232
|
+
scopePending = true;
|
|
154
233
|
}
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
234
|
+
}
|
|
235
|
+
const chunkedOnly = scopeRows.some((row) => !artifactByRow.get(row) && !failedRows.has(row) && mesh.files.hasText(row.id));
|
|
236
|
+
if (!vectors && scopeDocumentIds.length === 0 && chunkedOnly) {
|
|
237
|
+
return ragSearch({
|
|
238
|
+
chatId,
|
|
239
|
+
query,
|
|
240
|
+
mode,
|
|
241
|
+
limit,
|
|
242
|
+
attachmentIds,
|
|
243
|
+
meshId,
|
|
244
|
+
immediate,
|
|
245
|
+
previousQuery,
|
|
246
|
+
excerptsOnly
|
|
247
|
+
}, { vectors: true });
|
|
248
|
+
}
|
|
249
|
+
if (scopePending && scopeDocumentIds.length === 0) {
|
|
250
|
+
return respond({
|
|
251
|
+
pendingReferenced: true,
|
|
252
|
+
indexEmpty,
|
|
253
|
+
hasKnowledge,
|
|
254
|
+
availableFiles,
|
|
255
|
+
failedFiles
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
if (scopeIds.size > 0 && scopeDocumentIds.length === 0) {
|
|
259
|
+
return respond({
|
|
260
|
+
indexEmpty,
|
|
261
|
+
hasKnowledge,
|
|
262
|
+
availableFiles,
|
|
263
|
+
failedFiles
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
const pendingReferenced = scopeDocumentIds.length === 0 &&
|
|
267
|
+
referencesPendingFile(query, rows.map((row) => ({
|
|
268
|
+
title: row.fileName,
|
|
269
|
+
pending: !artifactByRow.get(row) && !failedRows.has(row)
|
|
270
|
+
})));
|
|
271
|
+
if (pendingReferenced) {
|
|
272
|
+
return respond({ pendingReferenced, indexEmpty, hasKnowledge, availableFiles, failedFiles });
|
|
273
|
+
}
|
|
274
|
+
const failedReferenced = scopeRows.length === 0 &&
|
|
275
|
+
failedRows.size > 0 &&
|
|
276
|
+
referencesPendingFile(query, rows.map((row) => ({ title: row.fileName, pending: failedRows.has(row) })));
|
|
277
|
+
if (failedReferenced) {
|
|
278
|
+
return respond({
|
|
279
|
+
indexEmpty,
|
|
280
|
+
hasKnowledge,
|
|
281
|
+
availableFiles,
|
|
282
|
+
failedFiles: rows.filter((row) => failedRows.has(row)).map((row) => row.fileName)
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
const allFailed = indexEmpty && rows.length > 0 && rows.every((row) => failedRows.has(row));
|
|
286
|
+
if (allFailed) {
|
|
287
|
+
return respond({
|
|
288
|
+
indexEmpty,
|
|
289
|
+
hasKnowledge,
|
|
290
|
+
availableFiles,
|
|
291
|
+
failedFiles: rows.map((row) => row.fileName)
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
if (chatIndex instanceof PluginVectorIndex) {
|
|
295
|
+
const ctxSize = effectiveCtxSize ? await effectiveCtxSize(agent) : 0;
|
|
296
|
+
const documents = [...artifactByRow].flatMap(([row, artifactId]) => {
|
|
297
|
+
const owner = kbRows.includes(row) ? projectIndex : chatIndex;
|
|
298
|
+
return artifactId && owner instanceof PluginVectorIndex
|
|
299
|
+
? [{ documentId: artifactId, title: row.fileName, index: owner }]
|
|
300
|
+
: [];
|
|
301
|
+
});
|
|
302
|
+
const scopes = [
|
|
303
|
+
...(chatRows.length > 0 ? [chatIndex.scope(chatId)] : []),
|
|
304
|
+
...(kbRows.length > 0 && projectIndex instanceof PluginVectorIndex
|
|
305
|
+
? [projectIndex.scope(null)]
|
|
306
|
+
: [])
|
|
307
|
+
];
|
|
308
|
+
try {
|
|
309
|
+
const found = await searchPlugin({
|
|
310
|
+
query,
|
|
311
|
+
scopes,
|
|
312
|
+
documents,
|
|
313
|
+
scopeDocumentIds,
|
|
314
|
+
budgetTokens: retrievalBudgetTokens(ctxSize),
|
|
315
|
+
previousQuery: previousQuery ?? undefined,
|
|
316
|
+
limit: toolMode ? clampLimit(limit) : undefined
|
|
164
317
|
});
|
|
318
|
+
const hits = found.hits.map(toWireHit);
|
|
319
|
+
const searched = { searchedCount: hits.length, indexEmpty, hasKnowledge, availableFiles };
|
|
320
|
+
return toolMode
|
|
321
|
+
? respond({ ...searched, hits })
|
|
322
|
+
: respond({
|
|
323
|
+
...searched,
|
|
324
|
+
targetDocumentId: found.targetDocumentId,
|
|
325
|
+
failedFiles,
|
|
326
|
+
uncoveredItems: listed(found.uncovered),
|
|
327
|
+
outline: listed(found.outline),
|
|
328
|
+
hits
|
|
329
|
+
});
|
|
165
330
|
}
|
|
166
|
-
|
|
167
|
-
if (allFailed) {
|
|
331
|
+
catch (err) {
|
|
168
332
|
return respond({
|
|
169
333
|
indexEmpty,
|
|
170
334
|
hasKnowledge,
|
|
171
335
|
availableFiles,
|
|
172
|
-
|
|
336
|
+
errorMessage: errorMessage(err)
|
|
173
337
|
});
|
|
174
338
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
const hits = await searchRagChunks({
|
|
178
|
-
query,
|
|
179
|
-
chatId,
|
|
180
|
-
limit: clampLimit(limit),
|
|
181
|
-
liveSourceIds: rows.map((row) => row.id)
|
|
182
|
-
}, {
|
|
183
|
-
index: vectorIndex,
|
|
184
|
-
embed: (texts) => embed.embed({ texts }),
|
|
185
|
-
debug: (message) => mesh.log?.debug('rag', message)
|
|
186
|
-
});
|
|
187
|
-
return respond({
|
|
188
|
-
searchedCount: hits.length,
|
|
189
|
-
indexEmpty,
|
|
190
|
-
hasKnowledge,
|
|
191
|
-
availableFiles,
|
|
192
|
-
hits: hits.map(toWireHit)
|
|
193
|
-
});
|
|
194
|
-
}
|
|
195
|
-
catch (err) {
|
|
196
|
-
return respond({
|
|
197
|
-
indexEmpty,
|
|
198
|
-
hasKnowledge,
|
|
199
|
-
availableFiles,
|
|
200
|
-
errorMessage: errorMessage(err)
|
|
201
|
-
});
|
|
202
|
-
}
|
|
203
|
-
}
|
|
339
|
+
}
|
|
340
|
+
if (toolMode) {
|
|
204
341
|
try {
|
|
205
|
-
const
|
|
206
|
-
const result = await retrieveRagKnowledge({
|
|
342
|
+
const hits = await searchRagChunks({
|
|
207
343
|
query,
|
|
208
344
|
chatId,
|
|
209
|
-
|
|
210
|
-
liveSourceIds: rows.map((row) => row.id)
|
|
211
|
-
scopeDocumentIds: scopeDocumentIds.length > 0 ? scopeDocumentIds : undefined,
|
|
212
|
-
chunkBudgetTokens: retrievalBudgetTokens(ctxSize)
|
|
345
|
+
limit: clampLimit(limit),
|
|
346
|
+
liveSourceIds: rows.map((row) => row.id)
|
|
213
347
|
}, {
|
|
214
348
|
index: vectorIndex,
|
|
215
349
|
embed: (texts) => embed.embed({ texts }),
|
|
216
350
|
debug: (message) => mesh.log?.debug('rag', message)
|
|
217
351
|
});
|
|
218
352
|
return respond({
|
|
219
|
-
searchedCount:
|
|
353
|
+
searchedCount: hits.length,
|
|
220
354
|
indexEmpty,
|
|
221
355
|
hasKnowledge,
|
|
222
|
-
targetDocumentId: result.targetDocumentId,
|
|
223
356
|
availableFiles,
|
|
224
|
-
|
|
225
|
-
hits: result.hits.map(toWireHit)
|
|
357
|
+
hits: hits.map(toWireHit)
|
|
226
358
|
});
|
|
227
359
|
}
|
|
228
360
|
catch (err) {
|
|
@@ -230,13 +362,121 @@ export function createRagApi(pool, { index, embedder, effectiveCtxSize = null })
|
|
|
230
362
|
indexEmpty,
|
|
231
363
|
hasKnowledge,
|
|
232
364
|
availableFiles,
|
|
233
|
-
failedFiles,
|
|
234
365
|
errorMessage: errorMessage(err)
|
|
235
366
|
});
|
|
236
367
|
}
|
|
237
368
|
}
|
|
369
|
+
try {
|
|
370
|
+
const ctxSize = effectiveCtxSize ? await effectiveCtxSize(agent) : 0;
|
|
371
|
+
const result = await retrieveRagKnowledge({
|
|
372
|
+
query,
|
|
373
|
+
chatId,
|
|
374
|
+
documents: hydratedDocuments,
|
|
375
|
+
liveSourceIds: rows.map((row) => row.id),
|
|
376
|
+
scopeDocumentIds: scopeDocumentIds.length > 0 ? scopeDocumentIds : undefined,
|
|
377
|
+
chunkBudgetTokens: retrievalBudgetTokens(ctxSize)
|
|
378
|
+
}, {
|
|
379
|
+
index: vectorIndex,
|
|
380
|
+
embed: (texts) => embed.embed({ texts }),
|
|
381
|
+
debug: (message) => mesh.log?.debug('rag', message)
|
|
382
|
+
});
|
|
383
|
+
return respond({
|
|
384
|
+
searchedCount: result.searchedCount,
|
|
385
|
+
indexEmpty,
|
|
386
|
+
hasKnowledge,
|
|
387
|
+
targetDocumentId: result.targetDocumentId,
|
|
388
|
+
availableFiles,
|
|
389
|
+
failedFiles,
|
|
390
|
+
hits: result.hits.map(toWireHit)
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
catch (err) {
|
|
394
|
+
return respond({
|
|
395
|
+
indexEmpty,
|
|
396
|
+
hasKnowledge,
|
|
397
|
+
availableFiles,
|
|
398
|
+
failedFiles,
|
|
399
|
+
errorMessage: errorMessage(err)
|
|
400
|
+
});
|
|
401
|
+
}
|
|
238
402
|
};
|
|
403
|
+
return { ragSearch: (request) => ragSearch(request) };
|
|
404
|
+
}
|
|
405
|
+
async function textsOf(mesh, rows, maxChars = Infinity) {
|
|
406
|
+
const texts = [];
|
|
407
|
+
let chars = 0;
|
|
408
|
+
for (const row of rows) {
|
|
409
|
+
const text = await mesh.files.textOf(row.id, row.blobId);
|
|
410
|
+
chars += text?.content.length ?? 0;
|
|
411
|
+
if (chars > maxChars) {
|
|
412
|
+
return null;
|
|
413
|
+
}
|
|
414
|
+
texts.push(text);
|
|
415
|
+
}
|
|
416
|
+
return texts;
|
|
417
|
+
}
|
|
418
|
+
function pickFromText(rows, texts, { query, budgetTokens, maxBudgetTokens, synthesis, previousQuery }) {
|
|
419
|
+
const pendingIds = rows.filter((_, at) => !texts[at]).map((row) => row.id);
|
|
420
|
+
if (pendingIds.length === rows.length) {
|
|
421
|
+
return null;
|
|
422
|
+
}
|
|
423
|
+
const chunks = texts.flatMap((text) => text?.chunks ?? []);
|
|
424
|
+
const pick = rankLexical(chunks.map((chunk) => ({ ...chunk, score: 0 })), query, { budgetTokens, maxBudgetTokens, synthesis, previousQuery });
|
|
425
|
+
return pick && { ...pick, pendingIds };
|
|
426
|
+
}
|
|
427
|
+
async function unreadFiles(mesh, { files, rows, ids }) {
|
|
428
|
+
const unread = { pending: [], failed: [], states: [] };
|
|
429
|
+
for (const row of rows.filter((candidate) => ids.includes(candidate.id))) {
|
|
430
|
+
const file = files.find((candidate) => candidate.fileId === row.id);
|
|
431
|
+
const state = (status) => unread.states.push({ id: row.id, name: row.fileName, status });
|
|
432
|
+
if (file && pendingFileNames([file]).length > 0) {
|
|
433
|
+
unread.pending.push(row.fileName);
|
|
434
|
+
state(UNREAD.reading);
|
|
435
|
+
}
|
|
436
|
+
else if (file?.state === 'failed') {
|
|
437
|
+
unread.failed.push(row.fileName);
|
|
438
|
+
const rag = file.processors.find((processor) => processor.processorId === 'rag');
|
|
439
|
+
state(file.reason === 'no readable text' ||
|
|
440
|
+
rag?.reason === 'no readable text' ||
|
|
441
|
+
(await noText(mesh, row.id))
|
|
442
|
+
? UNREAD.noText
|
|
443
|
+
: UNREAD.unreadable);
|
|
444
|
+
}
|
|
445
|
+
else {
|
|
446
|
+
unread.pending.push(row.fileName);
|
|
447
|
+
state(UNREAD.remote);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
return unread;
|
|
451
|
+
}
|
|
452
|
+
async function listFiles(mesh, { files, rows }) {
|
|
453
|
+
const details = new Map();
|
|
454
|
+
for (const row of rows) {
|
|
455
|
+
const text = await mesh.files.textOf(row.id, row.blobId);
|
|
456
|
+
if (text) {
|
|
457
|
+
details.set(row.id, readable(text));
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
const unread = await unreadFiles(mesh, {
|
|
461
|
+
files,
|
|
462
|
+
rows,
|
|
463
|
+
ids: rows.filter((row) => !details.has(row.id)).map((row) => row.id)
|
|
464
|
+
});
|
|
465
|
+
for (const { id, status } of unread.states) {
|
|
466
|
+
details.set(id, status);
|
|
467
|
+
}
|
|
468
|
+
return manifestBlock(rows.map((row) => ({ title: row.fileName, detail: details.get(row.id) ?? '' })));
|
|
469
|
+
}
|
|
470
|
+
async function noText(mesh, knowledgeId) {
|
|
471
|
+
const latest = (await listIndexingByKnowledge(mesh, { knowledgeId })).reduce((current, row) => (!current || row.at > current.at ? row : current), null);
|
|
472
|
+
return latest?.status === 'failed' && parseFailure(latest.error).reason === 'no-text';
|
|
473
|
+
}
|
|
474
|
+
function listed(items) {
|
|
475
|
+
return items?.length ? [...items] : undefined;
|
|
239
476
|
}
|
|
477
|
+
const TEXT_CHARS = 4_000_000;
|
|
478
|
+
const WHOLE_CHARS = 32_000;
|
|
479
|
+
const MAX_LIST_CONTEXT_TOKENS = 3584;
|
|
240
480
|
function toWireHit(hit) {
|
|
241
481
|
return { id: hit.id, content: hit.content, metadataJson: hit.metadataJson, score: hit.score };
|
|
242
482
|
}
|