@qvac/core 1.0.0-rc.8 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +234 -3
- package/README.md +1 -0
- package/dist-lib/lib/api/attachments.d.ts +1 -1
- package/dist-lib/lib/api/attachments.js +11 -0
- package/dist-lib/lib/api/chunks.d.ts +4 -2
- package/dist-lib/lib/api/chunks.js +6 -3
- package/dist-lib/lib/api/devices.js +6 -18
- package/dist-lib/lib/api/execution.d.ts +6 -2
- package/dist-lib/lib/api/execution.js +29 -3
- package/dist-lib/lib/api/host-run-log.d.ts +7 -1
- package/dist-lib/lib/api/host-run-log.js +3 -1
- package/dist-lib/lib/api/index.d.ts +3 -1
- package/dist-lib/lib/api/index.js +17 -6
- package/dist-lib/lib/api/members.js +13 -29
- package/dist-lib/lib/api/mesh.d.ts +2 -1
- package/dist-lib/lib/api/projects.d.ts +1 -1
- package/dist-lib/lib/api/projects.js +9 -5
- package/dist-lib/lib/api/rag-label.d.ts +4 -0
- package/dist-lib/lib/api/rag-label.js +21 -0
- package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
- package/dist-lib/lib/api/rag-lexical.js +1058 -0
- package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
- package/dist-lib/lib/api/rag-manifest.js +46 -0
- package/dist-lib/lib/api/rag-outline.d.ts +36 -0
- package/dist-lib/lib/api/rag-outline.js +426 -0
- package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
- package/dist-lib/lib/api/rag-plugin.js +70 -0
- package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
- package/dist-lib/lib/api/rag-synthesis.js +102 -0
- package/dist-lib/lib/api/rag-targets.d.ts +23 -0
- package/dist-lib/lib/api/rag-targets.js +215 -0
- package/dist-lib/lib/api/rag-window.d.ts +5 -0
- package/dist-lib/lib/api/rag-window.js +46 -0
- package/dist-lib/lib/api/rag.d.ts +3 -2
- package/dist-lib/lib/api/rag.js +421 -181
- package/dist-lib/lib/api/served.js +8 -14
- package/dist-lib/lib/api/workspace.d.ts +8 -1
- package/dist-lib/lib/api/workspace.js +27 -13
- package/dist-lib/lib/assistant/harness.d.ts +9 -1
- package/dist-lib/lib/assistant/history.d.ts +11 -2
- package/dist-lib/lib/assistant/history.js +74 -4
- package/dist-lib/lib/assistant/media-round.d.ts +61 -0
- package/dist-lib/lib/assistant/media-round.js +111 -0
- package/dist-lib/lib/assistant/message.d.ts +7 -0
- package/dist-lib/lib/assistant/model-round.d.ts +12 -0
- package/dist-lib/lib/assistant/model-round.js +130 -17
- package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
- package/dist-lib/lib/assistant/pdf-pages.js +67 -3
- package/dist-lib/lib/assistant/round-host.d.ts +22 -0
- package/dist-lib/lib/assistant/round-host.js +142 -8
- package/dist-lib/lib/assistant/upload.d.ts +12 -0
- package/dist-lib/lib/assistant/upload.js +36 -0
- package/dist-lib/lib/assistant-client.d.ts +1 -0
- package/dist-lib/lib/assistant-client.js +40 -19
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +41 -1
- package/dist-lib/lib/assistant.d.ts +15 -0
- package/dist-lib/lib/assistant.js +474 -89
- package/dist-lib/lib/client.d.ts +1 -0
- package/dist-lib/lib/client.js +4 -1
- package/dist-lib/lib/compaction.js +2 -2
- package/dist-lib/lib/core/agent.js +12 -45
- package/dist-lib/lib/core/approval-rules.js +1 -8
- package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
- package/dist-lib/lib/core/archived-mesh.js +12 -8
- package/dist-lib/lib/core/chat-copy.d.ts +18 -3
- package/dist-lib/lib/core/chat-copy.js +74 -25
- package/dist-lib/lib/core/chat-fork.js +15 -6
- package/dist-lib/lib/core/chat-group.js +16 -24
- package/dist-lib/lib/core/chat-move.d.ts +9 -0
- package/dist-lib/lib/core/chat-move.js +94 -3
- package/dist-lib/lib/core/chat-order.js +12 -18
- package/dist-lib/lib/core/chat.js +38 -31
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +18 -25
- package/dist-lib/lib/core/collections.d.ts +2 -0
- package/dist-lib/lib/core/collections.js +2 -0
- package/dist-lib/lib/core/contract/activity.d.ts +4 -0
- package/dist-lib/lib/core/contract/activity.js +34 -4
- package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
- package/dist-lib/lib/core/contract/chat-group.js +21 -1
- package/dist-lib/lib/core/contract/chat.d.ts +9 -0
- package/dist-lib/lib/core/contract/chat.js +127 -5
- package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
- package/dist-lib/lib/core/contract/chunks.js +37 -6
- package/dist-lib/lib/core/contract/features.d.ts +21 -0
- package/dist-lib/lib/core/contract/features.js +22 -0
- package/dist-lib/lib/core/contract/index.js +2 -0
- package/dist-lib/lib/core/contract/indexing.js +4 -2
- package/dist-lib/lib/core/contract/knowledge.js +5 -4
- package/dist-lib/lib/core/contract/members.js +16 -10
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
- package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
- package/dist-lib/lib/core/contract/projects.d.ts +5 -0
- package/dist-lib/lib/core/contract/projects.js +13 -5
- package/dist-lib/lib/core/contract/roles.js +32 -4
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
- package/dist-lib/lib/core/contract/run-requests.js +35 -2
- package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
- package/dist-lib/lib/core/contract/tool-configs.js +3 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
- package/dist-lib/lib/core/contract/workspace.js +40 -7
- package/dist-lib/lib/core/contract/writers.js +2 -2
- package/dist-lib/lib/core/departures.d.ts +3 -1
- package/dist-lib/lib/core/departures.js +3 -0
- package/dist-lib/lib/core/devices.js +1 -8
- package/dist-lib/lib/core/files/attachments.d.ts +2 -0
- package/dist-lib/lib/core/files/attachments.js +19 -1
- package/dist-lib/lib/core/files/index.d.ts +20 -4
- package/dist-lib/lib/core/files/index.js +132 -21
- package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
- package/dist-lib/lib/core/files/processing/document.js +7 -0
- package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
- package/dist-lib/lib/core/files/processing/index.js +144 -12
- package/dist-lib/lib/core/files/processing/protocol.js +13 -34
- package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
- package/dist-lib/lib/core/files/rag/build.js +123 -19
- package/dist-lib/lib/core/files/rag/failure.js +2 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
- package/dist-lib/lib/core/files/rag/index.js +13 -1
- package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
- package/dist-lib/lib/core/files/rag/load.js +27 -6
- package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
- package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
- package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/produce.js +104 -16
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
- package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
- package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
- package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
- package/dist-lib/lib/core/files/records/knowledge.js +7 -40
- package/dist-lib/lib/core/files/records/reclaim.js +5 -10
- package/dist-lib/lib/core/files/upload.js +68 -41
- package/dist-lib/lib/core/fs-drives.js +14 -26
- package/dist-lib/lib/core/invite.d.ts +1 -0
- package/dist-lib/lib/core/invite.js +4 -2
- package/dist-lib/lib/core/local/files.d.ts +1 -1
- package/dist-lib/lib/core/local/files.js +5 -5
- package/dist-lib/lib/core/local/index.d.ts +13 -6
- package/dist-lib/lib/core/local/index.js +103 -69
- package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
- package/dist-lib/lib/core/local/run-logs.js +5 -5
- package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
- package/dist-lib/lib/core/mesh-processor.js +87 -0
- package/dist-lib/lib/core/mesh.d.ts +48 -5
- package/dist-lib/lib/core/mesh.js +318 -129
- package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
- package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
- package/dist-lib/lib/core/migrations/writer-links.js +32 -0
- package/dist-lib/lib/core/project-keys.d.ts +6 -0
- package/dist-lib/lib/core/project-keys.js +11 -0
- package/dist-lib/lib/core/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/public-mesh.js +10 -8
- package/dist-lib/lib/core/run-requests.d.ts +3 -0
- package/dist-lib/lib/core/run-requests.js +21 -1
- package/dist-lib/lib/core/skill-availability.d.ts +33 -0
- package/dist-lib/lib/core/skill-availability.js +67 -0
- package/dist-lib/lib/core/sync/background.d.ts +0 -1
- package/dist-lib/lib/core/sync/background.js +2 -1
- package/dist-lib/lib/core/sync/changed.d.ts +4 -2
- package/dist-lib/lib/core/sync/changed.js +1 -1
- package/dist-lib/lib/core/sync/index.js +52 -10
- package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
- package/dist-lib/lib/core/sync/view-updates.js +82 -91
- package/dist-lib/lib/core/sync-status.js +97 -41
- package/dist-lib/lib/core/tool-activity.js +6 -12
- package/dist-lib/lib/core/tool-configs.js +17 -29
- package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
- package/dist-lib/lib/core/upgrade-summary.js +67 -0
- package/dist-lib/lib/core/view-read.d.ts +4 -0
- package/dist-lib/lib/core/view-read.js +23 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
- package/dist-lib/lib/core/workspace-upgrade.js +27 -16
- package/dist-lib/lib/core.d.ts +39 -42
- package/dist-lib/lib/core.js +661 -639
- package/dist-lib/lib/errors.d.ts +20 -0
- package/dist-lib/lib/errors.js +20 -0
- package/dist-lib/lib/llm-model-names.d.ts +1 -1
- package/dist-lib/lib/llm-model-names.js +1 -1
- package/dist-lib/lib/mesh-scoped-engine.js +4 -0
- package/dist-lib/lib/model-context.js +1 -2
- package/dist-lib/lib/probe-rounds.d.ts +7 -2
- package/dist-lib/lib/probe-rounds.js +6 -1
- package/dist-lib/lib/run-transcript.d.ts +4 -1
- package/dist-lib/lib/run-transcript.js +15 -6
- package/dist-lib/schema/engine.js +83 -6
- package/dist-lib/schema/indexes.d.ts +3 -0
- package/dist-lib/schema/indexes.js +5 -0
- package/dist-lib/schema/local.js +11 -2
- package/dist-lib/schema/qvac.js +49 -5
- package/dist-lib/spec/bind.js +34 -0
- package/dist-lib/spec/calls.js +5 -0
- package/dist-lib/spec/capabilities.d.ts +17 -0
- package/dist-lib/spec/hrpc/hrpc.json +65 -0
- package/dist-lib/spec/hrpc/index.js +68 -5
- package/dist-lib/spec/hrpc/messages.js +260 -23
- package/dist-lib/spec/hyperschema/index.js +260 -23
- package/dist-lib/spec/hyperschema/schema.json +181 -0
- package/dist-lib/spec/hyperschema/types.d.ts +48 -0
- package/dist-lib/spec/local/hyperdb/messages.js +270 -224
- package/dist-lib/spec/local/hyperschema/index.js +73 -38
- package/dist-lib/spec/local/hyperschema/schema.json +31 -0
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
- package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
- package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
- package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
- package/package.json +75 -70
- package/tui/README.md +2 -1
- package/tui/app.mjs +28 -7
- package/tui/boot.mjs +5 -1
- package/tui/lib/attachment-files.mjs +82 -0
- package/tui/lib/reopen-meshes.mjs +15 -0
- package/tui/main.mjs +4 -43
- package/worker/assistant-sidecar.mjs +10 -3
- package/worker/stow-entry.mjs +3 -52
|
@@ -2,7 +2,8 @@ import type { LoggerInterface } from '@qvac/logging';
|
|
|
2
2
|
import ScopeLock from 'scope-lock';
|
|
3
3
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
4
4
|
import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
5
|
-
import { type
|
|
5
|
+
import { type VectorIndex } from 'rag-extract';
|
|
6
|
+
import type { KnowledgeExtractors } from './build.ts';
|
|
6
7
|
import { type IndexingEmbedder } from './produce.ts';
|
|
7
8
|
import { type IndexingDevice } from './policy.ts';
|
|
8
9
|
import type { FileDocument, FileProcessor, ProcessorHost } from '../processing/index.ts';
|
|
@@ -11,34 +12,41 @@ export interface RagProcessorOptions {
|
|
|
11
12
|
readonly model: string;
|
|
12
13
|
readonly embedder?: IndexingEmbedder;
|
|
13
14
|
readonly index?: VectorIndex | null;
|
|
14
|
-
readonly extractors?:
|
|
15
|
+
readonly extractors?: KnowledgeExtractors;
|
|
15
16
|
readonly batches?: ScopeLock;
|
|
16
17
|
readonly device?: IndexingDevice;
|
|
18
|
+
readonly deferChatVectors?: boolean;
|
|
17
19
|
readonly log: LoggerInterface | null;
|
|
18
20
|
}
|
|
19
21
|
export declare class RagProcessor implements FileProcessor {
|
|
20
22
|
readonly id = "rag";
|
|
21
23
|
readonly input = "knowledge";
|
|
22
24
|
readonly output = "blob";
|
|
25
|
+
readonly deferChatVectors: boolean;
|
|
23
26
|
readonly version: string;
|
|
24
27
|
private readonly _producer;
|
|
25
28
|
private readonly _device?;
|
|
26
29
|
readonly load?: (document: FileDocument, signal: AbortSignal) => Promise<void>;
|
|
30
|
+
readonly allLoaded?: () => Promise<void>;
|
|
27
31
|
private readonly _index;
|
|
28
32
|
private readonly _log;
|
|
29
33
|
constructor(options: RagProcessorOptions);
|
|
30
34
|
applies(_document: QvacKnowledge): boolean;
|
|
31
35
|
canProduce(document: QvacKnowledge, host: ProcessorHost): Promise<boolean>;
|
|
36
|
+
text(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
32
37
|
produce(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
33
38
|
private _settled;
|
|
34
39
|
evict(loaded: LoadedArtifact): Promise<void>;
|
|
35
40
|
close(): Promise<boolean>;
|
|
36
41
|
}
|
|
37
42
|
export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
|
|
43
|
+
export { PluginVectorIndex, knowledgeScope, pluginArtifactId } from './plugin-index.ts';
|
|
44
|
+
export type { PluginHit } from './plugin-index.ts';
|
|
38
45
|
export { joinedIndex } from './joined-index.ts';
|
|
39
46
|
export { canIndexKnowledge } from './policy.ts';
|
|
40
47
|
export type { IndexingDevice } from './policy.ts';
|
|
41
48
|
export type { IndexingEmbedder, IndexingProgress } from './produce.ts';
|
|
42
49
|
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.ts';
|
|
43
|
-
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
50
|
+
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, resolveTargetDocument, retrieveRagKnowledge, searchRagChunks, selectCoverage, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
51
|
+
export type { KnowledgeExtractors, ReadPdfPages } from './build.ts';
|
|
44
52
|
export type { RagArtifactPayload, RagDocumentRef, RagExtractors, RagSource, VectorHit, VectorIndex } from 'rag-extract';
|
|
@@ -3,15 +3,18 @@ import { pipelineVersionFor } from 'rag-extract';
|
|
|
3
3
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
4
4
|
import { IndexingPass } from './produce.js';
|
|
5
5
|
import { HydratePass } from './load.js';
|
|
6
|
+
import { PluginVectorIndex } from './plugin-index.js';
|
|
6
7
|
import { canIndexKnowledge } from './policy.js';
|
|
7
8
|
export class RagProcessor {
|
|
8
9
|
id = 'rag';
|
|
9
10
|
input = 'knowledge';
|
|
10
11
|
output = 'blob';
|
|
12
|
+
deferChatVectors;
|
|
11
13
|
version;
|
|
12
14
|
_producer;
|
|
13
15
|
_device;
|
|
14
16
|
load;
|
|
17
|
+
allLoaded;
|
|
15
18
|
_index;
|
|
16
19
|
_log;
|
|
17
20
|
constructor(options) {
|
|
@@ -19,6 +22,7 @@ export class RagProcessor {
|
|
|
19
22
|
this._index = options.index ?? null;
|
|
20
23
|
this._log = options.log;
|
|
21
24
|
this._device = options.device;
|
|
25
|
+
this.deferChatVectors = options.deferChatVectors ?? true;
|
|
22
26
|
const lock = options.batches ?? new ScopeLock();
|
|
23
27
|
const source = options.embedder;
|
|
24
28
|
const embedder = source && {
|
|
@@ -44,6 +48,10 @@ export class RagProcessor {
|
|
|
44
48
|
const loader = new HydratePass({ ...options, index: this._index, embedder });
|
|
45
49
|
this.load = (document, signal) => this._settled(document, loader.run(document, signal));
|
|
46
50
|
}
|
|
51
|
+
if (this._index instanceof PluginVectorIndex) {
|
|
52
|
+
const index = this._index;
|
|
53
|
+
this.allLoaded = () => index.sweep();
|
|
54
|
+
}
|
|
47
55
|
}
|
|
48
56
|
applies(_document) {
|
|
49
57
|
return true;
|
|
@@ -57,6 +65,9 @@ export class RagProcessor {
|
|
|
57
65
|
device: host.device
|
|
58
66
|
})));
|
|
59
67
|
}
|
|
68
|
+
async text(document, signal) {
|
|
69
|
+
await this._producer?.extract(document, signal);
|
|
70
|
+
}
|
|
60
71
|
async produce(document, signal) {
|
|
61
72
|
await this._settled(document, this._producer?.run(document, signal));
|
|
62
73
|
}
|
|
@@ -88,7 +99,8 @@ export class RagProcessor {
|
|
|
88
99
|
}
|
|
89
100
|
}
|
|
90
101
|
export { MAX_KNOWLEDGE_BYTES } from './limits.js';
|
|
102
|
+
export { PluginVectorIndex, knowledgeScope, pluginArtifactId } from './plugin-index.js';
|
|
91
103
|
export { joinedIndex } from './joined-index.js';
|
|
92
104
|
export { canIndexKnowledge } from './policy.js';
|
|
93
105
|
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.js';
|
|
94
|
-
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
106
|
+
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, resolveTargetDocument, retrieveRagKnowledge, searchRagChunks, selectCoverage, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
2
|
-
import { type
|
|
2
|
+
import { type VectorIndex } from 'rag-extract';
|
|
3
3
|
import type { FileDocument } from '../processing/index.ts';
|
|
4
4
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
5
|
-
import { type BuildEmbedder } from './build.ts';
|
|
5
|
+
import { type BuildEmbedder, type KnowledgeExtractors } from './build.ts';
|
|
6
6
|
export interface HydratePassOptions {
|
|
7
7
|
model: string;
|
|
8
8
|
index: VectorIndex;
|
|
9
9
|
embedder?: BuildEmbedder;
|
|
10
|
-
extractors?:
|
|
10
|
+
extractors?: KnowledgeExtractors;
|
|
11
11
|
}
|
|
12
12
|
export interface PickArtifactOptions {
|
|
13
13
|
readonly model: string;
|
|
@@ -22,7 +22,10 @@ export declare class HydratePass {
|
|
|
22
22
|
private readonly _index;
|
|
23
23
|
private readonly _embedder;
|
|
24
24
|
private readonly _extractors;
|
|
25
|
+
private _rebuilding;
|
|
26
|
+
private readonly _rebuildWaiters;
|
|
25
27
|
constructor(opts: HydratePassOptions);
|
|
26
28
|
run(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
27
29
|
private _rebuild;
|
|
30
|
+
private _rebuildOne;
|
|
28
31
|
}
|
|
@@ -21,6 +21,8 @@ export class HydratePass {
|
|
|
21
21
|
_index;
|
|
22
22
|
_embedder;
|
|
23
23
|
_extractors;
|
|
24
|
+
_rebuilding = false;
|
|
25
|
+
_rebuildWaiters = new Map();
|
|
24
26
|
constructor(opts) {
|
|
25
27
|
this._model = opts.model;
|
|
26
28
|
this._index = opts.index;
|
|
@@ -165,13 +167,32 @@ export class HydratePass {
|
|
|
165
167
|
}
|
|
166
168
|
}
|
|
167
169
|
async _rebuild(document, artifact, signal) {
|
|
170
|
+
const embedder = this._embedder;
|
|
171
|
+
if (!embedder || (document.artifact(artifact.id).rebuildFailures ?? 0) > EMBED_RETRY_LIMIT) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
if (this._rebuilding) {
|
|
175
|
+
this._rebuildWaiters.set(document.record.id, () => document.schedule(Date.now()));
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
this._rebuilding = true;
|
|
179
|
+
document.admit();
|
|
180
|
+
try {
|
|
181
|
+
await this._rebuildOne(document, { artifact, embedder, signal });
|
|
182
|
+
}
|
|
183
|
+
finally {
|
|
184
|
+
this._rebuilding = false;
|
|
185
|
+
const waiters = [...this._rebuildWaiters.values()];
|
|
186
|
+
this._rebuildWaiters.clear();
|
|
187
|
+
for (const wake of waiters) {
|
|
188
|
+
wake();
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
async _rebuildOne(document, { artifact, embedder, signal }) {
|
|
168
193
|
const knowledge = document.record;
|
|
169
194
|
const local = document.artifact(artifact.id);
|
|
170
|
-
if (
|
|
171
|
-
(local.rebuildFailures ?? 0) > EMBED_RETRY_LIMIT ||
|
|
172
|
-
document.paused() ||
|
|
173
|
-
document.closed() ||
|
|
174
|
-
signal.aborted) {
|
|
195
|
+
if (document.paused() || document.closed() || signal.aborted) {
|
|
175
196
|
return;
|
|
176
197
|
}
|
|
177
198
|
const retryAt = document.output(knowledge.id, knowledge.blobId).blockedUntil();
|
|
@@ -200,7 +221,7 @@ export class HydratePass {
|
|
|
200
221
|
}
|
|
201
222
|
document.load({ state: 'running', reason: 'rebuilding local index' });
|
|
202
223
|
const payload = await buildArtifactPayload({
|
|
203
|
-
embedder
|
|
224
|
+
embedder,
|
|
204
225
|
extractors: this._extractors,
|
|
205
226
|
knowledge,
|
|
206
227
|
artifactId: artifact.id,
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { KnowledgeQuery, KnowledgeService } from '@qvac/harness/lib/contract/knowledge.ts';
|
|
2
|
+
import type { VectorChunk, VectorHit, VectorIndex, VectorIndexInfo } from 'rag-extract';
|
|
3
|
+
export interface PluginHit extends VectorHit {
|
|
4
|
+
readonly artifactId: string;
|
|
5
|
+
}
|
|
6
|
+
export interface PluginIndexOptions {
|
|
7
|
+
readonly mesh: string;
|
|
8
|
+
readonly model: string;
|
|
9
|
+
}
|
|
10
|
+
export declare function knowledgeScope(mesh: string, chatId: string | null | undefined): string;
|
|
11
|
+
export declare function pluginArtifactId(mesh: string, artifactId: string): string;
|
|
12
|
+
export declare class PluginVectorIndex implements VectorIndex {
|
|
13
|
+
private readonly _knowledge;
|
|
14
|
+
private readonly _mesh;
|
|
15
|
+
private readonly _model;
|
|
16
|
+
private readonly _chunks;
|
|
17
|
+
private _dimensions;
|
|
18
|
+
private _listed;
|
|
19
|
+
private readonly _fed;
|
|
20
|
+
constructor(knowledge: KnowledgeService, { mesh, model }: PluginIndexOptions);
|
|
21
|
+
info(): VectorIndexInfo;
|
|
22
|
+
upsert(artifactId: string, chunks: ReadonlyArray<VectorChunk>): Promise<void>;
|
|
23
|
+
remove(artifactId: string): Promise<void>;
|
|
24
|
+
sweep(): Promise<void>;
|
|
25
|
+
scope(chatId: string | null): string;
|
|
26
|
+
pluginId(artifactId: string): string;
|
|
27
|
+
chunksOf(artifactId: string): number;
|
|
28
|
+
find(query: Omit<KnowledgeQuery, 'signal'>): Promise<PluginHit[]>;
|
|
29
|
+
search(): Promise<ReadonlyArray<VectorHit>>;
|
|
30
|
+
clear(): Promise<void>;
|
|
31
|
+
private _listedVersions;
|
|
32
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import { contentHash } from 'rag-extract/artifact';
|
|
3
|
+
import { isRecord } from '../../../guards.js';
|
|
4
|
+
export function knowledgeScope(mesh, chatId) {
|
|
5
|
+
return chatId ? `chat:${mesh}:${chatId}` : `kb:${mesh}`;
|
|
6
|
+
}
|
|
7
|
+
export function pluginArtifactId(mesh, artifactId) {
|
|
8
|
+
return `${mesh}/${artifactId}`;
|
|
9
|
+
}
|
|
10
|
+
export class PluginVectorIndex {
|
|
11
|
+
_knowledge;
|
|
12
|
+
_mesh;
|
|
13
|
+
_model;
|
|
14
|
+
_chunks = new Map();
|
|
15
|
+
_dimensions = null;
|
|
16
|
+
_listed = null;
|
|
17
|
+
_fed = new Set();
|
|
18
|
+
constructor(knowledge, { mesh, model }) {
|
|
19
|
+
this._knowledge = knowledge;
|
|
20
|
+
this._mesh = mesh;
|
|
21
|
+
this._model = model;
|
|
22
|
+
}
|
|
23
|
+
info() {
|
|
24
|
+
const chunks = [...this._chunks.values()].reduce((sum, count) => sum + count, 0);
|
|
25
|
+
return { artifacts: this._chunks.size, chunks, dimensions: this._dimensions };
|
|
26
|
+
}
|
|
27
|
+
async upsert(artifactId, chunks) {
|
|
28
|
+
const dimensions = chunks[0]?.embedding.length ?? 0;
|
|
29
|
+
const id = pluginArtifactId(this._mesh, artifactId);
|
|
30
|
+
const version = payloadVersion(chunks);
|
|
31
|
+
this._fed.add(id);
|
|
32
|
+
const held = await this._listedVersions();
|
|
33
|
+
if (held.get(id) !== version) {
|
|
34
|
+
await this._knowledge.upsert({
|
|
35
|
+
artifactId: id,
|
|
36
|
+
version,
|
|
37
|
+
scope: knowledgeScope(this._mesh, chatIdOf(chunks[0])),
|
|
38
|
+
chunks: chunks.map(toKnowledgeChunk),
|
|
39
|
+
embedding: {
|
|
40
|
+
model: this._model,
|
|
41
|
+
dimensions,
|
|
42
|
+
vectors: chunks.map((chunk) => [...chunk.embedding])
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
held.set(id, version);
|
|
46
|
+
}
|
|
47
|
+
this._chunks.set(artifactId, chunks.length);
|
|
48
|
+
this._dimensions ??= dimensions || null;
|
|
49
|
+
}
|
|
50
|
+
async remove(artifactId) {
|
|
51
|
+
const id = pluginArtifactId(this._mesh, artifactId);
|
|
52
|
+
this._fed.delete(id);
|
|
53
|
+
await this._knowledge.remove(id);
|
|
54
|
+
(await this._listedVersions()).delete(id);
|
|
55
|
+
this._chunks.delete(artifactId);
|
|
56
|
+
}
|
|
57
|
+
async sweep() {
|
|
58
|
+
const held = await this._listedVersions();
|
|
59
|
+
for (const id of [...held.keys()]) {
|
|
60
|
+
if (!id.startsWith(`${this._mesh}/`) || this._fed.has(id))
|
|
61
|
+
continue;
|
|
62
|
+
held.delete(id);
|
|
63
|
+
await this._knowledge.remove(id);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
scope(chatId) {
|
|
67
|
+
return knowledgeScope(this._mesh, chatId);
|
|
68
|
+
}
|
|
69
|
+
pluginId(artifactId) {
|
|
70
|
+
return pluginArtifactId(this._mesh, artifactId);
|
|
71
|
+
}
|
|
72
|
+
chunksOf(artifactId) {
|
|
73
|
+
return this._chunks.get(artifactId) ?? 0;
|
|
74
|
+
}
|
|
75
|
+
async find(query) {
|
|
76
|
+
const hits = await this._knowledge.search(query);
|
|
77
|
+
return hits.map((hit) => ({
|
|
78
|
+
artifactId: hit.artifactId,
|
|
79
|
+
id: hit.chunkId,
|
|
80
|
+
content: hit.text,
|
|
81
|
+
metadataJson: JSON.stringify(hit.metadata ?? {}),
|
|
82
|
+
score: hit.score
|
|
83
|
+
}));
|
|
84
|
+
}
|
|
85
|
+
search() {
|
|
86
|
+
return Promise.reject(new Error('the knowledge plugin is searched by scope, not by vector'));
|
|
87
|
+
}
|
|
88
|
+
async clear() {
|
|
89
|
+
this._chunks.clear();
|
|
90
|
+
}
|
|
91
|
+
_listedVersions() {
|
|
92
|
+
if (this._listed) {
|
|
93
|
+
return this._listed;
|
|
94
|
+
}
|
|
95
|
+
const listed = this._knowledge
|
|
96
|
+
.status()
|
|
97
|
+
.then(({ artifacts }) => new Map(artifacts.map((artifact) => [artifact.artifactId, artifact.version])));
|
|
98
|
+
listed.catch(() => {
|
|
99
|
+
this._listed = null;
|
|
100
|
+
});
|
|
101
|
+
this._listed = listed;
|
|
102
|
+
return listed;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
function payloadVersion(chunks) {
|
|
106
|
+
const payload = chunks.map((chunk) => [chunk.id, chunk.content, chunk.metadataJson]);
|
|
107
|
+
return contentHash(b4a.from(JSON.stringify(payload)));
|
|
108
|
+
}
|
|
109
|
+
function chatIdOf(chunk) {
|
|
110
|
+
const metadata = metadataOf(chunk);
|
|
111
|
+
return typeof metadata.chatId === 'string' ? metadata.chatId : null;
|
|
112
|
+
}
|
|
113
|
+
function toKnowledgeChunk(chunk) {
|
|
114
|
+
return { id: chunk.id, text: chunk.content, metadata: metadataOf(chunk) };
|
|
115
|
+
}
|
|
116
|
+
function metadataOf(chunk) {
|
|
117
|
+
if (!chunk?.metadataJson) {
|
|
118
|
+
return {};
|
|
119
|
+
}
|
|
120
|
+
try {
|
|
121
|
+
const parsed = JSON.parse(chunk.metadataJson);
|
|
122
|
+
return isRecord(parsed) ? parsed : {};
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
return {};
|
|
126
|
+
}
|
|
127
|
+
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
-
import { type
|
|
3
|
-
import { type BuildEmbedder } from './build.ts';
|
|
2
|
+
import { type BuildEmbedder, type KnowledgeExtractors } from './build.ts';
|
|
4
3
|
import { type FileDocument } from '../processing/index.ts';
|
|
5
4
|
export interface IndexingEmbedder extends BuildEmbedder {
|
|
6
5
|
}
|
|
@@ -16,6 +15,9 @@ export declare const EMBED_RETRY_LIMIT = 5;
|
|
|
16
15
|
export declare class IndexingPass {
|
|
17
16
|
private readonly _embedder;
|
|
18
17
|
private readonly _extractors;
|
|
19
|
-
constructor(embedder: IndexingEmbedder, extractors?:
|
|
18
|
+
constructor(embedder: IndexingEmbedder, extractors?: KnowledgeExtractors);
|
|
19
|
+
extract(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
20
|
+
private _bytes;
|
|
21
|
+
private _identity;
|
|
20
22
|
run(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
21
23
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { classifyRagDocument, contentHash, documentIndexArtifactId, encodeRagArtifact, pipelineVersionFor, RAG_DOCUMENT_KIND } from 'rag-extract';
|
|
2
|
-
import { buildArtifactPayload, EmbedderFailure, yieldTurn } from './build.js';
|
|
2
|
+
import { buildArtifactPayload, EmbedderFailure, prepareText, yieldTurn } from './build.js';
|
|
3
3
|
import { isClaimLive } from '../processing/index.js';
|
|
4
4
|
import { ClaimLostError } from '../processing/claims.js';
|
|
5
5
|
import { contentFailure, failureDetail, failureError, isContentFailure } from './failure.js';
|
|
@@ -12,6 +12,75 @@ export class IndexingPass {
|
|
|
12
12
|
this._embedder = embedder;
|
|
13
13
|
this._extractors = extractors;
|
|
14
14
|
}
|
|
15
|
+
async extract(document, signal) {
|
|
16
|
+
const knowledge = document.record;
|
|
17
|
+
if (!knowledge.chatId || document.state.artifactIdMemo === UNSUPPORTED) {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
const kept = await document.keptText();
|
|
21
|
+
if (kept) {
|
|
22
|
+
if (!kept.content.trim()) {
|
|
23
|
+
unreadable(document, { state: 'failed', reason: 'no readable text' });
|
|
24
|
+
}
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const bytes = await this._bytes(document, signal);
|
|
28
|
+
if (!bytes) {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
const { kind, refusal } = classify(knowledge, bytes);
|
|
32
|
+
if (refusal) {
|
|
33
|
+
refuse(document, refusal);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
if (kind === RAG_DOCUMENT_KIND.office && !this._extractors.office) {
|
|
37
|
+
unreadable(document, { state: 'failed', reason: 'extractor unavailable on this device' });
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const { artifactId } = this._identity(knowledge, bytes);
|
|
41
|
+
const prepared = await prepareText({
|
|
42
|
+
embedder: this._embedder,
|
|
43
|
+
extractors: this._extractors,
|
|
44
|
+
knowledge,
|
|
45
|
+
artifactId,
|
|
46
|
+
bytes,
|
|
47
|
+
signal
|
|
48
|
+
});
|
|
49
|
+
if (prepared.content.trim()) {
|
|
50
|
+
await document.keepText({ ...prepared, artifactId });
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
unreadable(document, { state: 'failed', reason: 'no readable text' });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
async _bytes(document, signal) {
|
|
57
|
+
const artifacts = await document.artifacts();
|
|
58
|
+
if (artifacts.some((row) => row.embedderVersion === pipelineVersionFor(this._embedder.model)) ||
|
|
59
|
+
document.source.blockedUntil() !== null ||
|
|
60
|
+
document.paused() ||
|
|
61
|
+
document.closed()) {
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
const read = await document.source.read(signal);
|
|
65
|
+
if (read.kind === 'malformed') {
|
|
66
|
+
unreadable(document, { state: 'failed', reason: 'invalid source reference' });
|
|
67
|
+
}
|
|
68
|
+
else if (read.kind === 'missing') {
|
|
69
|
+
document.schedule(read.retryAt);
|
|
70
|
+
document.work({ state: 'pending', reason: 'source unavailable', retryAt: read.retryAt });
|
|
71
|
+
}
|
|
72
|
+
return read.kind === 'bytes' ? read.bytes : null;
|
|
73
|
+
}
|
|
74
|
+
_identity(knowledge, bytes) {
|
|
75
|
+
const sourceHash = contentHash(bytes);
|
|
76
|
+
const artifactId = documentIndexArtifactId({
|
|
77
|
+
attachmentId: knowledge.id,
|
|
78
|
+
sourceBlobId: knowledge.blobId,
|
|
79
|
+
sourceHash,
|
|
80
|
+
embeddingModel: this._embedder.model
|
|
81
|
+
});
|
|
82
|
+
return { sourceHash, artifactId };
|
|
83
|
+
}
|
|
15
84
|
async run(document, signal) {
|
|
16
85
|
const knowledge = document.record;
|
|
17
86
|
const pipelineVersion = pipelineVersionFor(this._embedder.model);
|
|
@@ -68,14 +137,7 @@ export class IndexingPass {
|
|
|
68
137
|
return;
|
|
69
138
|
}
|
|
70
139
|
const bytes = read.bytes;
|
|
71
|
-
const kind =
|
|
72
|
-
fileName: knowledge.fileName,
|
|
73
|
-
mimeType: knowledge.mimeType,
|
|
74
|
-
bytes
|
|
75
|
-
});
|
|
76
|
-
const refusal = contentFailure(knowledge.fileName, bytes, {
|
|
77
|
-
supported: kind !== RAG_DOCUMENT_KIND.unsupported
|
|
78
|
-
});
|
|
140
|
+
const { kind, refusal } = classify(knowledge, bytes);
|
|
79
141
|
if (refusal?.reason === 'unsupported' && knowledge.chatId) {
|
|
80
142
|
local.artifactIdMemo = UNSUPPORTED;
|
|
81
143
|
document.work({
|
|
@@ -98,13 +160,7 @@ export class IndexingPass {
|
|
|
98
160
|
});
|
|
99
161
|
return;
|
|
100
162
|
}
|
|
101
|
-
const sourceHash =
|
|
102
|
-
const artifactId = documentIndexArtifactId({
|
|
103
|
-
attachmentId: knowledge.id,
|
|
104
|
-
sourceBlobId: knowledge.blobId,
|
|
105
|
-
sourceHash,
|
|
106
|
-
embeddingModel: this._embedder.model
|
|
107
|
-
});
|
|
163
|
+
const { sourceHash, artifactId } = this._identity(knowledge, bytes);
|
|
108
164
|
local.artifactIdMemo = artifactId;
|
|
109
165
|
if (await document.claims.blockedByStatus(artifactId)) {
|
|
110
166
|
return;
|
|
@@ -127,6 +183,7 @@ export class IndexingPass {
|
|
|
127
183
|
if (refusal) {
|
|
128
184
|
throw refusal;
|
|
129
185
|
}
|
|
186
|
+
const kept = await document.keptText(artifactId);
|
|
130
187
|
const payload = await buildArtifactPayload({
|
|
131
188
|
embedder: this._embedder,
|
|
132
189
|
extractors: this._extractors,
|
|
@@ -134,11 +191,13 @@ export class IndexingPass {
|
|
|
134
191
|
artifactId,
|
|
135
192
|
bytes,
|
|
136
193
|
signal,
|
|
194
|
+
kept,
|
|
137
195
|
onExtracting: (pages) => lease.keep(pages === 1),
|
|
138
196
|
onExtracted: async () => {
|
|
139
197
|
document.admit();
|
|
140
198
|
await lease.keep(true);
|
|
141
199
|
},
|
|
200
|
+
onChunked: kept ? undefined : (prepared) => document.keepText({ ...prepared, artifactId }),
|
|
142
201
|
onProgress: async (processedChunks, totalChunks) => {
|
|
143
202
|
lastTotal = totalChunks;
|
|
144
203
|
document.report({
|
|
@@ -186,6 +245,7 @@ export class IndexingPass {
|
|
|
186
245
|
});
|
|
187
246
|
}
|
|
188
247
|
catch (err) {
|
|
248
|
+
local.prepared = undefined;
|
|
189
249
|
if (document.closed()) {
|
|
190
250
|
return;
|
|
191
251
|
}
|
|
@@ -237,3 +297,31 @@ export class IndexingPass {
|
|
|
237
297
|
}
|
|
238
298
|
}
|
|
239
299
|
}
|
|
300
|
+
function classify(knowledge, bytes) {
|
|
301
|
+
const kind = classifyRagDocument({
|
|
302
|
+
fileName: knowledge.fileName,
|
|
303
|
+
mimeType: knowledge.mimeType,
|
|
304
|
+
bytes
|
|
305
|
+
});
|
|
306
|
+
const refusal = contentFailure(knowledge.fileName, bytes, {
|
|
307
|
+
supported: kind !== RAG_DOCUMENT_KIND.unsupported
|
|
308
|
+
});
|
|
309
|
+
return { kind, refusal };
|
|
310
|
+
}
|
|
311
|
+
function unreadable(document, work) {
|
|
312
|
+
document.state.artifactIdMemo = UNSUPPORTED;
|
|
313
|
+
document.work(work);
|
|
314
|
+
}
|
|
315
|
+
function refuse(document, refusal) {
|
|
316
|
+
if (refusal.reason !== 'unsupported') {
|
|
317
|
+
unreadable(document, { state: 'failed', reason: failureDetail(refusal) });
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
unreadable(document, { state: 'not-applicable', reason: 'unsupported file type' });
|
|
321
|
+
document.report({
|
|
322
|
+
stage: 'failed',
|
|
323
|
+
processedChunks: 0,
|
|
324
|
+
totalChunks: 0,
|
|
325
|
+
error: `${document.record.fileName} is not a supported Knowledge file.`
|
|
326
|
+
});
|
|
327
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { QvacKnowledge } from '../../../../../spec/mesh/hyperschema/types.d.ts';
|
|
2
2
|
import type { Mesh } from '../../../mesh.ts';
|
|
3
|
+
import type { KnowledgeExtractors } from '../build.ts';
|
|
3
4
|
import { type VectorHit } from 'rag-extract';
|
|
4
5
|
export declare function fullTextBudgetChars(ctxSize: number): number;
|
|
5
6
|
export declare function retrievalBudgetTokens(ctxSize: number): number | undefined;
|
|
@@ -22,5 +23,6 @@ export declare function formatFullDocumentContext(hits: ReadonlyArray<VectorHit>
|
|
|
22
23
|
export interface FullTextOptions {
|
|
23
24
|
readonly budgetChars: number;
|
|
24
25
|
readonly head?: boolean;
|
|
26
|
+
readonly extractors?: KnowledgeExtractors | undefined;
|
|
25
27
|
}
|
|
26
|
-
export declare function fullTextHits(mesh: Mesh, rows: ReadonlyArray<QvacKnowledge>, { budgetChars, head }: FullTextOptions): Promise<VectorHit[] | null>;
|
|
28
|
+
export declare function fullTextHits(mesh: Mesh, rows: ReadonlyArray<QvacKnowledge>, { budgetChars, head, extractors }: FullTextOptions): Promise<VectorHit[] | null>;
|