@qvac/core 0.0.0 → 0.1.1

package/dist-lib/lib/core/files/rag/produce.js

@@ -0,0 +1,214 @@
+import { encodeRagArtifact } from "./artifact/codec.js";
+import { contentHash, documentIndexArtifactId, pipelineVersionFor } from "./artifact/id.js";
+import { classifyRagDocument, RAG_DOCUMENT_KIND } from "./extract/document-types.js";
+import { buildArtifactPayload, EmbedderFailure } from "./build.js";
+import { isClaimLive } from "../processing/index.js";
+import { AbortedError } from '@qvac/harness/race-abort';
+const UNSUPPORTED = 'unsupported';
+const ERROR_LIMIT = 300;
+export const EMBED_RETRY_LIMIT = 5;
+export class IndexingPass {
+    _embedder;
+    _extractors;
+    constructor(embedder, extractors = {}) {
+        this._embedder = embedder;
+        this._extractors = extractors;
+    }
+    async run(document, signal) {
+        const knowledge = document.record;
+        const pipelineVersion = pipelineVersionFor(this._embedder.model);
+        const artifacts = await document.artifacts();
+        const published = artifacts.find((row) => row.embedderVersion === pipelineVersion);
+        if (published) {
+            await document.ownership.reclaimSuperseded(published.id, published.artifactBlobId);
+            await document.claims.settle(published.id);
+            return;
+        }
+        const local = document.state;
+        const memo = local.artifactIdMemo;
+        if (memo === UNSUPPORTED)
+            return;
+        if (memo && (await document.claims.blockedByStatus(memo)))
+            return;
+        if (!memo) {
+            const statuses = await document.statuses();
+            const now = Date.now();
+            for (const row of statuses) {
+                if (isClaimLive(row, now)) {
+                    document.schedule(row.expiresAt);
+                    return;
+                }
+                if (document.claims.isFailed(row.id))
+                    return;
+            }
+        }
+        const retryAt = document.source.blockedUntil();
+        if (retryAt !== null) {
+            document.schedule(retryAt);
+            return;
+        }
+        if (document.paused() || document.closed())
+            return;
+        const read = await document.source.read(signal);
+        if (read.kind === 'malformed') {
+            local.artifactIdMemo = UNSUPPORTED;
+            document.work({ state: 'failed', reason: 'invalid source reference' });
+            return;
+        }
+        if (read.kind === 'missing') {
+            const { retryAt } = read;
+            document.schedule(retryAt);
+            document.work({
+                state: 'pending',
+                reason: 'source unavailable',
+                retryAt
+            });
+            return;
+        }
+        const bytes = read.bytes;
+        const kind = classifyRagDocument({
+            fileName: knowledge.fileName,
+            mimeType: knowledge.mimeType,
+            bytes
+        });
+        if (kind === RAG_DOCUMENT_KIND.unsupported) {
+            local.artifactIdMemo = UNSUPPORTED;
+            document.work({
+                state: 'not-applicable',
+                reason: 'unsupported file type'
+            });
+            document.report({
+                stage: 'failed',
+                processedChunks: 0,
+                totalChunks: 0,
+                error: `${knowledge.fileName} is not a supported Knowledge file.`
+            });
+            return;
+        }
+        if (kind === RAG_DOCUMENT_KIND.office && !this._extractors.office) {
+            local.artifactIdMemo = UNSUPPORTED;
+            document.work({
+                state: 'pending',
+                reason: 'extractor unavailable on this device'
+            });
+            return;
+        }
+        const sourceHash = contentHash(bytes);
+        const artifactId = documentIndexArtifactId({
+            attachmentId: knowledge.id,
+            sourceBlobId: knowledge.blobId,
+            sourceHash,
+            embeddingModel: this._embedder.model
+        });
+        local.artifactIdMemo = artifactId;
+        if (await document.claims.blockedByStatus(artifactId))
+            return;
+        if (document.paused() || document.closed())
+            return;
+        const claim = await document.claims.claim(artifactId);
+        if (!claim)
+            return;
+        document.work({ state: 'running' });
+        let blobRef = null;
+        let lastTotal = 0;
+        try {
+            if (signal.aborted || document.paused() || document.closed()) {
+                throw new Error('interrupted: suspend');
+            }
+            const payload = await buildArtifactPayload({
+                embedder: this._embedder,
+                extractors: this._extractors,
+                knowledge,
+                artifactId,
+                bytes,
+                signal,
+                onProgress: (processedChunks, totalChunks) => {
+                    lastTotal = totalChunks;
+                    document.report({
+                        stage: 'embedding',
+                        processedChunks,
+                        totalChunks
+                    });
+                }
+            });
+            if (signal.aborted)
+                throw new AbortedError();
+            const built = await document.write(encodeRagArtifact(payload));
+            blobRef = built.blobRef;
+            const current = await document.current();
+            if (!current || current.deletedAt || signal.aborted) {
+                throw new Error('interrupted: file no longer available');
+            }
+            const landed = await document.publish({
+                id: artifactId,
+                sourceHash,
+                embedderVersion: pipelineVersion,
+                dimensions: payload.dimensions,
+                artifactBlobId: built.artifactBlobId
+            });
+            await document.ownership.onPublished(artifactId, built.blobRef, built.artifactBlobId, landed.artifactBlobId);
+            const { applied } = await document.resolve({
+                artifactId,
+                claimId: claim.claimId,
+                status: 'ready'
+            });
+            document.source.clear();
+            local.embedFailures = undefined;
+            document.report({
+                stage: 'ready',
+                processedChunks: payload.chunks.length,
+                totalChunks: payload.chunks.length
+            });
+            logger.info('rag', applied ? 'indexed' : 'indexed, claim superseded', {
+                id: artifactId,
+                chunks: payload.chunks.length
+            });
+        }
+        catch (err) {
+            if (document.closed())
+                return;
+            const interrupted = document.paused() || signal.aborted;
+            const raw = err instanceof Error ? err.message : String(err);
+            const message = interrupted
+                ? 'interrupted: suspend'
+                : raw.length > ERROR_LIMIT
+                    ? `${raw.slice(0, ERROR_LIMIT)}...`
+                    : raw;
+            const failures = !interrupted && err instanceof EmbedderFailure
+                ? (local.embedFailures = (local.embedFailures ?? 0) + 1)
+                : 0;
+            const retrying = failures > 0 && failures <= EMBED_RETRY_LIMIT;
+            const retryAt = retrying ? document.source.miss() : undefined;
+            document.work({
+                state: interrupted ? 'pending' : 'failed',
+                reason: message,
+                retryAt
+            });
+            if (retryAt)
+                document.schedule(retryAt);
+            else if (!interrupted)
+                document.claims.markFailed(artifactId);
+            if (blobRef)
+                await document.ownership.clearUnreferenced(artifactId, blobRef);
+            try {
+                await document.resolve({
+                    artifactId,
+                    claimId: claim.claimId,
+                    status: 'failed',
+                    error: message
+                });
+            }
+            catch {
+            }
+            if (!interrupted) {
+                document.report({
+                    stage: 'failed',
+                    processedChunks: 0,
+                    totalChunks: lastTotal,
+                    error: message
+                });
+            }
+            logger.warn('rag', 'indexing failed', { id: artifactId, error: message });
+        }
+    }
+}

package/dist-lib/lib/core/files/rag/retrieval/citation-match.d.ts

@@ -0,0 +1,2 @@
+import type { RagSource } from './sources.ts';
+export declare function citedByPhraseOverlap(answer: string, records: ReadonlyArray<RagSource>): string[];

package/dist-lib/lib/core/files/rag/retrieval/citation-match.js

@@ -0,0 +1,16 @@
+import { similarity } from 'asenion';
+const MIN_PHRASE_TOKENS = 4;
+export function citedByPhraseOverlap(answer, records) {
+    const cited = [];
+    const seen = new Set();
+    for (const record of records) {
+        const key = record.title.toLowerCase();
+        if (seen.has(key))
+            continue;
+        if (similarity(answer, record.excerpt, { phrase: MIN_PHRASE_TOKENS }) === 0)
+            continue;
+        seen.add(key);
+        cited.push(record.title);
+    }
+    return cited;
+}

package/dist-lib/lib/core/files/rag/retrieval/coverage.d.ts

@@ -0,0 +1,4 @@
+import type { VectorHit } from '../vector/types.ts';
+export declare function selectCoverage(hits: ReadonlyArray<VectorHit>, { budgetTokens }?: {
+    readonly budgetTokens?: number;
+}): ReadonlyArray<VectorHit>;

package/dist-lib/lib/core/files/rag/retrieval/coverage.js

@@ -0,0 +1,48 @@
+import { parseRagMetadata } from "../chunk/metadata.js";
+import { estimateRagTokens } from "../chunk/tokenizer.js";
+const DEFAULT_COVERAGE_BUDGET_TOKENS = 3072;
+export function selectCoverage(hits, { budgetTokens = DEFAULT_COVERAGE_BUDGET_TOKENS } = {}) {
+    const documents = groupByDocument(hits);
+    if (documents.length === 0)
+        return [];
+    const perDocument = Math.max(1, Math.floor(budgetTokens / documents.length));
+    const selected = [];
+    for (const document of documents)
+        selected.push(...stride(document, perDocument));
+    return selected;
+}
+function stride(document, budgetTokens) {
+    const costs = document.chunks.map((chunk) => estimateRagTokens(chunk.hit.content));
+    const total = costs.reduce((sum, cost) => sum + cost, 0);
+    if (total <= budgetTokens)
+        return document.chunks.map((chunk) => chunk.hit);
+    const last = document.chunks.length - 1;
+    const average = Math.max(1, Math.round(total / document.chunks.length));
+    const seats = Math.min(document.chunks.length, Math.max(1, Math.floor(budgetTokens / average)));
+    const step = seats > 1 ? last / (seats - 1) : 0;
+    const taken = [];
+    let used = 0;
+    for (let seat = 0; seat < seats; seat++) {
+        const at = Math.round(seat * step);
+        const cost = costs[at] ?? 0;
+        if (used + cost > budgetTokens && taken.length > 0)
+            continue;
+        taken.push(document.chunks[at].hit);
+        used += cost;
+    }
+    return taken;
+}
+function groupByDocument(hits) {
+    const byDocument = new Map();
+    for (const hit of hits) {
+        const metadata = parseRagMetadata(hit.metadataJson);
+        const documentId = metadata?.documentId ?? hit.id;
+        const chunks = byDocument.get(documentId) ?? [];
+        chunks.push({ index: metadata?.chunkIndex ?? 0, hit });
+        byDocument.set(documentId, chunks);
+    }
+    return [...byDocument].map(([documentId, chunks]) => ({
+        documentId,
+        chunks: chunks.sort((left, right) => left.index - right.index)
+    }));
+}

package/dist-lib/lib/core/files/rag/retrieval/diversity.d.ts

@@ -0,0 +1,10 @@
+import type { VectorCandidate, VectorHit } from '../vector/types.ts';
+export type RagScopeFilters = {
+    readonly chatId?: string;
+    readonly documentIds?: ReadonlyArray<string>;
+    readonly attachmentIds?: ReadonlyArray<string>;
+    readonly liveSourceIds?: ReadonlyArray<string>;
+};
+export declare function scopeFilterFor(filters: RagScopeFilters): ((candidate: VectorCandidate) => boolean) | undefined;
+export declare function filterHits(hits: ReadonlyArray<VectorHit>, filters: RagScopeFilters): ReadonlyArray<VectorHit>;
+export declare function applyPerSourceCap(hits: ReadonlyArray<VectorHit>, limit: number): ReadonlyArray<VectorHit>;

package/dist-lib/lib/core/files/rag/retrieval/diversity.js

@@ -0,0 +1,37 @@
+import { hitMatchesFilters, parseRagMetadata } from "../chunk/metadata.js";
+export function scopeFilterFor(filters) {
+    const liveSourceIds = filters.liveSourceIds ? new Set(filters.liveSourceIds) : undefined;
+    if (!filters.chatId &&
+        !filters.documentIds?.length &&
+        !filters.attachmentIds?.length &&
+        !liveSourceIds) {
+        return undefined;
+    }
+    return (candidate) => hitMatchesFilters(candidate, filters, liveSourceIds);
+}
+export function filterHits(hits, filters) {
+    const filter = scopeFilterFor(filters);
+    return filter ? hits.filter(filter) : hits;
+}
+export function applyPerSourceCap(hits, limit) {
+    if (hits.length <= limit)
+        return hits;
+    const seen = new Set();
+    const firstPass = [];
+    const leftovers = [];
+    for (const hit of hits) {
+        const key = sourceKey(hit);
+        if (seen.has(key)) {
+            leftovers.push(hit);
+        }
+        else {
+            seen.add(key);
+            firstPass.push(hit);
+        }
+    }
+    return [...firstPass, ...leftovers].slice(0, limit);
+}
+function sourceKey(hit) {
+    const metadata = parseRagMetadata(hit.metadataJson);
+    return metadata?.documentId ?? `__chunk:${hit.id}`;
+}

package/dist-lib/lib/core/files/rag/retrieval/floor.d.ts

@@ -0,0 +1,5 @@
+import '../../../../logger.ts';
+import type { VectorHit } from '../vector/types.ts';
+export declare const RAG_MIN_SIMILARITY = 0.4;
+export declare function applySimilarityFloor(hits: ReadonlyArray<VectorHit>, targetDocumentId: string | null): ReadonlyArray<VectorHit>;
+export declare function logSimilarityScores(candidates: ReadonlyArray<VectorHit>, kept: ReadonlyArray<VectorHit>): void;

package/dist-lib/lib/core/files/rag/retrieval/floor.js

@@ -0,0 +1,21 @@
+import "../../../../logger.js";
+import { parseRagMetadata } from "../chunk/metadata.js";
+export const RAG_MIN_SIMILARITY = 0.4;
+export function applySimilarityFloor(hits, targetDocumentId) {
+    if (targetDocumentId)
+        return hits;
+    return hits.filter((hit) => hit.score >= RAG_MIN_SIMILARITY);
+}
+export function logSimilarityScores(candidates, kept) {
+    if (candidates.length === 0) {
+        logger.debug('rag', 'semantic search returned 0 hits');
+        return;
+    }
+    const keptIds = new Set(kept.map((hit) => hit.id));
+    for (const [index, hit] of candidates.entries()) {
+        const source = parseRagMetadata(hit.metadataJson)?.title ?? hit.id;
+        const percent = (hit.score * 100).toFixed(1);
+        const status = keptIds.has(hit.id) ? 'kept' : 'dropped';
+        logger.debug('rag', `similarity #${index + 1} ${percent}% (${status}) - ${source}`);
+    }
+}

package/dist-lib/lib/core/files/rag/retrieval/format.d.ts

@@ -0,0 +1,6 @@
+type FormatHit = {
+    readonly content: string;
+    readonly metadataJson: string | null;
+};
+export declare function formatRagContext(hits: ReadonlyArray<FormatHit>, availableFiles?: ReadonlyArray<string>): string;
+export {};

package/dist-lib/lib/core/files/rag/retrieval/format.js

@@ -0,0 +1,21 @@
+import { parseRagMetadata } from "../chunk/metadata.js";
+export function formatRagContext(hits, availableFiles = []) {
+    const manifest = availableFiles.length > 0 ? [`Available files in this chat: ${availableFiles.join(', ')}`] : [];
+    return [
+        'The following knowledge excerpts are the primary source for answering the user.',
+        'Each excerpt is tagged with its source file name and chunk index in a [Source N: filename, ...] label.',
+        'The app renders source attribution separately. Do not echo the [Source N: ...] labels. When attribution helps, name only the files you used naturally in prose.',
+        ...manifest,
+        'If the user references a specific file or document by name, prioritise excerpts whose source matches that file.',
+        'If the user references a file that has no excerpts below, tell the user that file is not in the retrieved knowledge - do not invent an answer from other files.',
+        'If the excerpts collectively do not contain the answer, say the knowledge base did not contain it.',
+        ...hits.flatMap((hit, index) => ['', formatHitLabel(hit, index), hit.content])
+    ].join('\n');
+}
+function formatHitLabel(hit, index) {
+    const metadata = hit.metadataJson ? parseRagMetadata(hit.metadataJson) : null;
+    const title = metadata?.title ?? 'document';
+    const chunk = metadata ? `chunk ${metadata.chunkIndex}` : `chunk ${index}`;
+    const attachment = metadata?.attachmentId ? `, attachmentId=${metadata.attachmentId}` : '';
+    return `[Source ${index + 1}: ${title}, ${chunk}${attachment}]`;
+}

package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts

@@ -0,0 +1,22 @@
+import type { QvacKnowledge } from '../../../../../spec/mesh/hyperschema/types.d.ts';
+import type { Mesh } from '../../../mesh.ts';
+import type { VectorHit } from '../vector/types.ts';
+export declare function fullTextBudgetChars(ctxSize: number): number;
+export declare function isFullTextHit(hit: {
+    readonly id: string;
+}): boolean;
+export type FullTextCoverage = {
+    readonly unit: 'pages';
+    readonly included: number;
+    readonly total: number;
+} | {
+    readonly unit: 'chars';
+    readonly included: number;
+    readonly total: number;
+};
+export declare function isTruncatedFullTextHit(hit: VectorHit): boolean;
+export declare function formatFullDocumentContext(hits: ReadonlyArray<VectorHit>): string;
+export interface FullTextOptions {
+    readonly head?: boolean;
+}
+export declare function fullTextHits(mesh: Mesh, rows: ReadonlyArray<QvacKnowledge>, budgetChars: number, { head }?: FullTextOptions): Promise<VectorHit[] | null>;

package/dist-lib/lib/core/files/rag/retrieval/full-text.js

@@ -0,0 +1,137 @@
+import { parseRagMetadata } from "../chunk/metadata.js";
+import { classifyRagDocument, RAG_DOCUMENT_KIND } from "../extract/document-types.js";
+import { extractRagDocumentFromBytes } from "../extract/extract.js";
+import { extractPdfHead } from "../extract/pdf-text.js";
+const DEFAULT_FULL_TEXT_BUDGET_TOKENS = 4096;
+const CHARS_PER_TOKEN = 4;
+const FULL_TEXT_MAX_BYTES = 20 * 1024 * 1024;
+const FULL_TEXT_KINDS = new Set([
+    RAG_DOCUMENT_KIND.pdf,
+    RAG_DOCUMENT_KIND.rtf,
+    RAG_DOCUMENT_KIND.html,
+    RAG_DOCUMENT_KIND.csv,
+    RAG_DOCUMENT_KIND.tsv,
+    RAG_DOCUMENT_KIND.text
+]);
+export function fullTextBudgetChars(ctxSize) {
+    const tokens = ctxSize > 0 ? Math.floor(ctxSize / 2) : DEFAULT_FULL_TEXT_BUDGET_TOKENS;
+    return tokens * CHARS_PER_TOKEN;
+}
+const FULL_TEXT_HIT_PREFIX = 'full-text-';
+export function isFullTextHit(hit) {
+    return hit.id.startsWith(FULL_TEXT_HIT_PREFIX);
+}
+function fullTextExtras(hit) {
+    try {
+        return JSON.parse(hit.metadataJson);
+    }
+    catch {
+        return {};
+    }
+}
+export function isTruncatedFullTextHit(hit) {
+    return isFullTextHit(hit) && fullTextExtras(hit).truncated === true;
+}
+export function formatFullDocumentContext(hits) {
+    const truncated = hits.find(isTruncatedFullTextHit);
+    const parts = [
+        truncated
+            ? truncatedFraming(titleOf(truncated), fullTextExtras(truncated).coverage, hits.length > 1)
+            : 'The full text of the attached document(s) follows. Answer the user from it directly.'
+    ];
+    for (const hit of hits) {
+        const extras = fullTextExtras(hit);
+        const scope = extras.truncated && extras.coverage ? ` (${describeCoverage(extras.coverage)})` : '';
+        parts.push('', `--- ${titleOf(hit)}${scope} ---`, hit.content);
+    }
+    return parts.join('\n');
+}
+function titleOf(hit) {
+    return parseRagMetadata(hit.metadataJson)?.title ?? 'document';
+}
+function truncatedFraming(title, coverage, others) {
+    const read = coverage ? `Only the ${describeCoverage(coverage)}` : 'Only the opening';
+    const rest = others ? ' The other attached documents follow in full.' : '';
+    return `${read} of "${title}" could be read: it is too long for the model's context window. Do not try to read it again or fetch the rest with a tool, it would not fit either. Answer from what is here and say that your answer covers only the beginning of "${title}".${rest}`;
+}
+function describeCoverage(coverage) {
+    return coverage.unit === 'pages'
+        ? `first ${coverage.included} of ${coverage.total} pages`
+        : `first ${formatChars(coverage.included)} of about ${formatChars(coverage.total)} characters`;
+}
+function formatChars(count) {
+    return count >= 1000 ? `${Math.round(count / 1000)}k` : String(count);
+}
+export async function fullTextHits(mesh, rows, budgetChars, { head = false } = {}) {
+    if (rows.length === 0)
+        return null;
+    const kinds = rows.map((row) => classifyRagDocument({ fileName: row.fileName, mimeType: row.mimeType }));
+    if (!kinds.every((kind) => FULL_TEXT_KINDS.has(kind)))
+        return null;
+    const hits = [];
+    let used = 0;
+    for (const [index, row] of rows.entries()) {
+        if (row.byteLength > FULL_TEXT_MAX_BYTES)
+            return null;
+        const remaining = budgetChars - used;
+        if (!head && kinds[index] !== RAG_DOCUMENT_KIND.pdf && row.byteLength > remaining)
+            return null;
+        let read;
+        try {
+            const bytes = await mesh.files.readBytes(row.blobId);
+            read = await extractFullText(kinds[index], bytes, row, remaining);
+        }
+        catch (err) {
+            logger.warn('rag', 'full-text extraction failed; falling back to retrieval', { err });
+            return null;
+        }
+        if (!read.text)
+            return null;
+        const fits = read.complete && read.text.length <= remaining;
+        if (!fits && !head)
+            return null;
+        const text = fits ? read.text : read.text.slice(0, remaining);
+        used += text.length;
+        hits.push({
+            id: `${FULL_TEXT_HIT_PREFIX}${row.id}`,
+            content: text,
+            metadataJson: JSON.stringify({
+                documentId: `${FULL_TEXT_HIT_PREFIX}${row.id}`,
+                title: row.fileName,
+                mimeType: row.mimeType,
+                attachmentId: row.id,
+                chatId: row.chatId,
+                chunkIndex: 0,
+                tokenCount: Math.ceil(text.length / 4),
+                indexedAt: new Date().toISOString(),
+                ...(fits ? {} : { truncated: true, coverage: coverageOf(read, text.length) })
+            }),
+            score: 1
+        });
+        if (!fits)
+            break;
+    }
+    return hits;
+}
+function coverageOf(read, included) {
+    if (read.pages && read.pages.read < read.pages.total) {
+        return { unit: 'pages', included: read.pages.read, total: read.pages.total };
+    }
+    return { unit: 'chars', included, total: read.totalChars ?? read.text.length };
+}
+async function extractFullText(kind, bytes, row, remainingChars) {
+    if (kind === RAG_DOCUMENT_KIND.pdf) {
+        const pdf = await extractPdfHead(bytes, { maxChars: remainingChars });
+        return {
+            text: pdf.text,
+            complete: pdf.pagesRead === pdf.totalPages,
+            pages: { read: pdf.pagesRead, total: pdf.totalPages }
+        };
+    }
+    const document = await extractRagDocumentFromBytes({
+        fileName: row.fileName,
+        mimeType: row.mimeType,
+        bytes
+    });
+    return { text: document.content, complete: true, totalChars: document.content.length };
+}

package/dist-lib/lib/core/files/rag/retrieval/knowledge.d.ts

@@ -0,0 +1,2 @@
+import type { RagKnowledgeQuery, RagKnowledgeResult, RagRetrievalDeps } from './types.ts';
+export declare function retrieveRagKnowledge(query: RagKnowledgeQuery, deps: RagRetrievalDeps): Promise<RagKnowledgeResult>;

package/dist-lib/lib/core/files/rag/retrieval/knowledge.js

@@ -0,0 +1,53 @@
+import { applySimilarityFloor, logSimilarityScores } from "./floor.js";
+import { searchRagChunks } from "./search.js";
+import { DEFAULT_CHUNK_BUDGET_TOKENS, MAX_SELECTED_CHUNKS } from "./selection.js";
+import { resolveTargetDocument } from "./target.js";
+import { isWholeDocumentQuery } from "./whole-document-query.js";
+export async function retrieveRagKnowledge(query, deps) {
+    if (query.scopeDocumentIds?.length) {
+        const hits = await searchRagChunks({
+            query: query.query,
+            chatId: query.chatId,
+            liveSourceIds: query.liveSourceIds,
+            ...scopedShape(query.query, query.scopeDocumentIds)
+        }, deps);
+        logSimilarityScores(hits, hits);
+        return { hits, targetDocumentId: null, searchedCount: hits.length };
+    }
+    const targetDocumentId = resolveTargetDocument(query.query, query.documents);
+    if (!targetDocumentId && isWholeDocumentQuery(query.query)) {
+        const hits = await searchRagChunks({
+            query: query.query,
+            chatId: query.chatId,
+            liveSourceIds: query.liveSourceIds,
+            documentIds: query.documents.map((document) => document.documentId),
+            coverage: true,
+            chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS
+        }, deps);
+        return { hits, targetDocumentId, searchedCount: hits.length };
+    }
+    const { hits: searched, scoped } = await searchKnowledge(query, deps, targetDocumentId);
+    const hits = applySimilarityFloor(searched, scoped ? targetDocumentId : null);
+    logSimilarityScores(searched, hits);
+    return { hits, targetDocumentId, searchedCount: searched.length };
+}
+async function searchKnowledge(query, deps, targetDocumentId) {
+    const base = { query: query.query, chatId: query.chatId, liveSourceIds: query.liveSourceIds };
+    if (!targetDocumentId) {
+        const hits = await searchRagChunks({ ...base, documentFirst: true, chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS }, deps);
+        return { hits, scoped: false };
+    }
+    const scopedHits = await searchRagChunks({ ...base, ...scopedShape(query.query, [targetDocumentId]) }, deps);
+    if (scopedHits.length > 0)
+        return { hits: scopedHits, scoped: true };
+    const hits = await searchRagChunks({ ...base, documentFirst: true, chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS }, deps);
+    return { hits, scoped: false };
+}
+function scopedShape(query, documentIds) {
+    return {
+        documentIds,
+        chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS,
+        ...(documentIds.length === 1 ? { perDocChunkCap: MAX_SELECTED_CHUNKS } : {}),
+        ...(isWholeDocumentQuery(query) ? { coverage: true } : { documentFirst: true })
+    };
+}

package/dist-lib/lib/core/files/rag/retrieval/notices.d.ts

@@ -0,0 +1,16 @@
+export declare const RAG_FAILED_NOTICE = "Knowledge search failed and was not used.";
+export declare const RAG_EMPTY_NOTICE = "Knowledge is enabled, but no relevant excerpts were found.";
+export declare const RAG_UNAVAILABLE_NOTICE = "Knowledge is still being indexed and was not used yet.";
+export declare const RAG_KNOWLEDGE_OFF_NOTICE = "Knowledge is off for this agent, so the attached document was not used. Enable Knowledge to let the agent read it.";
+export declare const RAG_TRUNCATED_NOTICE = "Only the beginning of the attached document was used. Enable Knowledge to let the agent search all of it.";
+export declare function ragUnreadableNotice(files: ReadonlyArray<string>): string;
+export declare function chooseRagNotice(input: {
+    readonly errorMessage?: string | null;
+    readonly hitCount: number;
+    readonly searchedCount: number;
+    readonly indexEmpty: boolean;
+    readonly hasKnowledge: boolean;
+    readonly pendingReferenced?: boolean;
+    readonly failedFiles?: ReadonlyArray<string>;
+    readonly pendingFiles?: ReadonlyArray<string>;
+}): string | null;

package/dist-lib/lib/core/files/rag/retrieval/notices.js

@@ -0,0 +1,27 @@
+export const RAG_FAILED_NOTICE = 'Knowledge search failed and was not used.';
+export const RAG_EMPTY_NOTICE = 'Knowledge is enabled, but no relevant excerpts were found.';
+export const RAG_UNAVAILABLE_NOTICE = 'Knowledge is still being indexed and was not used yet.';
+export const RAG_KNOWLEDGE_OFF_NOTICE = 'Knowledge is off for this agent, so the attached document was not used. Enable Knowledge to let the agent read it.';
+export const RAG_TRUNCATED_NOTICE = 'Only the beginning of the attached document was used. Enable Knowledge to let the agent search all of it.';
+export function ragUnreadableNotice(files) {
+    const plural = files.length > 1;
+    return `Attached file${plural ? 's' : ''} could not be read and ${plural ? 'were' : 'was'} not used: ${files.join(', ')}.`;
+}
+export function chooseRagNotice(input) {
+    if (input.errorMessage)
+        return RAG_FAILED_NOTICE;
+    if (input.pendingReferenced)
+        return RAG_UNAVAILABLE_NOTICE;
+    if (input.failedFiles?.length)
+        return ragUnreadableNotice(input.failedFiles);
+    if (input.pendingFiles?.length) {
+        return `Files not yet available: ${input.pendingFiles.join(', ')}.`;
+    }
+    if (input.hitCount > 0)
+        return null;
+    if (input.searchedCount > 0)
+        return RAG_EMPTY_NOTICE;
+    if (input.indexEmpty && input.hasKnowledge)
+        return RAG_UNAVAILABLE_NOTICE;
+    return RAG_EMPTY_NOTICE;
+}