@qvac/core 0.0.0 → 0.1.0

package/dist-lib/lib/core/files/rag/retrieval/scoring.d.ts

@@ -0,0 +1,8 @@
+import type { VectorHit } from '../vector/types.ts';
+export type ScoredDocument = {
+    readonly documentId: string;
+    readonly title: string;
+    readonly score: number;
+    readonly hits: ReadonlyArray<VectorHit>;
+};
+export declare function scoreDocuments(hits: ReadonlyArray<VectorHit>, query: string): ReadonlyArray<ScoredDocument>;

package/dist-lib/lib/core/files/rag/retrieval/scoring.js

@@ -0,0 +1,82 @@
+import { normalize } from 'asenion';
+import { parseRagMetadata } from "../chunk/metadata.js";
+import { extractSalientTerms, isMeaningfulToken } from "./terms.js";
+const SEM_MAX_WEIGHT = 0.7;
+const SEM_MEAN_WEIGHT = 0.3;
+const SEM_TOP_K = 3;
+const LEX_PHRASE_WEIGHT = 1;
+const LEX_ENTITY_WEIGHT = 1;
+const LEX_TOKEN_OVERLAP_WEIGHT = 0.5;
+const TITLE_EXACT_SCORE = 1;
+const TITLE_TOKEN_SCORE = 0.6;
+const DOC_SEMANTIC_WEIGHT = 1;
+const DOC_LEXICAL_WEIGHT = 0.6;
+const DOC_TITLE_WEIGHT = 0.8;
+const ALL_ENTITIES_BOOST = 0.5;
+export function scoreDocuments(hits, query) {
+    const terms = extractSalientTerms(query);
+    const scored = [];
+    for (const [documentId, group] of groupHitsByDocument(hits)) {
+        const docHits = [...group.hits].sort((left, right) => right.score - left.score);
+        const contentNorm = normalize(docHits.map((hit) => hit.content).join(' '));
+        const titleNorm = normalize(group.title);
+        const allEntities = terms.entities.length > 0 &&
+            terms.entities.every((entity) => contentNorm.includes(entity) || titleNorm.includes(entity));
+        const score = DOC_SEMANTIC_WEIGHT * semanticScore(docHits) +
+            DOC_LEXICAL_WEIGHT * lexicalScore(terms, contentNorm, titleNorm) +
+            DOC_TITLE_WEIGHT * titleScore(terms, query, group.title) +
+            (allEntities ? ALL_ENTITIES_BOOST : 0);
+        scored.push({ documentId, title: group.title, score, hits: docHits });
+    }
+    return scored.sort((left, right) => right.score - left.score);
+}
+function groupHitsByDocument(hits) {
+    const groups = new Map();
+    for (const hit of hits) {
+        const metadata = parseRagMetadata(hit.metadataJson);
+        if (!metadata)
+            continue;
+        const group = groups.get(metadata.documentId) ?? { title: metadata.title, hits: [] };
+        group.hits.push(hit);
+        groups.set(metadata.documentId, group);
+    }
+    return groups;
+}
+function semanticScore(docHits) {
+    if (docHits.length === 0)
+        return 0;
+    const max = docHits[0]?.score ?? 0;
+    const top = docHits.slice(0, SEM_TOP_K);
+    const mean = top.reduce((sum, hit) => sum + hit.score, 0) / top.length;
+    return SEM_MAX_WEIGHT * max + SEM_MEAN_WEIGHT * mean;
+}
+function lexicalScore(terms, contentNorm, titleNorm) {
+    const phrase = terms.quotedPhrases.some((value) => contentNorm.includes(value) || titleNorm.includes(value))
+        ? 1
+        : 0;
+    const entityFraction = terms.entities.length === 0
+        ? 0
+        : terms.entities.filter((entity) => contentNorm.includes(entity) || titleNorm.includes(entity)).length / terms.entities.length;
+    const tokenOverlap = terms.tokens.length === 0 ? 0 : overlapCount(terms.tokens, contentNorm) / terms.tokens.length;
+    return (LEX_PHRASE_WEIGHT * phrase +
+        LEX_ENTITY_WEIGHT * entityFraction +
+        LEX_TOKEN_OVERLAP_WEIGHT * tokenOverlap);
+}
+function titleScore(terms, query, title) {
+    const stemTokens = normalize(title.replace(/\.[A-Za-z0-9]+$/, ''))
+        .split(' ')
+        .filter(isMeaningfulToken);
+    if (stemTokens.length === 0)
+        return 0;
+    const stem = stemTokens.join(' ');
+    if (normalize(query).includes(stem))
+        return TITLE_EXACT_SCORE;
+    const titleTokens = new Set(stemTokens);
+    const tokenHit = terms.tokens.some((token) => titleTokens.has(token));
+    const entityHit = terms.entities.some((entity) => stem.includes(entity));
+    return tokenHit || entityHit ? TITLE_TOKEN_SCORE : 0;
+}
+function overlapCount(tokens, contentNorm) {
+    const contentTokens = new Set(contentNorm.split(' '));
+    return tokens.filter((token) => contentTokens.has(token)).length;
+}

package/dist-lib/lib/core/files/rag/retrieval/search.d.ts

@@ -0,0 +1,3 @@
+import type { VectorHit } from '../vector/types.ts';
+import type { RagRetrievalDeps, RagRetrievalQuery } from './types.ts';
+export declare function searchRagChunks(query: RagRetrievalQuery, deps: RagRetrievalDeps): Promise<ReadonlyArray<VectorHit>>;

package/dist-lib/lib/core/files/rag/retrieval/search.js

@@ -0,0 +1,46 @@
+import { selectCoverage } from "./coverage.js";
+import { applyPerSourceCap, scopeFilterFor } from "./diversity.js";
+import { scoreDocuments } from "./scoring.js";
+import { selectChunks } from "./selection.js";
+const DEFAULT_LIMIT = 5;
+export async function searchRagChunks(query, deps) {
+    const trimmed = query.query.trim();
+    if (!trimmed)
+        return [];
+    if (deps.index.info().chunks === 0)
+        return [];
+    const embedding = await embedQuery(deps, trimmed);
+    const total = deps.index.info().chunks;
+    if (total === 0)
+        return [];
+    const hits = await deps.index.search(embedding, { limit: total, filter: scopeFilterFor(query) });
+    if (query.coverage) {
+        return selectCoverage(hits, {
+            ...(query.chunkBudgetTokens && query.chunkBudgetTokens > 0
+                ? { budgetTokens: query.chunkBudgetTokens }
+                : {})
+        });
+    }
+    if (useDocumentFirst(query))
+        return documentFirstSelect(query, hits);
+    return applyPerSourceCap(hits, query.limit ?? DEFAULT_LIMIT);
+}
+async function embedQuery(deps, text) {
+    const rows = await deps.embed([text]);
+    const embedding = rows[0];
+    if (!embedding)
+        throw new Error('rag retrieval: the embedder returned no query embedding');
+    return embedding;
+}
+function useDocumentFirst(query) {
+    return query.documentFirst === true && Boolean(query.chatId) && !query.attachmentIds?.length;
+}
+function documentFirstSelect(query, hits) {
+    const ranked = scoreDocuments(hits, query.query);
+    return selectChunks(ranked, {
+        ...(query.chunkBudgetTokens && query.chunkBudgetTokens > 0
+            ? { chunkBudgetTokens: query.chunkBudgetTokens }
+            : {}),
+        ...(query.perDocChunkCap ? { perDocChunkCap: query.perDocChunkCap } : {})
+    });
+}

package/dist-lib/lib/core/files/rag/retrieval/selection.d.ts

@@ -0,0 +1,10 @@
+import type { VectorHit } from '../vector/types.ts';
+import type { ScoredDocument } from './scoring.ts';
+export declare const DEFAULT_CHUNK_BUDGET_TOKENS = 3072;
+export declare const MAX_SELECTED_CHUNKS: number;
+export type SelectChunksOptions = {
+    readonly maxDocs?: number;
+    readonly chunkBudgetTokens?: number;
+    readonly perDocChunkCap?: number;
+};
+export declare function selectChunks(ranked: ReadonlyArray<ScoredDocument>, options?: SelectChunksOptions): ReadonlyArray<VectorHit>;

package/dist-lib/lib/core/files/rag/retrieval/selection.js

@@ -0,0 +1,63 @@
+import { parseRagMetadata } from "../chunk/metadata.js";
+import { estimateRagTokens } from "../chunk/tokenizer.js";
+const DEFAULT_MAX_DOCS = 8;
+const DEFAULT_PER_DOC_CHUNK_CAP = 4;
+export const DEFAULT_CHUNK_BUDGET_TOKENS = 3072;
+export const MAX_SELECTED_CHUNKS = DEFAULT_MAX_DOCS * DEFAULT_PER_DOC_CHUNK_CAP;
+export function selectChunks(ranked, options = {}) {
+    const topDocs = ranked.slice(0, options.maxDocs ?? DEFAULT_MAX_DOCS);
+    if (topDocs.length === 0)
+        return [];
+    const state = {
+        topDocs,
+        budget: options.chunkBudgetTokens ?? DEFAULT_CHUNK_BUDGET_TOKENS,
+        perDocCap: options.perDocChunkCap ?? DEFAULT_PER_DOC_CHUNK_CAP,
+        used: new Set(),
+        perDocCount: new Map(),
+        selected: [],
+        usedTokens: 0
+    };
+    seedCoverage(state);
+    fillBudget(state);
+    return state.selected
+        .sort((left, right) => left.docRank - right.docRank || left.chunkIndex - right.chunkIndex)
+        .map((entry) => entry.hit);
+}
+function seedCoverage(state) {
+    state.topDocs.forEach((doc, rank) => {
+        const best = doc.hits[0];
+        if (!best || state.used.has(best.id))
+            return;
+        if (rank === 0 || state.usedTokens + hitTokens(best) <= state.budget) {
+            take(state, rank, best);
+        }
+    });
+}
+function fillBudget(state) {
+    let added = true;
+    while (added) {
+        added = false;
+        state.topDocs.forEach((doc, rank) => {
+            if ((state.perDocCount.get(doc.documentId) ?? 0) >= state.perDocCap)
+                return;
+            const next = doc.hits.find((hit) => !state.used.has(hit.id));
+            if (next && state.usedTokens + hitTokens(next) <= state.budget) {
+                take(state, rank, next);
+                added = true;
+            }
+        });
+    }
+}
+function take(state, docRank, hit) {
+    state.used.add(hit.id);
+    const documentId = state.topDocs[docRank]?.documentId ?? '';
+    state.perDocCount.set(documentId, (state.perDocCount.get(documentId) ?? 0) + 1);
+    state.usedTokens += hitTokens(hit);
+    state.selected.push({ docRank, chunkIndex: chunkIndexOf(hit), hit });
+}
+function hitTokens(hit) {
+    return parseRagMetadata(hit.metadataJson)?.tokenCount ?? estimateRagTokens(hit.content);
+}
+function chunkIndexOf(hit) {
+    return parseRagMetadata(hit.metadataJson)?.chunkIndex ?? 0;
+}

package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.d.ts

@@ -0,0 +1,6 @@
+export type RagSourceTagFilter = {
+    readonly add: (sourceNames: ReadonlyArray<string>) => void;
+    readonly push: (text: string) => string;
+    readonly flush: () => string;
+};
+export declare function createRagSourceTagFilter(): RagSourceTagFilter;

package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.js

@@ -0,0 +1,79 @@
+const SOURCE_TAG_START = /\[\s*sources?(?:\s+\d+)?\s*:/i;
+const POSSIBLE_SOURCE_TAG_START = /^\[\s*(?:s(?:o(?:u(?:r(?:c(?:e(?:s)?)?)?)?)?)?)?(?:\s+\d*)?\s*$/i;
+export function createRagSourceTagFilter() {
+    const state = { buffer: '', hasSources: false };
+    return {
+        add(sourceNames) {
+            if (state.hasSources)
+                return;
+            for (const name of sourceNames) {
+                if (name.trim().length > 0) {
+                    state.hasSources = true;
+                    return;
+                }
+            }
+        },
+        push(text) {
+            state.buffer += text;
+            return drain(state);
+        },
+        flush() {
+            const start = SOURCE_TAG_START.exec(state.buffer);
+            if (!start) {
+                const output = state.buffer;
+                state.buffer = '';
+                return output;
+            }
+            const prefix = state.buffer.slice(0, start.index);
+            const tag = state.buffer.slice(start.index);
+            state.buffer = '';
+            if (state.hasSources)
+                return prefix.trimEnd();
+            return prefix + tag;
+        }
+    };
+}
+function drain(state) {
+    let output = '';
+    for (;;) {
+        const start = SOURCE_TAG_START.exec(state.buffer);
+        if (!start)
+            return output + emitNonTagText(state);
+        const end = state.buffer.indexOf(']', start.index);
+        if (end === -1) {
+            const holdStart = trailingWhitespaceStart(state.buffer, start.index);
+            output += state.buffer.slice(0, holdStart);
+            state.buffer = state.buffer.slice(holdStart);
+            return output;
+        }
+        const prefix = state.buffer.slice(0, start.index);
+        const tag = state.buffer.slice(start.index, end + 1);
+        state.buffer = state.buffer.slice(end + 1);
+        if (state.hasSources)
+            output += prefix.trimEnd();
+        else
+            output += prefix + tag;
+    }
+}
+function emitNonTagText(state) {
+    const possibleStart = possibleSourceTagStart(state.buffer);
+    if (possibleStart === -1) {
+        const output = state.buffer;
+        state.buffer = '';
+        return output;
+    }
+    const holdStart = trailingWhitespaceStart(state.buffer, possibleStart);
+    const output = state.buffer.slice(0, holdStart);
+    state.buffer = state.buffer.slice(holdStart);
+    return output;
+}
+function possibleSourceTagStart(text) {
+    const start = text.lastIndexOf('[');
+    return start !== -1 && POSSIBLE_SOURCE_TAG_START.test(text.slice(start)) ? start : -1;
+}
+function trailingWhitespaceStart(text, end) {
+    let start = end;
+    while (start > 0 && /\s/.test(text[start - 1] ?? ''))
+        start--;
+    return start;
+}

package/dist-lib/lib/core/files/rag/retrieval/sources.d.ts

@@ -0,0 +1,13 @@
+import { RAG_SOURCE_EXCERPT_CHARS } from '../../../../../schema/qvac.ts';
+import type { VectorHit } from '../vector/types.ts';
+export { RAG_SOURCE_EXCERPT_CHARS };
+export type RagSource = {
+    readonly documentId: string | null;
+    readonly title: string;
+    readonly chunkIndex: number;
+    readonly excerpt: string;
+    readonly attachmentId: string | null;
+    readonly artifactId: string | null;
+    readonly score: number;
+};
+export declare function sourcesFromHits(hits: ReadonlyArray<VectorHit>): RagSource[];

package/dist-lib/lib/core/files/rag/retrieval/sources.js

@@ -0,0 +1,20 @@
+import { MAX_RAG_SOURCES_PER_CHUNK, RAG_SOURCE_EXCERPT_CHARS } from "../../../../../schema/qvac.js";
+import { parseRagMetadata } from "../chunk/metadata.js";
+export { RAG_SOURCE_EXCERPT_CHARS };
+export function sourcesFromHits(hits) {
+    return hits.slice(0, MAX_RAG_SOURCES_PER_CHUNK).map((hit, index) => {
+        const metadata = parseRagMetadata(hit.metadataJson);
+        const chunkIndex = metadata && Number.isSafeInteger(metadata.chunkIndex) && metadata.chunkIndex >= 0
+            ? metadata.chunkIndex
+            : index;
+        return {
+            documentId: metadata?.documentId ?? null,
+            title: metadata?.title ?? 'document',
+            chunkIndex,
+            excerpt: hit.content.slice(0, RAG_SOURCE_EXCERPT_CHARS),
+            attachmentId: metadata?.attachmentId ?? null,
+            artifactId: metadata?.artifactId ?? null,
+            score: hit.score
+        };
+    });
+}

package/dist-lib/lib/core/files/rag/retrieval/target.d.ts

@@ -0,0 +1,13 @@
+export type RagDocumentRef = {
+    readonly documentId: string;
+    readonly title: string;
+};
+export declare function resolveTargetDocument(query: string, documents: ReadonlyArray<RagDocumentRef>): string | null;
+export declare function referencedDocuments<T extends {
+    readonly title: string;
+}>(query: string, documents: ReadonlyArray<T>): ReadonlyArray<T>;
+export declare function referencesPendingFile(query: string, documents: ReadonlyArray<{
+    readonly title: string;
+    readonly pending: boolean;
+}>): boolean;
+export declare function containsFileName(haystack: string, name: string): boolean;

package/dist-lib/lib/core/files/rag/retrieval/target.js

@@ -0,0 +1,89 @@
+import { has } from 'asenion';
+const TYPE_KEYWORDS = [
+    ['pdf', ['pdf']],
+    ['html', ['html', 'htm']],
+    ['csv', ['csv']],
+    ['spreadsheet', ['csv', 'xlsx', 'xls']],
+    ['json', ['json']],
+    ['markdown', ['md', 'markdown']],
+    ['xml', ['xml']],
+    ['yaml', ['yaml', 'yml']]
+];
+const MIN_STEM_LENGTH = 4;
+const STEM_STOPWORDS = new Set([
+    'report',
+    'summary',
+    'notes',
+    'note',
+    'data',
+    'test',
+    'tests',
+    'file',
+    'files',
+    'document',
+    'documents',
+    'draft',
+    'final',
+    'image',
+    'images',
+    'photo',
+    'doc',
+    'docs',
+    'untitled',
+    'example',
+    'sample',
+    'output',
+    'input',
+    'readme',
+    'index',
+    'main',
+    'temp',
+    'guide'
+]);
+export function resolveTargetDocument(query, documents) {
+    const named = documents.filter((document) => mentionsName(query.toLowerCase(), document.title));
+    const matches = named.length ? named : referencedDocuments(query, documents);
+    return matches.length === 1 ? matches[0].documentId : null;
+}
+export function referencedDocuments(query, documents) {
+    const haystack = query.toLowerCase();
+    return documents.filter((document) => mentionsName(haystack, document.title) || referencesFileType(haystack, document.title));
+}
+function mentionsName(haystack, title) {
+    const name = title.toLowerCase();
+    if (containsFileName(haystack, name))
+        return true;
+    const stem = name.replace(/\.[^.]+$/, '');
+    if (stem.length < MIN_STEM_LENGTH || STEM_STOPWORDS.has(stem) || !looksDistinctive(stem)) {
+        return false;
+    }
+    return has(haystack, stem);
+}
+export function referencesPendingFile(query, documents) {
+    return referencedDocuments(query, documents).some((document) => document.pending);
+}
+function referencesFileType(haystack, title) {
+    const ext = title.toLowerCase().match(/\.([^.]+)$/)?.[1];
+    if (!ext)
+        return false;
+    return TYPE_KEYWORDS.some(([word, exts]) => exts.includes(ext) && referencesTypeWord(haystack, word));
+}
+function referencesTypeWord(haystack, word) {
+    const w = word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    for (const m of haystack.matchAll(new RegExp(`(^|[^a-z0-9._-])${w}(?=$|[^a-z0-9._-]|\\.(?:$|[^a-z0-9._-]))`, 'g'))) {
+        const before = haystack.slice(0, (m.index ?? 0) + m[1].length);
+        const after = haystack.slice((m.index ?? 0) + m[0].length);
+        const asFormat = /(^|[^a-z0-9])(?:in|to|into|as)\s+(?:an?\s+)?$/.test(before) ||
+            /^\s+format(?![a-z0-9])/.test(after);
+        if (!asFormat)
+            return true;
+    }
+    return false;
+}
+function looksDistinctive(stem) {
+    return /[0-9_\-. ]/.test(stem);
+}
+export function containsFileName(haystack, name) {
+    const escaped = name.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    return new RegExp(`(^|[^a-z0-9._-])${escaped}(?=$|[^a-z0-9._-]|\\.(?:$|[^a-z0-9._-]))`).test(haystack.toLowerCase());
+}

package/dist-lib/lib/core/files/rag/retrieval/terms.d.ts

@@ -0,0 +1,8 @@
+export type SalientTerms = {
+    readonly tokens: ReadonlyArray<string>;
+    readonly quotedPhrases: ReadonlyArray<string>;
+    readonly entities: ReadonlyArray<string>;
+};
+export declare function isMeaningfulToken(token: string): boolean;
+export declare function extractSalientTerms(query: string): SalientTerms;
+export declare function uniqueStrings(values: ReadonlyArray<string>): ReadonlyArray<string>;

package/dist-lib/lib/core/files/rag/retrieval/terms.js

@@ -0,0 +1,70 @@
+import { closedClass, normalize, tag } from 'asenion';
+import { tokenizeForRag } from "../chunk/tokenizer.js";
+const MIN_TERM_LENGTH = 3;
+const GENERIC_WORDS = new Set([
+    'give',
+    'show',
+    'tell',
+    'get',
+    'find',
+    'what',
+    'data',
+    'info',
+    'information',
+    'please',
+    'want'
+]);
+export function isMeaningfulToken(token) {
+    if (token.length < MIN_TERM_LENGTH)
+        return false;
+    return !closedClass(token) && !GENERIC_WORDS.has(token);
+}
+export function extractSalientTerms(query) {
+    const quotedPhrases = [];
+    const quotePattern = /"([^"]+)"|'([^']+)'/g;
+    let quoted = quotePattern.exec(query);
+    while (quoted) {
+        const phrase = normalize(quoted[1] ?? quoted[2] ?? '');
+        if (phrase)
+            quotedPhrases.push(phrase);
+        quoted = quotePattern.exec(query);
+    }
+    const tokens = uniqueStrings(tokenizeForRag(query)
+        .map((token) => token.text.trim().toLowerCase())
+        .filter((token) => isMeaningfulToken(token) && /[\p{L}\p{N}]/u.test(token)));
+    return { tokens, quotedPhrases, entities: extractEntities(query) };
+}
+function extractEntities(query) {
+    const words = query.split(/\s+/).filter(Boolean);
+    const entities = [];
+    let run = [];
+    const flush = () => {
+        if (run.length >= 2) {
+            const normalized = normalize(run.join(' '));
+            if (names(normalized))
+                entities.push(normalized);
+        }
+        run = [];
+    };
+    for (const word of words) {
+        if (isCapitalizedWord(word)) {
+            run.push(word);
+        }
+        else {
+            flush();
+        }
+    }
+    flush();
+    return uniqueStrings(entities);
+}
+function names(run) {
+    return tag(run).some(({ word, tag: part }) => isMeaningfulToken(word) && NAMING.has(part));
+}
+const NAMING = new Set(['noun', 'adj', 'num']);
+function isCapitalizedWord(word) {
+    const firstLetter = word.match(/[\p{L}]/u)?.[0];
+    return Boolean(firstLetter) && firstLetter === firstLetter?.toUpperCase();
+}
+export function uniqueStrings(values) {
+    return [...new Set(values)];
+}

package/dist-lib/lib/core/files/rag/retrieval/types.d.ts

@@ -0,0 +1,28 @@
+import type { VectorHit, VectorIndex } from '../vector/types.ts';
+import type { RagScopeFilters } from './diversity.ts';
+import type { RagDocumentRef } from './target.ts';
+export type RagEmbedFn = (texts: string[]) => Promise<number[][]>;
+export type RagRetrievalDeps = {
+    readonly index: VectorIndex;
+    readonly embed: RagEmbedFn;
+};
+export type RagRetrievalQuery = RagScopeFilters & {
+    readonly query: string;
+    readonly limit?: number;
+    readonly documentFirst?: boolean;
+    readonly chunkBudgetTokens?: number;
+    readonly coverage?: boolean;
+    readonly perDocChunkCap?: number;
+};
+export type RagKnowledgeQuery = {
+    readonly query: string;
+    readonly chatId: string;
+    readonly documents: ReadonlyArray<RagDocumentRef>;
+    readonly liveSourceIds?: ReadonlyArray<string>;
+    readonly scopeDocumentIds?: ReadonlyArray<string>;
+};
+export type RagKnowledgeResult = {
+    readonly hits: ReadonlyArray<VectorHit>;
+    readonly targetDocumentId: string | null;
+    readonly searchedCount: number;
+};

package/dist-lib/lib/core/files/rag/retrieval/types.js

@@ -0,0 +1 @@
+export {};

package/dist-lib/lib/core/files/rag/retrieval/web-sources.d.ts

@@ -0,0 +1,8 @@
+import type { RagSource } from './sources.ts';
+type WebRagSource = RagSource & {
+    readonly url: string;
+};
+export declare const WEB_TOOLS: string[];
+export declare function webSourcesFromToolResult(result: unknown): WebRagSource[];
+export declare function withoutDataChannel(result: unknown): unknown;
+export {};

package/dist-lib/lib/core/files/rag/retrieval/web-sources.js

@@ -0,0 +1,41 @@
+import { MAX_RAG_SOURCES_PER_CHUNK, MAX_RAG_SOURCE_URL_CHARS, RAG_SOURCE_EXCERPT_CHARS } from "../../../../../schema/qvac.js";
+export const WEB_TOOLS = ['web_search', 'web_fetch'];
+export function webSourcesFromToolResult(result) {
+    const results = structuredResults(result);
+    const records = [];
+    for (const entry of results) {
+        if (records.length >= MAX_RAG_SOURCES_PER_CHUNK)
+            break;
+        if (typeof entry?.url !== 'string' || entry.url.length === 0)
+            continue;
+        if (entry.url.length > MAX_RAG_SOURCE_URL_CHARS)
+            continue;
+        records.push({
+            documentId: null,
+            title: typeof entry.title === 'string' && entry.title ? entry.title : entry.url,
+            chunkIndex: records.length,
+            excerpt: (typeof entry.content === 'string' ? entry.content : '').slice(0, RAG_SOURCE_EXCERPT_CHARS),
+            attachmentId: null,
+            artifactId: null,
+            score: 0,
+            url: entry.url
+        });
+    }
+    return records;
+}
+export function withoutDataChannel(result) {
+    if (result && typeof result === 'object' && 'data' in result) {
+        const { data: _data, ...rest } = result;
+        return rest;
+    }
+    return result;
+}
+function structuredResults(result) {
+    if (!result || typeof result !== 'object')
+        return [];
+    const data = result.data;
+    if (!data || typeof data !== 'object')
+        return [];
+    const results = data.results;
+    return Array.isArray(results) ? results : [];
+}

package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.d.ts

@@ -0,0 +1 @@
+export declare function isWholeDocumentQuery(query: string): boolean;