@devflow-tools/knowledge-engine 0.16.27 → 0.16.28

package/dist/knowledge-engine.js

@@ -2,8 +2,8 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync
 import { join } from "path";
 import { createHash } from "node:crypto";
 import { tmpdir } from "node:os";
-import { fetchHtmlSmart, fetchHtmlBrowser, closeBrowser } from "./fetcher.js";
-import { htmlToMarkdown, KNOWLEDGE_CHUNK_POLICY_VERSION, sliceByHeadings } from "./markdown-converter.js";
+import { fetchHtmlKnowledgeSource, closeBrowser } from "./fetcher.js";
+import { KNOWLEDGE_CHUNK_POLICY_VERSION, sliceByHeadings } from "./markdown-converter.js";
 import { KnowledgeHybridSearch } from "./hybrid-search.js";
 import { KnowledgeStore } from "./knowledge-store.js";
 import { EmbeddingProvider } from "@devflow-tools/memory-engine";
@@ -11,22 +11,165 @@ import { BUILTIN_SOURCES } from "./types.js";
 import { fetchFromGitHub } from "./git-fetcher.js";
 import { fetchFromContext7 } from "./context7-fetcher.js";
 import { selectKnowledgeSources } from "./source-selector.js";
-import { estimateKnowledgeTokens as estimateTokens, resolveKnowledgeTitle } from "./result-quality.js";
+import { estimateKnowledgeTokens as estimateTokens, lexicalRelevance, resolveKnowledgeTitle, } from "./result-quality.js";
 import { detectProjectKnowledgeVersions, expectedVersionForSource, isKnowledgeVersionCompatible, } from './source-readiness.js';
 import { evaluateKnowledgeQuality } from './knowledge-quality.js';
+import { KnowledgeCompiler } from './knowledge-compiler.js';
+import { SourceCatalog } from './source-catalog.js';
+import { compileDependencySource } from './dependency-source-compiler.js';
+import { expandKnowledgeEvidenceGraph } from './evidence-graph.js';
+import { KNOWLEDGE_RETRIEVAL_SCORING_PROFILE } from './retrieval-scoring-profile.js';
 export const DEFAULT_KNOWLEDGE_TOKEN_BUDGET = 3000;
 export const estimateKnowledgeTokens = estimateTokens;
+function normalizeActiveSkill(value) {
+    return value.normalize('NFKC').trim().replace(/^\/?devflow:/iu, '').toLocaleLowerCase('en-US');
+}
+function dependencyPackagesForMissingNeeds(needs, dependencies) {
+    const normalizedDependencies = new Map(Object.keys(dependencies).map(name => [name.toLocaleLowerCase('en-US'), name]));
+    const aliases = {
+        angular: '@angular/core',
+        nest: '@nestjs/core',
+        nextjs: 'next',
+        node: '@types/node',
+        'node.js': '@types/node',
+        tailwind: 'tailwindcss',
+    };
+    return [...new Set(needs.flatMap(need => {
+            if (!need.entity)
+                return [];
+            const key = need.entity.normalize('NFKC').trim().toLocaleLowerCase('en-US');
+            const packageName = normalizedDependencies.get(key)
+                ?? normalizedDependencies.get(aliases[key] ?? '');
+            return packageName ? [packageName] : [];
+        }))];
+}
+function buildSafeFtsQuery(value) {
+    const terms = value.normalize('NFKC')
+        .match(/[\p{L}\p{N}_@.-]{2,}/gu)
+        ?.map(term => term.replace(/^\.+|\.+$/gu, '').replace(/"/gu, ''))
+        .filter(Boolean)
+        .slice(0, 24) ?? [];
+    return [...new Set(terms)].map(term => `"${term}"`).join(' OR ');
+}
+function dedupeKnowledgeEntries(entries) {
+    const seen = new Set();
+    return entries.filter(entry => {
+        if (seen.has(entry.id))
+            return false;
+        seen.add(entry.id);
+        return true;
+    });
+}
+function provenanceComplete(value) {
+    if (!value)
+        return false;
+    try {
+        const parsed = JSON.parse(value);
+        return parsed.immutable === true
+            && typeof parsed.sourceId === 'string'
+            && typeof parsed.sourceUri === 'string'
+            && typeof parsed.contentHash === 'string';
+    }
+    catch {
+        return false;
+    }
+}
+function authorityFromProvenance(value, source) {
+    if (value) {
+        try {
+            const parsed = JSON.parse(value);
+            if (parsed.sourceKind === 'official')
+                return 'official';
+            if (parsed.sourceKind === 'project')
+                return 'project';
+            if (parsed.sourceKind === 'plugin' || parsed.sourceKind === 'dependency')
+                return 'maintainer';
+        }
+        catch {
+            // Legacy chunks use source-name authority.
+        }
+    }
+    return /react|typescript|node|vue|angular|mdn|official/iu.test(source) ? 'official' : 'community';
+}
+function addNecessaryAdjacentEntries(input) {
+    const chunks = new Map(input.chunks.map(chunk => [chunk.id, chunk]));
+    const selected = new Set(input.candidates.map(candidate => candidate.id));
+    const additions = [];
+    const requiredSymbols = input.knowledgeNeeds.flatMap(need => need.symbol ? [normalizeLookup(need.symbol)] : []);
+    const requiredEntities = input.knowledgeNeeds.flatMap(need => need.entity ? [normalizeLookup(need.entity)] : []);
+    for (const candidate of input.candidates) {
+        if (input.candidates.length + additions.length >= input.limit)
+            break;
+        const chunk = chunks.get(candidate.id);
+        if (!chunk?.page_id)
+            continue;
+        const neighbors = input.chunks.filter(neighbor => neighbor.page_id === chunk.page_id
+            && Math.abs(neighbor.chunk_index - chunk.chunk_index) === 1
+            && !selected.has(neighbor.id));
+        for (const neighbor of neighbors) {
+            const symbols = parseJsonStrings(neighbor.symbols).map(normalizeLookup);
+            const entities = parseJsonStrings(neighbor.entities).map(normalizeLookup);
+            const preservesNeed = requiredSymbols.some(symbol => symbols.includes(symbol))
+                || requiredEntities.some(entity => entities.includes(entity) || normalizeLookup(neighbor.source).includes(entity));
+            if (!preservesNeed || lexicalRelevance(input.query, neighbor) < 0.12)
+                continue;
+            const excerpt = neighbor.content.trim();
+            if (!excerpt)
+                continue;
+            selected.add(neighbor.id);
+            additions.push({
+                id: neighbor.id,
+                source: neighbor.source,
+                title: resolveKnowledgeTitle({
+                    headingPath: neighbor.heading_path, documentTitle: neighbor.document_title, content: neighbor.content,
+                    source: neighbor.source, url: neighbor.url,
+                }) ?? neighbor.source,
+                content: excerpt,
+                excerpt,
+                applicability: ['adjacent:necessary', `page:${neighbor.page_id}`],
+                url: neighbor.url,
+                version: neighbor.version ?? undefined,
+                embedding: [],
+                score: Math.max(0.1, (candidate.score ?? 0.4) * 0.7),
+            });
+            break;
+        }
+    }
+    return [...input.candidates, ...additions]
+        .sort((left, right) => (right.scoreDetails?.finalScore ?? right.score ?? 0)
+        - (left.scoreDetails?.finalScore ?? left.score ?? 0))
+        .slice(0, input.limit);
+}
+function parseJsonStrings(value) {
+    if (!value)
+        return [];
+    try {
+        const parsed = JSON.parse(value);
+        return Array.isArray(parsed) ? parsed.filter((item) => typeof item === 'string') : [];
+    }
+    catch {
+        return [];
+    }
+}
+function normalizeLookup(value) {
+    return value.normalize('NFKC').trim().toLocaleLowerCase('en-US');
+}
 export class KnowledgeEngine {
     projectRoot;
     hybridSearch;
     store;
     embedder;
+    compiler;
+    catalog;
     compatibilityRejectionCount = 0;
     lowScoreRejectionCount = 0;
     constructor(opts) {
         this.projectRoot = opts.projectRoot;
         this.embedder = opts.embedder ?? new EmbeddingProvider();
         this.store = new KnowledgeStore(join(this.knowledgeDir, 'knowledge.db'));
+        this.compiler = new KnowledgeCompiler(this.store);
+        this.catalog = new SourceCatalog(this.store, this.projectRoot);
+        this.catalog.refreshProjectProfile();
         this.hybridSearch = new KnowledgeHybridSearch(this.store, this.embedder);
         for (const suffix of ['', '-wal', '-shm']) {
             rmSync(join(this.knowledgeDir, `vectors.db${suffix}`), { force: true });
@@ -42,6 +185,20 @@ export class KnowledgeEngine {
         const dir = this.pluginDir(source.name);
         mkdirSync(dir, { recursive: true });
         const manifestPath = join(dir, "manifest.json");
+        const sourceKind = source.pluginName === 'builtin' ? 'official' : 'plugin';
+        const sourceSchemaId = `schema:${source.name}:v1`;
+        this.catalog.registerSourceSchema({
+            id: sourceSchemaId,
+            sourceId: source.name,
+            sourceKind,
+            authority: sourceKind === 'official' ? 'official' : 'maintainer',
+            versionPolicy: source.version && source.version !== 'current'
+                ? { mode: 'exact_major', expectedVersion: source.version }
+                : { mode: 'unversioned' },
+            allowedEntities: [source.name],
+            supportedTaskKinds: ['overview', 'create', 'modify', 'debug', 'test', 'build', 'migrate', 'explain'],
+            refreshStrategy: 'on_demand',
+        });
         if (existsSync(manifestPath)) {
             const existing = JSON.parse(readFileSync(manifestPath, "utf-8"));
             // Merge URLs — don't overwrite existing URLs
@@ -55,6 +212,8 @@ export class KnowledgeEngine {
                 sourceUrl: existing.sourceUrl,
                 renderJs: source.renderJs ?? existing.renderJs ?? false,
                 chunkPolicyVersion: existing.chunkPolicyVersion ?? 0,
+                sourceSchemaId,
+                sourceKind,
             }, null, 2), "utf-8");
         }
         else {
@@ -66,6 +225,8 @@ export class KnowledgeEngine {
                 version: source.version,
                 renderJs: source.renderJs ?? false,
                 chunkPolicyVersion: KNOWLEDGE_CHUNK_POLICY_VERSION,
+                sourceSchemaId,
+                sourceKind,
             }, null, 2), "utf-8");
         }
     }
@@ -129,29 +290,19 @@ export class KnowledgeEngine {
                     : [{ ...c, title: resolveKnowledgeTitle({ headingPath: c.headingPath, documentTitle: c.title, content: c.content, source: name, url: c.path ?? c.sourceUrl }) }];
             }).filter(c => Boolean(c.title));
             if (totalResult.length > 0) {
-                // Store combined chunks
-                const docId = `${name}:github-docs`;
-                const fullContent = totalResult.map(c => c.content).join('\n\n');
-                const contentHash = createHash('sha256').update(fullContent).digest('hex');
+                const fullContent = totalResult.map(c => `# ${c.title ?? name}\n\n${c.content}`).join('\n\n');
                 const ghPath = Array.isArray(sourceConfig.githubDocsPath) ? sourceConfig.githubDocsPath[0] : sourceConfig.githubDocsPath;
-                const doc = {
-                    id: docId, source: name, url: `https://github.com/${sourceConfig.githubRepo}/tree/main/${ghPath}`,
-                    title: `${name} docs (GitHub)`,
-                    version: sourceConfig.version,
-                    contentHash,
-                };
-                const sqliteChunks = totalResult.map((c, i) => {
-                    const chunkId = `${docId}:chunk-${String(i + 1).padStart(3, '0')}`;
-                    return {
-                        id: chunkId, docId, index: i,
-                        headingPath: c.headingPath ?? c.title ?? name,
-                        content: c.content,
-                        codeBlocks: c.codeBlocks,
-                        tokenCount: estimateKnowledgeTokens(c.content),
-                    };
-                });
+                let compiled;
                 try {
-                    await this.replaceDocument(doc, sqliteChunks, options.deferVectorBackfill);
+                    compiled = await this.replaceCompiledSource({
+                        source: name,
+                        manifest,
+                        sourceUri: `https://github.com/${sourceConfig.githubRepo}/tree/main/${ghPath}`,
+                        title: `${name} docs (GitHub)`,
+                        markdown: fullContent,
+                        version: sourceConfig.version,
+                        deferVectorBackfill: options.deferVectorBackfill,
+                    });
                     this.store.deactivateDocument(`${name}:local-policy-v${KNOWLEDGE_CHUNK_POLICY_VERSION}`);
                 }
                 catch (error) {
@@ -179,10 +330,10 @@ export class KnowledgeEngine {
                     ...manifest,
                     urls: manifest.urls,
                     fetchedAt: Date.now(),
-                    chunkCount: totalResult.length,
+                    chunkCount: compiled.chunks.length,
                     chunkPolicyVersion: KNOWLEDGE_CHUNK_POLICY_VERSION,
                 }, null, 2), "utf-8");
-                return [{ source: name, added: totalResult.length, updated: 0, errors: [] }];
+                return [{ source: name, added: compiled.chunks.length, updated: 0, errors: [] }];
             }
             // L1+L2 empty — fall through to L3 HTML
         }
@@ -194,14 +345,19 @@ export class KnowledgeEngine {
             const errors = [];
             let added = 0;
             try {
-                const html = manifest.renderJs
-                    ? await fetchHtmlBrowser(url)
-                    : await fetchHtmlSmart(url);
-                const markdown = htmlToMarkdown(html);
-                const chunks = sliceByHeadings(markdown);
+                const raw = await fetchHtmlKnowledgeSource(url, { renderJs: manifest.renderJs });
+                const compiled = await this.replaceCompiledSource({
+                    source: name,
+                    manifest,
+                    sourceUri: raw.sourceUri,
+                    markdown: raw.markdown,
+                    version: manifest.version,
+                    deferVectorBackfill: options.deferVectorBackfill,
+                });
+                const chunks = compiled.chunks;
                 const chunkFiles = chunks.map((chunk) => ({
                     id: `chunk-${String(nextChunkIndex++).padStart(3, "0")}`,
-                    title: chunk.title,
+                    title: chunk.headingBreadcrumbs.at(-1) ?? compiled.page.title,
                     section: new URL(url).pathname,
                     content: chunk.content,
                     sourceUrl: url,
@@ -210,26 +366,6 @@ export class KnowledgeEngine {
                 for (const cf of chunkFiles) {
                     writeFileSync(join(dir, `${cf.id}.json`), JSON.stringify(cf, null, 2), "utf-8");
                 }
-                // Also populate SQLite store
-                const docId = `${name}:${new URL(url).pathname}`;
-                const contentHash = createHash('sha256').update(markdown).digest('hex');
-                const doc = {
-                    id: docId, source: name, url,
-                    title: chunks[0]?.title ?? name,
-                    version: manifest.version,
-                    contentHash,
-                };
-                const sqliteChunks = chunks.map((c, i) => {
-                    const chunkId = `${docId}:chunk-${String(i + 1).padStart(3, '0')}`;
-                    return {
-                        id: chunkId, docId, index: i,
-                        headingPath: c.headingPath ?? c.title,
-                        content: c.content,
-                        codeBlocks: c.codeBlocks,
-                        tokenCount: estimateKnowledgeTokens(c.content),
-                    };
-                });
-                await this.replaceDocument(doc, sqliteChunks, options.deferVectorBackfill);
                 added = chunkFiles.length;
                 totalChunkCount += added;
             }
@@ -384,23 +520,153 @@ export class KnowledgeEngine {
         const embeddings = await this.embedder.embedBatch(texts);
         this.store.replaceDocument({ doc, chunks, embeddings });
     }
+    async replaceCompiledSource(input) {
+        const sourceSchemaId = input.manifest.sourceSchemaId ?? `schema:${input.source}:v1`;
+        const sourceKind = input.manifest.sourceKind ?? 'plugin';
+        if (!this.store.getSourceSchema(sourceSchemaId)) {
+            this.catalog.registerSourceSchema({
+                id: sourceSchemaId,
+                sourceId: input.source,
+                sourceKind,
+                authority: sourceKind === 'official' ? 'official' : 'maintainer',
+                versionPolicy: input.version && input.version !== 'current'
+                    ? { mode: 'exact_major', expectedVersion: input.version }
+                    : { mode: 'unversioned' },
+                allowedEntities: [input.source],
+                supportedTaskKinds: ['overview', 'create', 'modify', 'debug', 'test', 'build', 'migrate', 'explain'],
+                refreshStrategy: 'on_demand',
+            });
+        }
+        const inputHash = createHash('sha256').update(input.markdown).digest('hex');
+        const refresh = this.catalog.beginRefresh(input.source, inputHash);
+        try {
+            const compiled = this.compiler.compile({
+                projectRoot: this.projectRoot,
+                sourceId: input.source,
+                sourceSchemaId,
+                sourceKind,
+                sourceUri: input.sourceUri,
+                markdown: input.markdown,
+                title: input.title,
+                version: input.version,
+            });
+            let embeddings = [];
+            if (!input.deferVectorBackfill) {
+                const warmed = await this.embedder.warmup();
+                if (!warmed) {
+                    throw this.embedder.getLastFailure?.()
+                        ?? new Error(this.embedder.getLastError() ?? 'Knowledge embedding warmup failed');
+                }
+                embeddings = await this.embedder.embedBatch(compiled.chunks.map(chunk => chunk.embeddingText));
+                if (embeddings.length !== compiled.chunks.length) {
+                    throw new Error(`Knowledge embedding count mismatch: ${embeddings.length}/${compiled.chunks.length}`);
+                }
+            }
+            this.store.replaceCompiledPage(compiled.page, compiled.chunks);
+            if (embeddings.length > 0) {
+                this.store.insertEmbeddingBatch(compiled.chunks.map((chunk, index) => ({
+                    id: chunk.id,
+                    embedding: embeddings[index],
+                })));
+            }
+            this.store.deactivateSourceDocuments(input.source, compiled.page.id);
+            const vectors = embeddings.length;
+            const coverage = compiled.chunks.length > 0 ? vectors / compiled.chunks.length : 0;
+            this.catalog.activateSource({
+                schemaId: sourceSchemaId,
+                version: input.version === 'current' ? undefined : input.version,
+                contentHash: compiled.page.contentHash,
+                readiness: coverage >= 0.95
+                    ? {
+                        status: 'ready', activeContentHash: compiled.page.contentHash,
+                        documents: 1, chunks: compiled.chunks.length, vectors, coverage,
+                        lastSuccessfulRefresh: Date.now(), lastAttempt: Date.now(),
+                    }
+                    : {
+                        status: 'degraded', activeContentHash: compiled.page.contentHash,
+                        documents: 1, chunks: compiled.chunks.length, vectors, coverage,
+                        lastAttempt: Date.now(),
+                        degradation: {
+                            reason: 'vector_backfill_pending', category: 'vector_degraded', recoverable: true,
+                            previousHealthyRevisionRetained: false,
+                        },
+                    },
+                entities: [input.source, ...compiled.page.entities],
+                symbols: compiled.page.symbols,
+                refreshJobId: refresh.id,
+            });
+            return compiled;
+        }
+        catch (error) {
+            this.catalog.failRefresh(refresh.id, error);
+            throw error;
+        }
+    }
+    compileDependency(packageName) {
+        return compileDependencySource({
+            projectRoot: this.projectRoot,
+            packageName,
+            store: this.store,
+            deferVectorBackfill: true,
+        });
+    }
+    async getProjectProfile() {
+        return this.store.getProjectProfile(this.projectRoot) ?? this.catalog.refreshProjectProfile();
+    }
+    async getSourceCatalog() {
+        return this.catalog.getCatalog();
+    }
+    async getCoverageGaps() {
+        return this.store.listCoverageGaps(this.projectRoot);
+    }
     async search(query, options) {
         return (await this.searchWithStatus(query, options)).entries;
     }
     async searchWithStatus(query, options) {
         const limit = options?.limit ?? 10;
-        const availableSources = this.store.getDistinctSources();
-        const selection = selectKnowledgeSources({
+        const activeSkill = options?.activeSkill ?? options?.projectSignals?.activeSkill;
+        const taskAware = Boolean(options?.action || options?.knowledgeNeeds?.length);
+        const projectProfile = this.catalog.refreshProjectProfile({
+            activeSkills: activeSkill ? [normalizeActiveSkill(activeSkill)] : [],
+        });
+        const selectionInput = () => ({
             query,
-            availableSources,
+            availableSources: this.store.getDistinctSources(),
             explicitSources: options?.sources,
             projectRoot: this.projectRoot,
             projectSignals: {
                 ...options?.projectSignals,
-                activeSkill: options?.activeSkill ?? options?.projectSignals?.activeSkill,
+                activeSkill,
                 relevantFiles: options?.relevantFiles ?? options?.projectSignals?.relevantFiles,
             },
+            projectProfile: taskAware ? projectProfile : null,
+            catalog: taskAware ? this.catalog.getCatalog() : null,
+            action: options?.action,
+            knowledgeNeeds: options?.knowledgeNeeds,
         });
+        let selection = selectKnowledgeSources(selectionInput());
+        const compileFailures = [];
+        for (const packageName of taskAware
+            ? dependencyPackagesForMissingNeeds(selection.missingNeeds, projectProfile.dependencies)
+            : []) {
+            try {
+                this.compileDependency(packageName);
+            }
+            catch (error) {
+                const reason = error instanceof Error ? error.message : String(error);
+                this.catalog.recordCoverageGap({
+                    sourceId: `dependency:${packageName}`,
+                    kind: 'compile_failure',
+                    entity: packageName,
+                    reason,
+                });
+                compileFailures.push({ sourceId: `dependency:${packageName}`, kind: 'compile_failure', entity: packageName, reason });
+            }
+        }
+        if (selection.missingNeeds.length > 0)
+            selection = selectKnowledgeSources(selectionInput());
+        for (const gap of selection.coverageGaps)
+            this.catalog.recordCoverageGap(gap);
         this.compatibilityRejectionCount += selection.rejectedSources.length;
         const sources = selection.sources;
         const budgetTokens = Math.max(0, options?.budgetTokens ?? DEFAULT_KNOWLEDGE_TOKEN_BUDGET);
@@ -409,17 +675,39 @@ export class KnowledgeEngine {
         const readinessWarnings = this.store.getSourceReadiness()
             .filter(row => sources.includes(row.source) && row.status !== 'ready')
             .map(row => `knowledge-source-degraded:${row.source}:${row.failure_category ?? 'unknown'}`);
+        const coverageWarnings = selection.coverageGaps.length > 0 ? ['knowledge_source_missing'] : [];
+        if (sources.length === 0) {
+            return {
+                entries: [],
+                warnings: [...coverageWarnings, ...readinessWarnings, 'knowledge_insufficient'],
+                budgetTokens,
+                tokenCount: 0,
+                truncatedCount: 0,
+                rejectedIncompatibleSources: selection.rejectedSources.length,
+                retrievalDiagnostics: {
+                    selectedSources: [],
+                    sourceSignals: selection.signals,
+                    sourceRejections: selection.rejections,
+                    coverageGaps: [...selection.coverageGaps, ...compileFailures],
+                    compatibleChunks: 0,
+                    hybridCandidates: 0,
+                    qualityRejected: {},
+                },
+            };
+        }
         const allChunks = this.store.listChunks({ sources });
         const chunks = allChunks.filter(chunk => chunk.chunk_policy_version === KNOWLEDGE_CHUNK_POLICY_VERSION
             && isKnowledgeVersionCompatible(chunk.source, chunk.version, projectVersions));
         const rejectedByQualityGate = allChunks.length - chunks.length;
         if (chunks.length === 0) {
             return {
-                entries: [], warnings: [...readinessWarnings, 'knowledge_insufficient'], budgetTokens, tokenCount: 0, truncatedCount: 0,
+                entries: [], warnings: [...coverageWarnings, ...readinessWarnings, 'knowledge_insufficient'], budgetTokens, tokenCount: 0, truncatedCount: 0,
                 rejectedIncompatibleSources: selection.rejectedSources.length + rejectedByQualityGate,
                 retrievalDiagnostics: {
                     selectedSources: sources,
                     sourceSignals: selection.signals,
+                    sourceRejections: selection.rejections,
+                    coverageGaps: [...selection.coverageGaps, ...compileFailures],
                     compatibleChunks: 0,
                     hybridCandidates: 0,
                     qualityRejected: {},
@@ -445,12 +733,14 @@ export class KnowledgeEngine {
             const packed = this.packEntries(candidates, chunks, budgetTokens);
             return {
                 ...packed,
-                warnings: readinessWarnings,
+                warnings: [...coverageWarnings, ...readinessWarnings],
                 budgetTokens,
                 rejectedIncompatibleSources: selection.rejectedSources.length + rejectedByQualityGate,
                 retrievalDiagnostics: {
                     selectedSources: sources,
                     sourceSignals: selection.signals,
+                    sourceRejections: selection.rejections,
+                    coverageGaps: [...selection.coverageGaps, ...compileFailures],
                     compatibleChunks: chunks.length,
                     hybridCandidates: candidates.length,
                     qualityRejected: {},
@@ -462,19 +752,32 @@ export class KnowledgeEngine {
             bm25Results = this.store.searchBM25(query, limit * 3, sources);
         }
         catch {
-            // Vector retrieval can still answer malformed FTS queries.
+            // Retry below with a bounded literal-term query.
         }
-        const hybrid = await this.hybridSearch.searchWithStatus(query, chunks, bm25Results, limit, sources, options?.minScore);
+        if (bm25Results.length === 0) {
+            const safeQuery = buildSafeFtsQuery(query);
+            if (safeQuery) {
+                try {
+                    bm25Results = this.store.searchBM25(safeQuery, limit * 3, sources);
+                }
+                catch {
+                    // Vector retrieval can still answer when FTS cannot parse any bounded fallback.
+                }
+            }
+        }
+        const hybrid = await this.hybridSearch.searchWithStatus(query, chunks, bm25Results, limit, sources, options?.minScore, options?.knowledgeNeeds);
         this.lowScoreRejectionCount += hybrid.rejectedLowScore;
         const chunkMap = new Map(chunks.map(chunk => [chunk.id, chunk]));
         const qualityRejected = {};
-        const candidates = hybrid.results.flatMap(result => {
+        const seedResults = hybrid.results.flatMap(result => {
             const chunk = chunkMap.get(result.id);
             const quality = evaluateKnowledgeQuality({
                 query,
                 chunk,
                 retrievalScore: result.score,
                 versionCompatible: isKnowledgeVersionCompatible(chunk.source, chunk.version, projectVersions),
+                action: options?.action,
+                knowledgeNeeds: options?.knowledgeNeeds,
             });
             if (!quality.eligible) {
                 const reason = quality.reason ?? 'unknown';
@@ -493,40 +796,119 @@ export class KnowledgeEngine {
                     version: chunk.version ?? undefined,
                     embedding: [],
                     score: Math.round(quality.score * 10000) / 10000,
+                    scoreDetails: {
+                        ...result.scoreDetails,
+                        qualityScore: Math.round(quality.score * 10000) / 10000,
+                        finalScore: Math.round((0.65 * result.scoreDetails.finalScore + 0.35 * quality.score) * 10000) / 10000,
+                        versionCompatible: true,
+                    },
                 }];
         });
+        const graphExpansions = expandKnowledgeEvidenceGraph({
+            query,
+            seeds: seedResults.map(result => ({ id: result.id, score: result.scoreDetails?.finalScore ?? result.score ?? 0 })),
+            chunks,
+            knowledgeNeeds: options?.knowledgeNeeds,
+            limit: Math.max(0, limit - Math.min(limit, seedResults.length)),
+        });
+        const graphCandidates = graphExpansions.flatMap(expansion => {
+            const chunk = chunkMap.get(expansion.id);
+            if (!chunk)
+                return [];
+            const quality = evaluateKnowledgeQuality({
+                query,
+                chunk,
+                retrievalScore: expansion.score,
+                versionCompatible: isKnowledgeVersionCompatible(chunk.source, chunk.version, projectVersions),
+                action: options?.action,
+                knowledgeNeeds: options?.knowledgeNeeds,
+            });
+            if (!quality.eligible)
+                return [];
+            return [{
+                    id: chunk.id,
+                    source: chunk.source,
+                    title: resolveKnowledgeTitle({
+                        headingPath: chunk.heading_path, documentTitle: chunk.document_title, content: chunk.content,
+                        source: chunk.source, url: chunk.url,
+                    }) ?? chunk.source,
+                    content: quality.excerpt,
+                    excerpt: quality.excerpt,
+                    matchingTerms: quality.matchingTerms,
+                    applicability: [...quality.applicability, `graph:${expansion.relation}`],
+                    url: chunk.url,
+                    version: chunk.version ?? undefined,
+                    embedding: [],
+                    score: expansion.score,
+                    scoreDetails: {
+                        profileVersion: KNOWLEDGE_RETRIEVAL_SCORING_PROFILE.version,
+                        channelRanks: {},
+                        rawScore: expansion.score, normalizedScore: expansion.score, pageScore: expansion.score,
+                        qualityScore: quality.score, finalScore: expansion.score,
+                        authority: authorityFromProvenance(chunk.provenance, chunk.source),
+                        versionCompatible: true, provenanceComplete: provenanceComplete(chunk.provenance),
+                        graphRelation: {
+                            seedId: expansion.seedId, type: expansion.relation, evidence: expansion.relationEvidence,
+                        },
+                    },
+                }];
+        });
+        const rankedCandidates = dedupeKnowledgeEntries([...seedResults, ...graphCandidates])
+            .sort((left, right) => (right.scoreDetails?.finalScore ?? right.score ?? 0)
+            - (left.scoreDetails?.finalScore ?? left.score ?? 0));
+        const candidates = addNecessaryAdjacentEntries({
+            candidates: rankedCandidates, chunks, query, knowledgeNeeds: options?.knowledgeNeeds ?? [], limit,
+        });
         const packed = this.packEntries(candidates, chunks, budgetTokens);
         return {
             ...packed,
-            warnings: [...hybrid.warnings, ...readinessWarnings, ...(candidates.length === 0 ? ['knowledge_insufficient'] : [])],
+            warnings: [...hybrid.warnings, ...coverageWarnings, ...readinessWarnings, ...(candidates.length === 0 ? ['knowledge_insufficient'] : [])],
             budgetTokens,
             rejectedLowScore: hybrid.rejectedLowScore,
             rejectedIncompatibleSources: selection.rejectedSources.length + rejectedByQualityGate,
             retrievalDiagnostics: {
                 selectedSources: sources,
                 sourceSignals: selection.signals,
+                sourceRejections: selection.rejections,
+                coverageGaps: [...selection.coverageGaps, ...compileFailures],
                 compatibleChunks: chunks.length,
                 hybridCandidates: hybrid.results.length,
                 qualityRejected,
+                channelCandidateCounts: hybrid.channelCandidateCounts,
+                graphExpansions: graphCandidates.length,
             },
         };
     }
     packEntries(candidates, chunks, budgetTokens) {
         const chunkMap = new Map(chunks.map(chunk => [chunk.id, chunk]));
         const entries = [];
+        const sourceTokens = new Map();
+        const pageTokens = new Map();
         let tokenCount = 0;
         let truncatedCount = 0;
         for (const entry of candidates) {
+            const chunk = chunkMap.get(entry.id);
             const entryTokens = (entry.excerpt
                 ? estimateKnowledgeTokens(entry.content)
-                : chunkMap.get(entry.id)?.token_count ?? estimateKnowledgeTokens(entry.content))
+                : chunk?.token_count ?? estimateKnowledgeTokens(entry.content))
                 + estimateKnowledgeTokens(entry.title);
-            if (tokenCount + entryTokens > budgetTokens) {
+            const structuredPage = chunk?.page_id ?? null;
+            const source = chunk?.source ?? entry.source;
+            const sourceLimit = Math.max(entryTokens, Math.floor(budgetTokens * 0.60));
+            const pageLimit = Math.max(entryTokens, Math.floor(budgetTokens * 0.40));
+            const overSourceBudget = Boolean(structuredPage)
+                && (sourceTokens.get(source) ?? 0) + entryTokens > sourceLimit;
+            const overPageBudget = Boolean(structuredPage)
+                && (pageTokens.get(structuredPage) ?? 0) + entryTokens > pageLimit;
+            if (tokenCount + entryTokens > budgetTokens || overSourceBudget || overPageBudget) {
                 truncatedCount++;
                 continue;
             }
             entries.push(entry);
             tokenCount += entryTokens;
+            sourceTokens.set(source, (sourceTokens.get(source) ?? 0) + entryTokens);
+            if (structuredPage)
+                pageTokens.set(structuredPage, (pageTokens.get(structuredPage) ?? 0) + entryTokens);
         }
         return { entries, tokenCount, truncatedCount };
     }