@devflow-tools/knowledge-engine 0.17.8 → 0.18.1

package/dist/knowledge-engine.js

@@ -14,13 +14,16 @@ import { estimateKnowledgeTokens as estimateTokens, lexicalRelevance, resolveKno
 import { detectProjectKnowledgeVersions, expectedVersionForSource, isKnowledgeVersionCompatible, } from './source-readiness.js';
 import { evaluateKnowledgeQuality } from './knowledge-quality.js';
 import { KnowledgeCompiler } from './knowledge-compiler.js';
-import { SourceCatalog } from './source-catalog.js';
+import { buildProjectKnowledgeProfile, SourceCatalog } from './source-catalog.js';
 import { compileDependencySource } from './dependency-source-compiler.js';
 import { expandKnowledgeEvidenceGraph } from './evidence-graph.js';
 import { KNOWLEDGE_RETRIEVAL_SCORING_PROFILE } from './retrieval-scoring-profile.js';
 import { planKnowledgeQuery } from './knowledge-query-planner.js';
 import { emitSemanticControlLog } from '@devflow-tools/telemetry';
 import { createLexicalOnlyEmbeddingProvider } from './embedding-provider.js';
+import { requestValidatedEmbeddingBatch } from '@devflow-tools/provider-runtime';
+import { KnowledgeSourceService } from './knowledge-source-service.js';
+import { KnowledgeQueryService, } from './knowledge-query-service.js';
 export const DEFAULT_KNOWLEDGE_TOKEN_BUDGET = 3000;
 export const estimateKnowledgeTokens = estimateTokens;
 function normalizeActiveSkill(value) {
@@ -45,13 +48,16 @@ function dependencyPackagesForMissingNeeds(needs, dependencies) {
             return packageName ? [packageName] : [];
         }))];
 }
-function buildSafeFtsQuery(value) {
+function compileKnowledgeFtsQuery(value) {
     const terms = value.normalize('NFKC')
         .match(/[\p{L}\p{N}_@.-]{2,}/gu)
         ?.map(term => term.replace(/^\.+|\.+$/gu, '').replace(/"/gu, ''))
         .filter(Boolean)
         .slice(0, 24) ?? [];
-    return [...new Set(terms)].map(term => `"${term}"`).join(' OR ');
+    const uniqueTerms = [...new Set(terms)];
+    return uniqueTerms.length > 0
+        ? uniqueTerms.map(term => `"${term}"`).join(' OR ')
+        : undefined;
 }
 function dedupeKnowledgeEntries(entries) {
     const seen = new Set();
@@ -171,27 +177,36 @@ function normalizeLookup(value) {
 }
 export class KnowledgeEngine {
     projectRoot;
+    projectId;
+    knowledgeAssetRoot;
     hybridSearch;
     store;
     embedder;
     compiler;
     catalog;
+    sources;
+    query;
     compatibilityRejectionCount = 0;
     lowScoreRejectionCount = 0;
     constructor(opts) {
         this.projectRoot = opts.projectRoot;
+        this.projectId = opts.projectId;
+        this.knowledgeAssetRoot = opts.dataSource?.assetRoot ?? join(opts.projectRoot, '.devflow', 'knowledge');
         this.embedder = opts.embedder ?? createLexicalOnlyEmbeddingProvider(opts.projectRoot);
-        this.store = new KnowledgeStore(join(this.knowledgeDir, 'knowledge.db'));
-        this.compiler = new KnowledgeCompiler(this.store);
-        this.catalog = new SourceCatalog(this.store, this.projectRoot);
-        this.catalog.refreshProjectProfile();
+        this.store = new KnowledgeStore(opts.dataSource?.databasePath ?? join(this.knowledgeDir, 'knowledge.db'), this.embedder.getCapability('knowledge_retrieval').profile);
+        this.compiler = new KnowledgeCompiler();
+        this.catalog = new SourceCatalog(this.store, this.projectRoot, this.projectId);
+        this.sources = new KnowledgeSourceService(this.store, this.catalog, this.projectRoot);
+        this.query = new KnowledgeQueryService({
+            searchReadOnly: (query, options) => this.searchReadOnlyInternal(query, options),
+        });
         this.hybridSearch = new KnowledgeHybridSearch(this.store, this.embedder);
         for (const suffix of ['', '-wal', '-shm']) {
             rmSync(join(this.knowledgeDir, `vectors.db${suffix}`), { force: true });
         }
     }
     get knowledgeDir() {
-        return join(this.projectRoot, ".devflow", "knowledge");
+        return this.knowledgeAssetRoot;
     }
     pluginDir(pluginName) {
         return join(this.knowledgeDir, pluginName);
@@ -199,7 +214,7 @@ export class KnowledgeEngine {
     ensureManifestSourceSchema(name, manifest, version) {
         const sourceSchemaId = manifest.sourceSchemaId ?? `schema:${name}:v1`;
         const sourceKind = manifest.sourceKind ?? (manifest.pluginName === 'builtin' ? 'official' : 'plugin');
-        this.catalog.registerSourceSchema({
+        this.sources.registerSourceSchema({
             id: sourceSchemaId,
             sourceId: name,
             sourceKind,
@@ -214,17 +229,18 @@ export class KnowledgeEngine {
     resolvePersistedSourceVersion(name, manifest, existingVersion) {
         if (manifest.version && manifest.version !== 'current')
             return manifest.version;
-        const expectedVersion = expectedVersionForSource(name, detectProjectKnowledgeVersions(this.projectRoot));
+        const sourceKind = manifest.sourceKind ?? (manifest.pluginName === 'builtin' ? 'official' : 'plugin');
+        const expectedVersion = expectedVersionForSource(name, detectProjectKnowledgeVersions(this.projectRoot), sourceKind);
         const persistedVersion = existingVersion && existingVersion !== 'current' ? existingVersion : undefined;
         return expectedVersion ?? persistedVersion;
     }
-    async registerSource(source) {
+    async registerSource(source, operationTimestamp = Date.now()) {
         const dir = this.pluginDir(source.name);
         mkdirSync(dir, { recursive: true });
         const manifestPath = join(dir, "manifest.json");
         const sourceKind = source.pluginName === 'builtin' ? 'official' : 'plugin';
         const sourceSchemaId = `schema:${source.name}:v1`;
-        this.catalog.registerSourceSchema({
+        this.sources.registerSourceSchema({
             id: sourceSchemaId,
             sourceId: source.name,
             sourceKind,
@@ -235,7 +251,7 @@ export class KnowledgeEngine {
             allowedEntities: [source.name],
             supportedTaskKinds: ['overview', 'create', 'modify', 'debug', 'test', 'build', 'migrate', 'explain'],
             refreshStrategy: 'on_demand',
-        });
+        }, operationTimestamp);
         if (existsSync(manifestPath)) {
             const existing = JSON.parse(readFileSync(manifestPath, "utf-8"));
             // Merge URLs — don't overwrite existing URLs
@@ -327,7 +343,7 @@ export class KnowledgeEngine {
                 })),
                 ...gitResult,
             ].flatMap(c => {
-                const chunks = sliceByHeadings(c.content, { minTokens: 1, targetTokens: 1200, maxTokens: 3000 });
+                const chunks = sliceByHeadings(c.content, { targetTokens: 1200, maxTokens: 3000 });
                 return chunks.length > 0
                     ? chunks.map(chunk => ({ ...c, title: resolveKnowledgeTitle({ headingPath: chunk.headingPath, documentTitle: c.title, content: chunk.content, source: name, url: c.path ?? c.sourceUrl }), content: chunk.content, headingPath: chunk.headingPath, codeBlocks: chunk.codeBlocks }))
                     : [{ ...c, title: resolveKnowledgeTitle({ headingPath: c.headingPath, documentTitle: c.title, content: c.content, source: name, url: c.path ?? c.sourceUrl }) }];
@@ -357,19 +373,6 @@ export class KnowledgeEngine {
                             errors: [error.message],
                         }];
                 }
-                // Also write JSON files for backward compat
-                this.clearChunkFiles(dir);
-                for (const [i, c] of totalResult.entries()) {
-                    const cf = {
-                        id: `chunk-${String(i + 1).padStart(3, "0")}`,
-                        title: c.title ?? name,
-                        section: c.path,
-                        content: c.content,
-                        sourceUrl: `https://github.com/${sourceConfig.githubRepo}`,
-                        fetchedAt: Date.now(),
-                    };
-                    writeFileSync(join(dir, `${cf.id}.json`), JSON.stringify(cf, null, 2), "utf-8");
-                }
                 writeFileSync(manifestPath, JSON.stringify({
                     ...manifest,
                     urls: manifest.urls,
@@ -406,18 +409,6 @@ export class KnowledgeEngine {
                     version: manifest.version,
                     deferVectorBackfill: options.deferVectorBackfill,
                 });
-                this.clearChunkFiles(dir);
-                for (const [index, chunk] of compiled.chunks.entries()) {
-                    const file = {
-                        id: `chunk-${String(index + 1).padStart(3, '0')}`,
-                        title: chunk.headingBreadcrumbs.at(-1) ?? compiled.page.title,
-                        section: chunk.headingBreadcrumbs.join(' > '),
-                        content: chunk.content,
-                        sourceUrl: fetchedDocuments[0].url,
-                        fetchedAt: Date.now(),
-                    };
-                    writeFileSync(join(dir, `${file.id}.json`), JSON.stringify(file, null, 2), 'utf8');
-                }
                 const successful = results.find(result => result.errors.length === 0);
                 if (successful)
                     successful.added = compiled.chunks.length;
@@ -449,139 +440,33 @@ export class KnowledgeEngine {
         await closeBrowser();
         return results;
     }
-    reingestLocalSources(source) {
-        if (!existsSync(this.knowledgeDir))
-            return [];
-        const sources = source
-            ? [source]
-            : readdirSync(this.knowledgeDir, { withFileTypes: true })
-                .filter(entry => entry.isDirectory())
-                .map(entry => entry.name)
-                .sort();
-        const results = [];
-        for (const name of sources) {
-            const dir = this.pluginDir(name);
-            const manifestPath = join(dir, 'manifest.json');
-            if (!existsSync(manifestPath))
-                continue;
-            try {
-                const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
-                let chunkFiles = readdirSync(dir)
-                    .filter(file => /^chunk-\d+\.json$/u.test(file))
-                    .sort()
-                    .flatMap(file => {
-                    try {
-                        const chunk = JSON.parse(readFileSync(join(dir, file), 'utf8'));
-                        return typeof chunk.content === 'string' && chunk.content.trim()
-                            ? [{
-                                    title: chunk.title?.trim() || name,
-                                    content: chunk.content,
-                                    sourceUrl: chunk.sourceUrl || manifest.sourceUrl || manifest.urls?.[0] || '',
-                                }]
-                            : [];
-                    }
-                    catch {
-                        return [];
-                    }
-                });
-                if (chunkFiles.length === 0) {
-                    chunkFiles = this.store.getDocChunks(name, 100_000).flatMap(chunk => typeof chunk.content === 'string' && chunk.content.trim()
-                        ? [{
-                                title: String(chunk.heading_path || name),
-                                content: chunk.content,
-                                sourceUrl: manifest.sourceUrl || manifest.urls?.[0] || '',
-                            }]
-                        : []);
-                }
-                if (chunkFiles.length === 0) {
-                    results.push({ source: name, documentsRetired: 0, chunks: 0, errors: ['no local chunks'] });
-                    continue;
-                }
-                const combined = chunkFiles.map(chunk => `# ${chunk.title}\n\n${chunk.content}`).join('\n\n');
-                const existingVersion = this.store.listChunks({ sources: [name], limit: 1 })[0]?.version;
-                const version = this.resolvePersistedSourceVersion(name, manifest, existingVersion);
-                const sourceSchemaId = this.ensureManifestSourceSchema(name, manifest, version);
-                const url = chunkFiles.find(chunk => chunk.sourceUrl)?.sourceUrl
-                    || manifest.urls?.[0]
-                    || `devflow://knowledge/${name}`;
-                const compiled = this.compiler.compileAndStore({
-                    projectRoot: this.projectRoot,
-                    sourceId: name,
-                    sourceSchemaId,
-                    sourceKind: manifest.sourceKind ?? (manifest.pluginName === 'builtin' ? 'official' : 'plugin'),
-                    sourceUri: url,
-                    markdown: combined,
-                    title: `${name} local knowledge`,
-                    version,
-                });
-                const documentsRetired = this.store.deactivateSourceDocuments(name, compiled.page.id);
-                const storedChunks = this.store.listChunks({ sources: [name] });
-                const vectors = this.store.countEmbeddings(name);
-                const coverage = storedChunks.length > 0 ? vectors / storedChunks.length : 0;
-                const entities = [...new Set([
-                        name,
-                        ...storedChunks.flatMap(chunk => parseJsonStrings(chunk.entities)),
-                    ])];
-                const symbols = [...new Set(storedChunks.flatMap(chunk => [
-                        ...parseJsonStrings(chunk.symbols),
-                        ...parseJsonStrings(chunk.api_identifiers),
-                    ]))];
-                this.catalog.activateSource({
-                    schemaId: sourceSchemaId,
-                    version,
-                    contentHash: compiled.page.contentHash,
-                    readiness: coverage >= 0.95
-                        ? {
-                            status: 'ready', activeContentHash: compiled.page.contentHash,
-                            documents: 1, chunks: storedChunks.length, vectors, coverage,
-                            lastSuccessfulRefresh: Date.now(), lastAttempt: Date.now(),
-                        }
-                        : {
-                            status: 'degraded', activeContentHash: compiled.page.contentHash,
-                            documents: 1, chunks: storedChunks.length, vectors, coverage,
-                            lastAttempt: Date.now(),
-                            degradation: {
-                                reason: 'vector_backfill_pending', category: 'vector_degraded', recoverable: true,
-                                previousHealthyRevisionRetained: false,
-                            },
-                        },
-                    entities,
-                    symbols,
-                });
-                this.clearChunkFiles(dir);
-                for (const [index, chunk] of compiled.chunks.entries()) {
-                    const file = {
-                        id: `chunk-${String(index + 1).padStart(3, '0')}`,
-                        title: chunk.headingBreadcrumbs.at(-1) ?? compiled.page.title,
-                        section: chunk.headingBreadcrumbs.join(' > '),
-                        content: chunk.content,
-                        sourceUrl: url,
-                        fetchedAt: Date.now(),
-                    };
-                    writeFileSync(join(dir, `${file.id}.json`), JSON.stringify(file, null, 2), 'utf8');
-                }
-                writeFileSync(manifestPath, JSON.stringify({
-                    ...manifest,
-                    version: version ?? manifest.version,
-                    fetchedAt: Date.now(),
-                    chunkCount: compiled.chunks.length,
-                    chunkPolicyVersion: KNOWLEDGE_CHUNK_POLICY_VERSION,
-                }, null, 2), 'utf8');
-                results.push({ source: name, documentsRetired, chunks: compiled.chunks.length, errors: [] });
-            }
-            catch (error) {
-                results.push({
-                    source: name,
-                    documentsRetired: 0,
-                    chunks: 0,
-                    errors: [error instanceof Error ? error.message : String(error)],
-                });
-            }
-        }
-        this.store.refreshSourceReadiness();
-        return results;
+    async ingestSourceMarkdown(input) {
+        const operationTimestamp = input.operationTimestamp ?? Date.now();
+        const manifestPath = join(this.pluginDir(input.source), 'manifest.json');
+        if (!existsSync(manifestPath))
+            throw new Error(`KNOWLEDGE_SOURCE_NOT_REGISTERED:${input.source}`);
+        const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
+        const compiled = await this.replaceCompiledSource({
+            source: input.source,
+            manifest,
+            sourceUri: input.sourceUri,
+            title: input.title,
+            markdown: input.markdown,
+            version: this.resolvePersistedSourceVersion(input.source, manifest, input.version ?? manifest.version),
+            deferVectorBackfill: input.deferVectorBackfill,
+            operationTimestamp,
+        });
+        writeFileSync(manifestPath, JSON.stringify({
+            ...manifest,
+            version: this.resolvePersistedSourceVersion(input.source, manifest, input.version ?? manifest.version),
+            fetchedAt: operationTimestamp,
+            chunkCount: compiled.chunks.length,
+            chunkPolicyVersion: KNOWLEDGE_CHUNK_POLICY_VERSION,
+        }, null, 2), 'utf8');
+        return compiled;
     }
     activatePersistedLocalSources(source) {
+        const operationTimestamp = Date.now();
         if (!existsSync(this.knowledgeDir))
             return [];
         const activeCatalogSources = new Map(this.catalog.getCatalog().sources.map(entry => [entry.sourceId, entry]));
@@ -613,27 +498,20 @@ export class KnowledgeEngine {
                 if (currentPolicy && currentSemantic && currentCatalog)
                     continue;
                 if (!currentPolicy || !currentSemantic) {
-                    const migrated = this.reingestLocalSources(name)[0];
-                    const migratedChunks = this.store.countChunks(name);
-                    const migratedVectors = this.store.countEmbeddings(name);
-                    const errors = migrated?.errors ?? ['local policy migration produced no result'];
-                    if (errors.length === 0) {
-                        const activated = this.catalog.getCatalog().sources.find(entry => entry.sourceId === name);
-                        if (activated)
-                            activeCatalogSources.set(name, activated);
-                    }
                     results.push({
                         source: name,
-                        chunks: migratedChunks,
-                        vectors: migratedVectors,
-                        activated: errors.length === 0 && migratedChunks > 0,
-                        errors,
+                        chunks: this.store.countChunks(name),
+                        vectors: this.store.countEmbeddings(name),
+                        activated: false,
+                        errors: [`KNOWLEDGE_SOURCE_UNSUPPORTED:${name}:chunk_policy_or_provenance`],
                     });
                     continue;
                 }
                 const sourceSchemaId = this.ensureManifestSourceSchema(name, manifest, version);
+                const schema = this.store.getSourceSchema(sourceSchemaId);
+                if (!schema)
+                    throw new Error(`Knowledge source schema ${sourceSchemaId} is not registered`);
                 const vectors = this.store.countEmbeddings(name);
-                const coverage = vectors / chunks.length;
                 const contentHash = createHash('sha256')
                     .update(chunks.map(chunk => `${chunk.id}\0${chunk.content_hash || chunk.content}`).sort().join('\n'))
                     .digest('hex');
@@ -645,30 +523,17 @@ export class KnowledgeEngine {
                         ...parseJsonStrings(chunk.symbols),
                         ...parseJsonStrings(chunk.api_identifiers),
                     ]))];
-                const now = Date.now();
-                this.catalog.activateSource({
-                    schemaId: sourceSchemaId,
+                this.sources.activateExistingRevision({
+                    schema,
+                    sourceLocator: chunks.find(chunk => chunk.source_url)?.source_url
+                        || chunks.find(chunk => chunk.url)?.url
+                        || `devflow://knowledge/${name}`,
                     version,
                     contentHash,
-                    readiness: coverage >= 0.95
-                        ? {
-                            status: 'ready', activeContentHash: contentHash,
-                            documents: new Set(chunks.map(chunk => chunk.doc_id)).size,
-                            chunks: chunks.length, vectors, coverage,
-                            lastSuccessfulRefresh: now, lastAttempt: now,
-                        }
-                        : {
-                            status: 'degraded', activeContentHash: contentHash,
-                            documents: new Set(chunks.map(chunk => chunk.doc_id)).size,
-                            chunks: chunks.length, vectors, coverage,
-                            lastAttempt: now,
-                            degradation: {
-                                reason: 'vector_backfill_pending', category: 'vector_degraded', recoverable: true,
-                                previousHealthyRevisionRetained: vectors > 0,
-                            },
-                        },
+                    documents: new Set(chunks.map(chunk => chunk.doc_id)).size,
                     entities,
                     symbols,
+                    operationTimestamp,
                 });
                 const activated = this.catalog.getCatalog().sources.find(entry => entry.sourceId === name);
                 if (activated)
@@ -687,12 +552,6 @@ export class KnowledgeEngine {
         }
         return results;
     }
-    clearChunkFiles(dir) {
-        for (const file of readdirSync(dir)) {
-            if (/^chunk-\d+\.json$/.test(file))
-                rmSync(join(dir, file), { force: true });
-        }
-    }
     async replaceDocument(doc, chunks, deferVectorBackfill = false) {
         if (deferVectorBackfill) {
             this.store.replaceDocumentChunks({ doc, chunks });
@@ -704,14 +563,19 @@ export class KnowledgeEngine {
                 ?? new Error(this.embedder.getLastError() ?? 'Knowledge embedding warmup failed');
         }
         const texts = chunks.map(chunk => `${chunk.headingPath}\n${chunk.content}`);
-        const embeddings = await this.embedder.embedBatch(texts);
+        const embeddings = await requestValidatedEmbeddingBatch({
+            provider: this.embedder,
+            purpose: 'knowledge_retrieval',
+            texts,
+        });
         this.store.replaceDocument({ doc, chunks, embeddings });
     }
     async replaceCompiledSource(input) {
+        const operationTimestamp = input.operationTimestamp ?? Date.now();
         const sourceSchemaId = input.manifest.sourceSchemaId ?? `schema:${input.source}:v1`;
         const sourceKind = input.manifest.sourceKind ?? 'plugin';
         if (!this.store.getSourceSchema(sourceSchemaId)) {
-            this.catalog.registerSourceSchema({
+            this.sources.registerSourceSchema({
                 id: sourceSchemaId,
                 sourceId: input.source,
                 sourceKind,
@@ -724,88 +588,55 @@ export class KnowledgeEngine {
                 refreshStrategy: 'on_demand',
             });
         }
-        const inputHash = createHash('sha256').update(input.markdown).digest('hex');
-        const refresh = this.catalog.beginRefresh(input.source, inputHash);
-        try {
-            const compiled = this.compiler.compile({
-                projectRoot: this.projectRoot,
-                sourceId: input.source,
-                sourceSchemaId,
-                sourceKind,
-                sourceUri: input.sourceUri,
-                markdown: input.markdown,
-                title: input.title,
-                version: input.version,
-            });
-            let embeddings = [];
-            if (!input.deferVectorBackfill) {
-                const warmed = await this.embedder.warmup();
-                if (!warmed) {
-                    throw this.embedder.getLastFailure?.()
-                        ?? new Error(this.embedder.getLastError() ?? 'Knowledge embedding warmup failed');
-                }
-                embeddings = await this.embedder.embedBatch(compiled.chunks.map(chunk => chunk.embeddingText));
-                if (embeddings.length !== compiled.chunks.length) {
-                    throw new Error(`Knowledge embedding count mismatch: ${embeddings.length}/${compiled.chunks.length}`);
-                }
-            }
-            this.store.replaceCompiledPage(compiled.page, compiled.chunks);
-            this.store.replaceKnowledgeAnalysisManifest(compiled.analysis.manifest, compiled.analysis.quarantined);
-            if (embeddings.length > 0) {
-                this.store.insertEmbeddingBatch(compiled.chunks.map((chunk, index) => ({
-                    id: chunk.id,
-                    embedding: embeddings[index],
-                })));
+        const schema = this.store.getSourceSchema(sourceSchemaId);
+        if (!schema)
+            throw new Error(`Knowledge source schema ${sourceSchemaId} is not registered`);
+        const compiled = this.compiler.compile({
+            projectRoot: this.projectRoot,
+            projectId: this.catalog.projectId,
+            sourceId: input.source,
+            sourceSchemaId,
+            sourceKind,
+            sourceUri: input.sourceUri,
+            markdown: input.markdown,
+            title: input.title,
+            version: input.version === 'current' ? undefined : input.version,
+            compiledAt: operationTimestamp,
+            retrievedAt: operationTimestamp,
+        });
+        let embeddings = [];
+        if (!input.deferVectorBackfill) {
+            const warmed = await this.embedder.warmup();
+            if (!warmed) {
+                throw this.embedder.getLastFailure?.()
+                    ?? new Error(this.embedder.getLastError() ?? 'Knowledge embedding warmup failed');
             }
-            this.store.deactivateSourceDocuments(input.source, compiled.page.id);
-            const vectors = embeddings.length;
-            const coverage = compiled.chunks.length > 0 ? vectors / compiled.chunks.length : 0;
-            this.catalog.activateSource({
-                schemaId: sourceSchemaId,
-                version: input.version === 'current' ? undefined : input.version,
-                contentHash: compiled.page.contentHash,
-                readiness: coverage >= 0.95
-                    ? {
-                        status: 'ready', activeContentHash: compiled.page.contentHash,
-                        documents: 1, chunks: compiled.chunks.length, vectors, coverage,
-                        lastSuccessfulRefresh: Date.now(), lastAttempt: Date.now(),
-                    }
-                    : {
-                        status: 'degraded', activeContentHash: compiled.page.contentHash,
-                        documents: 1, chunks: compiled.chunks.length, vectors, coverage,
-                        lastAttempt: Date.now(),
-                        degradation: {
-                            reason: 'vector_backfill_pending', category: 'vector_degraded', recoverable: true,
-                            previousHealthyRevisionRetained: false,
-                        },
-                    },
-                entities: [input.source, ...compiled.page.entities],
-                symbols: compiled.page.symbols,
-                refreshJobId: refresh.id,
+            embeddings = await requestValidatedEmbeddingBatch({
+                provider: this.embedder,
+                purpose: 'knowledge_retrieval',
+                texts: compiled.chunks.map(chunk => chunk.embeddingText),
             });
-            return compiled;
-        }
-        catch (error) {
-            this.catalog.failRefresh(refresh.id, error);
-            throw error;
         }
+        this.sources.publishCompiledSource({ compiled, schema, embeddings, operationTimestamp });
+        return compiled;
     }
     compileDependency(packageName) {
         return compileDependencySource({
             projectRoot: this.projectRoot,
+            projectId: this.projectId,
             packageName,
             store: this.store,
             deferVectorBackfill: true,
         });
     }
     async getProjectProfile() {
-        return this.store.getProjectProfile(this.projectRoot) ?? this.catalog.refreshProjectProfile();
+        return this.store.getProjectProfile(this.catalog.projectId) ?? this.catalog.refreshProjectProfile();
     }
     async getSourceCatalog() {
         return this.catalog.getCatalog();
     }
     async getCoverageGaps() {
-        return this.store.listCoverageGaps(this.projectRoot);
+        return this.store.listCoverageGaps(this.catalog.projectId);
     }
     async search(query, options) {
         return (await this.searchWithStatus(query, options)).entries;
@@ -822,7 +653,7 @@ export class KnowledgeEngine {
             queryPlanHash: options?.queryPlanHash,
         };
         try {
-            const result = await this.searchWithStatusInternal(query, options);
+            const result = await this.query.search(query, options);
             const abstained = result.retrievalDiagnostics?.abstentionReason;
             emitSemanticControlLog({
                 event: 'knowledge.query.executed',
@@ -866,44 +697,18 @@ export class KnowledgeEngine {
             throw error;
         }
     }
-    async searchWithStatusInternal(query, options) {
+    async searchReadOnlyInternal(query, options) {
         const limit = options?.limit ?? 10;
         const queryDecision = planKnowledgeQuery({
             action: options?.action,
             knowledgeNeeds: options?.knowledgeNeeds,
             retrievalIntent: options?.retrievalIntent ?? 'explicit',
         });
-        if (queryDecision.requirement === 'knowledge_not_required') {
-            return {
-                entries: [],
-                warnings: ['knowledge_not_required'],
-                budgetTokens: Math.max(0, options?.budgetTokens ?? DEFAULT_KNOWLEDGE_TOKEN_BUDGET),
-                tokenCount: 0,
-                truncatedCount: 0,
-                rejectedLowScore: 0,
-                rejectedIncompatibleSources: 0,
-                retrievalDiagnostics: {
-                    selectedSources: [],
-                    sourceSignals: [],
-                    sourceRejections: [],
-                    coverageGaps: [],
-                    compatibleChunks: 0,
-                    hybridCandidates: 0,
-                    qualityRejected: {},
-                    abstentionReason: queryDecision.reason,
-                },
-            };
-        }
         const activeSkill = options?.activeSkill ?? options?.projectSignals?.activeSkill;
         const taskAware = Boolean(options?.action || queryDecision.needs.length);
-        let availableSources = this.store.getDistinctSources();
-        if (taskAware) {
-            const catalogSources = new Set(this.catalog.getCatalog().sources.map(entry => entry.sourceId));
-            if (availableSources.some(source => !catalogSources.has(source))) {
-                this.activatePersistedLocalSources();
-            }
-        }
-        const projectProfile = this.catalog.refreshProjectProfile({
+        const availableSources = this.store.getDistinctSources();
+        const persistedProfile = this.store.getProjectProfile(this.catalog.projectId);
+        const projectProfile = persistedProfile ?? buildProjectKnowledgeProfile(this.projectRoot, this.catalog.projectId, {
             activeSkills: activeSkill ? [normalizeActiveSkill(activeSkill)] : [],
         });
         const selectionInput = () => ({
@@ -921,65 +726,36 @@ export class KnowledgeEngine {
             action: options?.action,
             knowledgeNeeds: queryDecision.needs,
         });
-        let selection = selectKnowledgeSources(selectionInput());
-        const compileFailures = [];
-        for (const packageName of taskAware
-            ? dependencyPackagesForMissingNeeds(selection.missingNeeds, projectProfile.dependencies)
-            : []) {
-            try {
-                this.compileDependency(packageName);
-            }
-            catch (error) {
-                const reason = error instanceof Error ? error.message : String(error);
-                this.catalog.recordCoverageGap({
+        const selection = selectKnowledgeSources(selectionInput());
+        const compileFailures = selection.missingNeeds.flatMap(need => {
+            const packageName = dependencyPackagesForMissingNeeds([need], projectProfile.dependencies)[0];
+            return packageName ? [{
                     sourceId: `dependency:${packageName}`,
-                    kind: 'compile_failure',
+                    kind: 'missing_source',
                     entity: packageName,
-                    reason,
-                });
-                compileFailures.push({ sourceId: `dependency:${packageName}`, kind: 'compile_failure', entity: packageName, reason });
-            }
-        }
-        if (selection.missingNeeds.length > 0) {
-            availableSources = this.store.getDistinctSources();
-            selection = selectKnowledgeSources(selectionInput());
-        }
-        for (const gap of selection.coverageGaps)
-            this.catalog.recordCoverageGap(gap);
+                    reason: `Dependency knowledge source ${packageName} requires an explicit ingest command`,
+                }] : [];
+        });
         this.compatibilityRejectionCount += selection.rejectedSources.length;
-        const sources = selection.sources;
-        const semanticMigrationWarnings = [];
-        for (const source of sources) {
-            const legacy = this.store.listChunks({ sources: [source], limit: 200_000 })
-                .some(chunk => !provenanceComplete(chunk.provenance));
-            if (!legacy)
-                continue;
-            const migrated = this.reingestLocalSources(source)[0];
-            if (!migrated || migrated.errors.length > 0 || migrated.chunks === 0) {
-                semanticMigrationWarnings.push(`knowledge-semantic-migration-failed:${source}:${migrated?.errors[0] ?? 'no_result'}`);
-            }
-            else {
-                semanticMigrationWarnings.push(`knowledge-semantic-migrated:${source}:${migrated.chunks}`);
-                try {
-                    const backfill = await this.backfillVectors({ source, batchSize: 32 });
-                    semanticMigrationWarnings.push(`knowledge-semantic-vectors:${source}:${backfill.processed}:${backfill.remaining}`);
-                }
-                catch (error) {
-                    semanticMigrationWarnings.push(`knowledge-semantic-vector-degraded:${source}:${error instanceof Error ? error.message : String(error)}`);
-                }
-            }
-        }
+        const unsupportedSources = new Set(selection.sources.filter(source => this.store.listChunks({ sources: [source], limit: 200_000 })
+            .some(chunk => !provenanceComplete(chunk.provenance))));
+        const sources = selection.sources.filter(source => !unsupportedSources.has(source));
+        const unsupportedSourceWarnings = [...unsupportedSources]
+            .map(source => `knowledge-source-unsupported:${source}`);
         const budgetTokens = Math.max(0, options?.budgetTokens ?? DEFAULT_KNOWLEDGE_TOKEN_BUDGET);
         const projectVersions = detectProjectKnowledgeVersions(this.projectRoot);
-        this.store.refreshSourceReadiness(Object.fromEntries(sources.map(source => [source, expectedVersionForSource(source, projectVersions)])));
-        const readinessWarnings = this.store.getSourceReadiness()
-            .filter(row => sources.includes(row.source) && row.status !== 'ready')
-            .map(row => `knowledge-source-degraded:${row.source}:${row.failure_category ?? 'unknown'}`);
+        const catalogBySource = new Map(this.catalog.getCatalog().sources.map(entry => [entry.sourceId, entry]));
+        const readinessWarnings = sources.flatMap(source => {
+            const readiness = catalogBySource.get(source)?.readiness;
+            return readiness && readiness.status !== 'ready'
+                ? [`knowledge-source-degraded:${source}:${readiness.degradation?.category ?? readiness.status}`]
+                : [];
+        });
         const coverageWarnings = selection.coverageGaps.length > 0 ? ['knowledge_source_missing'] : [];
         if (sources.length === 0) {
             return {
                 entries: [],
-                warnings: [...semanticMigrationWarnings, ...coverageWarnings, ...readinessWarnings, 'knowledge_insufficient'],
+                warnings: [...unsupportedSourceWarnings, ...coverageWarnings, ...readinessWarnings, 'knowledge_insufficient'],
                 budgetTokens,
                 tokenCount: 0,
                 truncatedCount: 0,
@@ -997,11 +773,11 @@ export class KnowledgeEngine {
         }
         const allChunks = this.store.listChunks({ sources });
         const chunks = allChunks.filter(chunk => chunk.chunk_policy_version === KNOWLEDGE_CHUNK_POLICY_VERSION
-            && isKnowledgeVersionCompatible(chunk.source, chunk.version, projectVersions));
+            && isKnowledgeVersionCompatible(chunk.source, chunk.version, projectVersions, catalogBySource.get(chunk.source)?.sourceKind ?? 'plugin'));
         const rejectedByQualityGate = allChunks.length - chunks.length;
         if (chunks.length === 0) {
             return {
-                entries: [], warnings: [...semanticMigrationWarnings, ...coverageWarnings, ...readinessWarnings, 'knowledge_insufficient'], budgetTokens, tokenCount: 0, truncatedCount: 0,
+                entries: [], warnings: [...unsupportedSourceWarnings, ...coverageWarnings, ...readinessWarnings, 'knowledge_insufficient'], budgetTokens, tokenCount: 0, truncatedCount: 0,
                 rejectedIncompatibleSources: selection.rejectedSources.length + rejectedByQualityGate,
                 retrievalDiagnostics: {
                     selectedSources: sources,
@@ -1033,7 +809,7 @@ export class KnowledgeEngine {
             const packed = this.packEntries(candidates, chunks, budgetTokens);
             return {
                 ...packed,
-                warnings: [...semanticMigrationWarnings, ...coverageWarnings, ...readinessWarnings],
+                warnings: [...unsupportedSourceWarnings, ...coverageWarnings, ...readinessWarnings],
                 budgetTokens,
                 rejectedIncompatibleSources: selection.rejectedSources.length + rejectedByQualityGate,
                 retrievalDiagnostics: {
@@ -1047,25 +823,11 @@ export class KnowledgeEngine {
                 },
             };
         }
-        let bm25Results = [];
-        try {
-            bm25Results = this.store.searchBM25(query, limit * 3, sources);
-        }
-        catch {
-            // Retry below with a bounded literal-term query.
-        }
-        if (bm25Results.length === 0) {
-            const safeQuery = buildSafeFtsQuery(query);
-            if (safeQuery) {
-                try {
-                    bm25Results = this.store.searchBM25(safeQuery, limit * 3, sources);
-                }
-                catch {
-                    // Vector retrieval can still answer when FTS cannot parse any bounded fallback.
-                }
-            }
-        }
-        const hybrid = await this.hybridSearch.searchWithStatus(query, chunks, bm25Results, limit, sources, options?.minScore, options?.knowledgeNeeds);
+        const lexicalQuery = compileKnowledgeFtsQuery(query);
+        const bm25Results = lexicalQuery
+            ? this.store.searchBM25(lexicalQuery, limit * 3, sources)
+            : [];
+        const hybrid = await this.hybridSearch.searchWithStatus(query, chunks, bm25Results, limit, sources, options?.minScore, options?.knowledgeNeeds, options?.embeddingProfileHash);
         this.lowScoreRejectionCount += hybrid.rejectedLowScore;
         const chunkMap = new Map(chunks.map(chunk => [chunk.id, chunk]));
         const qualityRejected = {};
@@ -1075,7 +837,7 @@ export class KnowledgeEngine {
                 query,
                 chunk,
                 retrievalScore: result.score,
-                versionCompatible: isKnowledgeVersionCompatible(chunk.source, chunk.version, projectVersions),
+                versionCompatible: isKnowledgeVersionCompatible(chunk.source, chunk.version, projectVersions, catalogBySource.get(chunk.source)?.sourceKind ?? 'plugin'),
                 action: options?.action,
                 knowledgeNeeds: options?.knowledgeNeeds,
             });
@@ -1120,7 +882,7 @@ export class KnowledgeEngine {
                 query,
                 chunk,
                 retrievalScore: expansion.score,
-                versionCompatible: isKnowledgeVersionCompatible(chunk.source, chunk.version, projectVersions),
+                versionCompatible: isKnowledgeVersionCompatible(chunk.source, chunk.version, projectVersions, catalogBySource.get(chunk.source)?.sourceKind ?? 'plugin'),
                 action: options?.action,
                 knowledgeNeeds: options?.knowledgeNeeds,
             });
@@ -1163,7 +925,7 @@ export class KnowledgeEngine {
         const packed = this.packEntries(candidates, chunks, budgetTokens);
         return {
             ...packed,
-            warnings: [...semanticMigrationWarnings, ...hybrid.warnings, ...coverageWarnings, ...readinessWarnings, ...(candidates.length === 0 ? ['knowledge_insufficient'] : [])],
+            warnings: [...unsupportedSourceWarnings, ...hybrid.warnings, ...coverageWarnings, ...readinessWarnings, ...(candidates.length === 0 ? ['knowledge_insufficient'] : [])],
             budgetTokens,
             rejectedLowScore: hybrid.rejectedLowScore,
             rejectedIncompatibleSources: selection.rejectedSources.length + rejectedByQualityGate,
@@ -1176,6 +938,9 @@ export class KnowledgeEngine {
                 hybridCandidates: hybrid.results.length,
                 qualityRejected,
                 channelCandidateCounts: hybrid.channelCandidateCounts,
+                vectorStatus: hybrid.vectorStatus,
+                embeddingProfileHash: hybrid.embeddingCapability.profile.profileHash,
+                embeddingGeneration: hybrid.embeddingCapability.profile.generation,
                 graphExpansions: graphCandidates.length,
             },
         };
@@ -1185,6 +950,11 @@ export class KnowledgeEngine {
         const entries = [];
         const sourceTokens = new Map();
         const pageTokens = new Map();
+        const structuredPages = new Set(candidates.flatMap(entry => {
+            const pageId = chunkMap.get(entry.id)?.page_id;
+            return pageId ? [pageId] : [];
+        }));
+        const enforcePageDiversity = structuredPages.size > 1;
         let tokenCount = 0;
         let truncatedCount = 0;
         for (const entry of candidates) {
@@ -1199,7 +969,7 @@ export class KnowledgeEngine {
             const pageLimit = Math.max(entryTokens, Math.floor(budgetTokens * 0.40));
             const overSourceBudget = Boolean(structuredPage)
                 && (sourceTokens.get(source) ?? 0) + entryTokens > sourceLimit;
-            const overPageBudget = Boolean(structuredPage)
+            const overPageBudget = enforcePageDiversity && Boolean(structuredPage)
                 && (pageTokens.get(structuredPage) ?? 0) + entryTokens > pageLimit;
             if (tokenCount + entryTokens > budgetTokens || overSourceBudget || overPageBudget) {
                 truncatedCount++;
@@ -1230,7 +1000,11 @@ export class KnowledgeEngine {
             });
             if (chunks.length === 0)
                 break;
-            const embeddings = await this.embedder.embedBatch(chunks.map(chunk => `${chunk.heading_path}\n${chunk.content}`));
+            const embeddings = await requestValidatedEmbeddingBatch({
+                provider: this.embedder,
+                purpose: 'knowledge_retrieval',
+                texts: chunks.map(chunk => `${chunk.heading_path}\n${chunk.content}`),
+            });
             this.store.insertEmbeddingBatch(chunks.map((chunk, index) => ({
                 id: chunk.id,
                 embedding: embeddings[index],
@@ -1241,37 +1015,10 @@ export class KnowledgeEngine {
                 - this.store.countEmbeddings(options.source);
             options.onProgress?.({ processed, remaining, lastId });
         }
-        const versions = detectProjectKnowledgeVersions(this.projectRoot);
-        this.store.refreshSourceReadiness(Object.fromEntries(this.store.getDistinctSources().map(source => [source, expectedVersionForSource(source, versions)])));
         for (const entry of this.catalog.getCatalog().sources) {
             if (options.source && entry.sourceId !== options.source)
                 continue;
-            const chunks = this.store.countChunks(entry.sourceId);
-            const vectors = this.store.countEmbeddings(entry.sourceId);
-            const coverage = chunks > 0 ? vectors / chunks : 0;
-            this.catalog.activateSource({
-                schemaId: entry.schemaId,
-                version: entry.version,
-                contentHash: entry.contentHash,
-                readiness: coverage >= 0.95
-                    ? {
-                        status: 'ready', activeContentHash: entry.contentHash,
-                        documents: entry.readiness.documents, chunks, vectors, coverage,
-                        lastSuccessfulRefresh: Date.now(), lastAttempt: Date.now(),
-                    }
-                    : {
-                        status: 'degraded', activeContentHash: entry.contentHash,
-                        documents: entry.readiness.documents, chunks, vectors, coverage,
-                        lastAttempt: Date.now(),
-                        degradation: {
-                            reason: 'vector_backfill_pending', category: 'vector_degraded', recoverable: true,
-                            previousHealthyRevisionRetained: true,
-                        },
-                    },
-                entities: entry.entities,
-                symbols: entry.symbols,
-                taskKinds: entry.taskKinds,
-            });
+            this.sources.refreshIndexReadiness(entry, options.operationTimestamp ?? Date.now());
         }
         return {
             processed,