agentic-qe 3.7.14 → 3.7.15

package/dist/learning/pattern-store.js

@@ -8,7 +8,7 @@
 import { v4 as uuidv4 } from 'uuid';
 import { ok, err } from '../shared/types/index.js';
 import { toErrorMessage, toError } from '../shared/error-utils.js';
-import { calculateQualityScore, shouldPromotePattern, validateQEPattern, mapQEDomainToAQE, } from './qe-patterns.js';
+import { calculateQualityScore, shouldPromotePattern, validateQEPattern, mapQEDomainToAQE, PROMOTION_THRESHOLD, } from './qe-patterns.js';
 /**
  * Default pattern store configuration
  */
@@ -21,7 +21,7 @@ export const DEFAULT_PATTERN_STORE_CONFIG = {
         efSearch: 100,
         maxElements: 50000,
     },
-    promotionThreshold: 3,
+    promotionThreshold: PROMOTION_THRESHOLD,
     minConfidence: 0.3,
     maxPatternsPerDomain: 5000,
     autoCleanup: true,
@@ -55,6 +55,7 @@ export class PatternStore {
     cleanupTimer;
     // Optional SQLite persistence delegate for delete/promote
     sqliteStore = null;
+    loadingPromise = null;
     // In-memory caches for fast access
     patternCache = new Map();
     domainIndex = new Map();
@@ -75,12 +76,23 @@ export class PatternStore {
         this.config = { ...DEFAULT_PATTERN_STORE_CONFIG, ...config };
     }
     /**
-     * Set SQLite persistence delegate for delete/promote operations.
-     * When set, PatternStore will forward these operations to SQLite
-     * in addition to updating the in-memory cache.
+     * Set SQLite persistence delegate and load patterns into memory.
+     *
+     * When set, PatternStore will:
+     * 1. Load existing patterns from SQLite into the in-memory cache
+     * 2. Forward create/delete/promote operations to SQLite for persistence
+     * 3. Persist embeddings alongside patterns on store()
      */
     setSqliteStore(store) {
         this.sqliteStore = store;
+        // Load patterns from SQLite if we're already initialized
+        // (setSqliteStore is called after initialize() in QEReasoningBank)
+        // Store promise so concurrent store/search calls can await it
+        if (this.initialized) {
+            this.loadingPromise = this.loadPatterns().catch((e) => console.warn('[PatternStore] Failed to load patterns after setSqliteStore:', e)).finally(() => {
+                this.loadingPromise = null;
+            });
+        }
     }
     /**
      * Initialize the pattern store
@@ -161,6 +173,47 @@ export class PatternStore {
             const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('HNSW init timeout')), timeoutMs));
             await Promise.race([initPromise, timeoutPromise]);
             this.hnswAvailable = this.hnswIndex.isNativeAvailable();
+            // Load existing embeddings from SQLite into HNSW index (capped to prevent timeout)
+            if (this.sqliteStore) {
+                try {
+                    const embeddings = this.sqliteStore.getAllEmbeddings();
+                    const maxBootstrap = this.config.hnsw.maxElements;
+                    let loaded = 0;
+                    for (const { patternId, embedding } of embeddings) {
+                        if (loaded >= maxBootstrap)
+                            break;
+                        if (!embedding || embedding.length !== this.config.embeddingDimension)
+                            continue;
+                        const pattern = this.patternCache.get(patternId);
+                        if (!pattern)
+                            continue;
+                        try {
+                            await this.hnswIndex.insert(patternId, embedding, {
+                                filePath: pattern.patternType,
+                                lineCoverage: pattern.confidence * 100,
+                                branchCoverage: pattern.qualityScore * 100,
+                                functionCoverage: 0,
+                                statementCoverage: 0,
+                                uncoveredLineCount: 0,
+                                uncoveredBranchCount: 0,
+                                riskScore: 1 - pattern.confidence,
+                                lastUpdated: Date.now(),
+                                totalLines: 0,
+                            });
+                            loaded++;
+                        }
+                        catch {
+                            // Duplicate or invalid — skip
+                        }
+                    }
+                    if (loaded > 0) {
+                        console.log(`[PatternStore] Loaded ${loaded} embeddings from SQLite into HNSW`);
+                    }
+                }
+                catch (error) {
+                    console.warn('[PatternStore] Failed to load SQLite embeddings into HNSW:', toErrorMessage(error));
+                }
+            }
             console.log(`[PatternStore] HNSW lazy-initialized (native: ${this.hnswAvailable})`);
         }
         catch (error) {
@@ -170,13 +223,28 @@ export class PatternStore {
         }
     }
     /**
-     * Load existing patterns from memory with timeout protection
+     * Load existing patterns from SQLite into in-memory cache.
+     *
+     * Previously this was a no-op after Issue #258 removed kv_store duplication,
+     * but that left 15,634 SQLite patterns invisible to search on every restart.
+     * Now properly loads from SQLitePatternStore when wired.
      */
     async loadPatterns() {
-        // Patterns are loaded from qe_patterns table by SQLitePatternStore.
-        // PatternStore's in-memory cache is populated via indexPattern() calls
-        // from the ReasoningBank when it loads from the relational store.
-        // Previously this loaded from kv_store, which duplicated storage (Issue #258).
+        if (!this.sqliteStore) {
+            return; // SQLite not wired yet — will be loaded after setSqliteStore()
+        }
+        try {
+            const patterns = this.sqliteStore.getPatterns({ limit: 50000 });
+            for (const pattern of patterns) {
+                this.indexPattern(pattern);
+            }
+            if (patterns.length > 0) {
+                console.log(`[PatternStore] Loaded ${patterns.length} patterns from SQLite into memory cache`);
+            }
+        }
+        catch (error) {
+            console.warn('[PatternStore] Failed to load patterns from SQLite:', toErrorMessage(error));
+        }
     }
     /**
      * Index a pattern in local caches
@@ -193,8 +261,13 @@ export class PatternStore {
             this.typeIndex.set(pattern.patternType, new Set());
         }
         this.typeIndex.get(pattern.patternType).add(pattern.id);
-        // Tier index
-        this.tierIndex.get(pattern.tier).add(pattern.id);
+        // Tier index (defensive: coerce unexpected tier values to 'short-term')
+        const tier = (pattern.tier === 'long-term') ? 'long-term' : 'short-term';
+        if (pattern.tier !== tier) {
+            // Pattern has invalid tier from SQLite — store corrected copy in cache
+            pattern.tier = tier;
+        }
+        this.tierIndex.get(tier).add(pattern.id);
     }
     /**
      * Remove pattern from local indices
@@ -212,6 +285,9 @@ export class PatternStore {
         if (!this.initialized) {
             await this.initialize();
         }
+        if (this.loadingPromise) {
+            await this.loadingPromise;
+        }
         // Validate pattern
         const validation = validateQEPattern(pattern);
         if (!validation.valid) {
@@ -227,11 +303,17 @@ export class PatternStore {
             // Run cleanup for this domain
             await this.cleanupDomain(pattern.qeDomain);
         }
-        // Patterns are persisted to qe_patterns table by SQLitePatternStore.
-        // PatternStore only maintains in-memory cache + HNSW index for fast search.
-        // Previously this wrote to kv_store, causing 229MB bloat (Issue #258).
-        // Index locally
+        // Index in memory cache
         this.indexPattern(pattern);
+        // Persist to SQLite (pattern + embedding atomically)
+        if (this.sqliteStore) {
+            try {
+                this.sqliteStore.storePattern(pattern, pattern.embedding);
+            }
+            catch (error) {
+                console.warn(`[PatternStore] SQLite persist failed for ${pattern.id}:`, toErrorMessage(error));
+            }
+        }
         // Add to HNSW if embedding is available (lazy-load HNSW only when needed)
         if (pattern.embedding) {
             const hnsw = await this.ensureHNSW();
@@ -327,6 +409,9 @@ export class PatternStore {
         if (!this.initialized) {
             await this.initialize();
         }
+        if (this.loadingPromise) {
+            await this.loadingPromise;
+        }
         return this.patternCache.get(id) ?? null;
     }
     /**
@@ -336,6 +421,9 @@ export class PatternStore {
         if (!this.initialized) {
             await this.initialize();
         }
+        if (this.loadingPromise) {
+            await this.loadingPromise;
+        }
         const startTime = performance.now();
         const limit = options.limit || 10;
         const results = [];
@@ -362,10 +450,69 @@ export class PatternStore {
                     }
                 }
             }
+            // FTS5 hybrid search: blend BM25 text relevance with vector similarity
+            // 75% vector score + 25% FTS5 score for patterns found by both
+            if (typeof query === 'string' && query.trim() && this.sqliteStore) {
+                try {
+                    const ftsResults = this.sqliteStore.searchFTS(query, limit * 2);
+                    if (ftsResults.length > 0) {
+                        const ftsScoreMap = new Map(ftsResults.map(r => [r.id, r.ftsScore]));
+                        const existingIds = new Set(results.map(r => r.pattern.id));
+                        // Boost existing vector results that also match FTS5
+                        for (const result of results) {
+                            const ftsScore = ftsScoreMap.get(result.pattern.id);
+                            if (ftsScore !== undefined) {
+                                result.score = 0.75 * result.score + 0.25 * ftsScore;
+                            }
+                        }
+                        // Add FTS5-only results not already in vector results
+                        for (const ftsResult of ftsResults) {
+                            if (existingIds.has(ftsResult.id))
+                                continue;
+                            const pattern = await this.get(ftsResult.id);
+                            if (pattern && this.matchesFilters(pattern, options)) {
+                                const reuseInfo = this.calculateReuseInfo(pattern, ftsResult.ftsScore);
+                                results.push({
+                                    pattern,
+                                    score: 0.5 * ftsResult.ftsScore, // FTS-only: exact keyword match is valuable
+                                    matchType: 'exact',
+                                    similarity: ftsResult.ftsScore,
+                                    canReuse: reuseInfo.canReuse,
+                                    estimatedTokenSavings: reuseInfo.estimatedTokenSavings,
+                                    reuseConfidence: reuseInfo.reuseConfidence,
+                                });
+                            }
+                        }
+                    }
+                }
+                catch {
+                    // FTS5 unavailable, continue with text fallback
+                }
+            }
             // Text search fallback or additional
             if (typeof query === 'string' || results.length < limit) {
                 const textResults = await this.searchByText(typeof query === 'string' ? query : '', options, limit - results.length);
-                results.push(...textResults);
+                // Deduplicate: only add text results not already present
+                const existingIds = new Set(results.map(r => r.pattern.id));
+                for (const tr of textResults) {
+                    if (!existingIds.has(tr.pattern.id)) {
+                        results.push(tr);
+                    }
+                }
+            }
+            // Apply temporal decay: boost recent patterns, penalize stale ones
+            // Half-life of 30 days — patterns used recently score higher
+            const TEMPORAL_HALF_LIFE_MS = 30 * 24 * 60 * 60 * 1000;
+            const now = Date.now();
+            for (const result of results) {
+                const lastUsed = result.pattern.lastUsedAt?.getTime() ?? result.pattern.createdAt.getTime();
+                const ageMs = now - lastUsed;
+                const decayFactor = Math.pow(0.5, ageMs / TEMPORAL_HALF_LIFE_MS);
+                // Only boost patterns that have been used — new untested patterns get neutral score
+                const effectiveDecay = result.pattern.usageCount > 0 ? decayFactor : 0.5;
+                // Multiplicative decay: preserves relative ordering from search scoring
+                // while penalizing stale patterns (decayFactor is 0-1)
+                result.score = result.score * (0.7 + 0.3 * effectiveDecay);
             }
             // Sort by score and limit
             results.sort((a, b) => b.score - a.score);
@@ -555,6 +702,15 @@ export class PatternStore {
             qualityScore,
             lastUsedAt: now,
         };
+        // Persist usage to SQLite
+        if (this.sqliteStore) {
+            try {
+                this.sqliteStore.recordUsage(id, success);
+            }
+            catch (error) {
+                console.warn(`[PatternStore] SQLite recordUsage failed for ${id}:`, toErrorMessage(error));
+            }
+        }
         // Check for promotion (ADR-052: shouldPromotePattern returns PromotionCheck object)
         const promotionCheck = shouldPromotePattern(updated);
         const shouldPromote = promotionCheck.meetsUsageCriteria &&
@@ -564,7 +720,7 @@ export class PatternStore {
             await this.promote(id);
         }
         else {
-            // Update cache only - persistence handled by SQLitePatternStore
+            // Update in-memory cache
             this.patternCache.set(id, updated);
         }
         return ok(undefined);
@@ -699,7 +855,10 @@ export class PatternStore {
             }
             // Check for removal (short-term, old, low quality)
             if (pattern.tier === 'short-term') {
-                const ageMs = Date.now() - pattern.createdAt.getTime();
+                const createdTime = pattern.createdAt instanceof Date
+                    ? pattern.createdAt.getTime()
+                    : new Date(pattern.createdAt).getTime();
+                const ageMs = Date.now() - createdTime;
                 const isOld = ageMs > 7 * 24 * 60 * 60 * 1000; // 7 days
                 const isLowQuality = pattern.qualityScore < 0.2;
                 const isUnused = pattern.usageCount === 0 && ageMs > 24 * 60 * 60 * 1000; // 1 day

package/dist/learning/qe-hooks.d.ts

@@ -25,6 +25,7 @@ export declare const QE_HOOK_EVENTS: {
     readonly PatternLearned: "qe:pattern-learned";
     readonly PatternApplied: "qe:pattern-applied";
     readonly PatternPromoted: "qe:pattern-promoted";
+    readonly PreCompaction: "qe:pre-compaction";
 };
 export type QEHookEvent = (typeof QE_HOOK_EVENTS)[keyof typeof QE_HOOK_EVENTS];
 /**

package/dist/learning/qe-hooks.js

@@ -34,6 +34,8 @@ export const QE_HOOK_EVENTS = {
     PatternLearned: 'qe:pattern-learned',
     PatternApplied: 'qe:pattern-applied',
     PatternPromoted: 'qe:pattern-promoted',
+    // Session lifecycle
+    PreCompaction: 'qe:pre-compaction',
 };
 // ============================================================================
 // QE Hook Handlers
@@ -434,6 +436,34 @@ export function createQEHookHandlers(reasoningBank) {
                 data: { patternId, newTier },
             };
         },
+        // ========================================================================
+        // Session Lifecycle Hooks
+        // ========================================================================
+        [QE_HOOK_EVENTS.PreCompaction]: async (ctx) => {
+            const stats = { experiencesFlushed: 0, patternsPromoted: 0 };
+            // Flush pending experiences before context compaction
+            if (ctx.data?.experienceCaptureService) {
+                const service = ctx.data.experienceCaptureService;
+                const pendingCount = service.getPendingCount?.() ?? 0;
+                if (pendingCount > 0) {
+                    const flushed = await service.flushPending?.();
+                    stats.experiencesFlushed = flushed ?? pendingCount;
+                }
+            }
+            // Promote eligible patterns before compaction
+            if (ctx.data?.patternLifecycleManager) {
+                const manager = ctx.data.patternLifecycleManager;
+                const promotionResult = manager.runPromotionSweep?.();
+                if (promotionResult) {
+                    stats.patternsPromoted = promotionResult.promoted ?? 0;
+                }
+            }
+            console.log('[QEHooks] Pre-compaction flush:', stats);
+            return {
+                success: true,
+                data: stats,
+            };
+        },
     };
 }
 // ============================================================================

package/dist/learning/qe-patterns.d.ts

@@ -195,6 +195,12 @@ export declare function calculateQualityScore(pattern: {
 /**
  * Pattern promotion check result
  */
+/**
+ * Shared promotion threshold: minimum successful uses before a short-term
+ * pattern can be promoted to long-term. Used by pattern-store, pattern-lifecycle,
+ * experience-capture, and shouldPromotePattern().
+ */
+export declare const PROMOTION_THRESHOLD = 3;
 export interface PromotionCheck {
     meetsUsageCriteria: boolean;
     meetsQualityCriteria: boolean;

package/dist/learning/qe-patterns.js

@@ -72,6 +72,15 @@ export function calculateQualityScore(pattern) {
     const usageScore = Math.min(pattern.usageCount / 100, 1);
     return (pattern.confidence * 0.3 + usageScore * 0.2 + pattern.successRate * 0.5);
 }
+/**
+ * Pattern promotion check result
+ */
+/**
+ * Shared promotion threshold: minimum successful uses before a short-term
+ * pattern can be promoted to long-term. Used by pattern-store, pattern-lifecycle,
+ * experience-capture, and shouldPromotePattern().
+ */
+export const PROMOTION_THRESHOLD = 3;
 /**
  * Check if pattern should be promoted to long-term storage
  * Requires 3+ successful uses as per ADR-021
@@ -82,7 +91,7 @@ export function calculateQualityScore(pattern) {
  * @param coherenceThreshold - Threshold for coherence violation (default: 0.4)
  */
 export function shouldPromotePattern(pattern, coherenceEnergy, coherenceThreshold = 0.4) {
-    const meetsUsageCriteria = pattern.tier === 'short-term' && pattern.successfulUses >= 3;
+    const meetsUsageCriteria = pattern.tier === 'short-term' && pattern.successfulUses >= PROMOTION_THRESHOLD;
     const meetsQualityCriteria = pattern.successRate >= 0.7 && pattern.confidence >= 0.6;
     // NEW: Coherence criteria - only block if coherence energy is provided and exceeds threshold
     const meetsCoherenceCriteria = coherenceEnergy === undefined || coherenceEnergy < coherenceThreshold;

package/dist/learning/sqlite-persistence.d.ts

@@ -36,6 +36,13 @@ export interface SQLitePersistenceConfig {
     useUnified: boolean;
 }
 export declare const DEFAULT_SQLITE_CONFIG: SQLitePersistenceConfig;
+/**
+ * Hash-based embedding generation (no ONNX dependency).
+ * Deterministic: same text always produces the same embedding.
+ * FALLBACK ONLY — real embeddings use all-MiniLM-L6-v2 via computeBatchEmbeddings().
+ * Kept for environments where ONNX/@xenova/transformers is unavailable.
+ */
+export declare function hashEmbedding(text: string, dimension?: number): number[];
 /**
  * SQLite-based pattern persistence
  */
@@ -78,6 +85,23 @@ export declare class SQLitePatternStore {
         domain?: QEDomain;
         limit?: number;
     }): QEPattern[];
+    /**
+     * FTS5 full-text search for patterns.
+     * Returns pattern IDs with BM25 relevance scores.
+     */
+    searchFTS(query: string, limit?: number): Array<{
+        id: string;
+        ftsScore: number;
+    }>;
+    /**
+     * Ghost pattern check: find patterns in SQLite that have no embeddings.
+     * Used by aqe_health to detect data integrity issues.
+     */
+    getGhostPatternCount(): {
+        total: number;
+        withoutEmbeddings: number;
+        sampleGhostIds: string[];
+    };
     /**
      * Get all embeddings for HNSW indexing
      */
@@ -159,6 +183,22 @@ export declare class SQLitePatternStore {
             avgSimilarity: number;
         }>;
     };
+    /**
+     * Backfill embeddings for patterns that don't have them.
+     * Uses real all-MiniLM-L6-v2 transformer embeddings via @xenova/transformers.
+     * Falls back to hash-based embeddings if ONNX is unavailable.
+     * Skips bench/test patterns (id LIKE 'bench-%').
+     *
+     * @param batchSize - Patterns per inference batch (default: 32, matches model batch size)
+     * @returns Stats about the backfill operation
+     */
+    backfillEmbeddings(batchSize?: number): Promise<{
+        processed: number;
+        skipped: number;
+        errors: number;
+        alreadyHad: number;
+        method: 'transformer' | 'hash-fallback';
+    }>;
     /**
      * Close the database
      */