agentic-qe 3.15.11 → 3.15.12

package/dist/learning/qe-reasoning-bank.js

@@ -23,6 +23,7 @@ import { createPatternStore, } from './pattern-store.js';
 import { createSQLitePatternStore, } from './sqlite-persistence.js';
 import { getWitnessChain } from '../audit/witness-chain.js';
 import { getActiveEmbeddingSpaceIdentity } from './real-embeddings.js';
+import { EmbedderRecovery } from './embedder-recovery.js';
 import { reconcileRvfWithSqlite } from './rvf-reconcile.js';
 import { PatternMutationError } from './pattern-mutation-error.js';
 // Import extracted modules
@@ -67,6 +68,11 @@ export class QEReasoningBank {
     coherenceService;
     config;
     embeddingProvenance = new WeakMap();
+    /**
+     * #1187/#1204: whether the embedder works. While it does not, patterns are
+     * stored text-only and searched by text; it is retried in the background.
+     */
+    embedderRecovery;
     patternStore;
     initialized = false;
     sqliteStore = null;
@@ -106,6 +112,11 @@ export class QEReasoningBank {
         this.eventBus = eventBus;
         this.coherenceService = coherenceService;
         this.config = { ...DEFAULT_QE_REASONING_BANK_CONFIG, ...config };
+        this.embedderRecovery = new EmbedderRecovery({
+            retryIntervalMs: this.config.embedderRetryIntervalMs,
+            onRecovered: () => logger.info('Embedder available again: patterns are stored and searched with vectors'),
+            onRetryFailed: (error) => logger.debug('Embedder still unavailable', { error: toErrorMessage(error) }),
+        });
         if (config.rvfDualWriter) {
             this.rvfDualWriter = config.rvfDualWriter;
         }
@@ -318,14 +329,14 @@ export class QEReasoningBank {
         }
         // Generate embedding if not provided
         if (!options.embedding) {
-            const embedding = await this.embed(`${options.name} ${options.description} ${options.context?.tags?.join(' ') || ''}`);
+            const embedding = await this.embedForPatterns(`${options.name} ${options.description} ${options.context?.tags?.join(' ') || ''}`);
             // #981: with no embedding runtime, embed() returns a hash-fallback vector
             // that has no embedding-space provenance, and the pattern store refuses
             // to persist it (VECTOR_SPACE_UNVERIFIED), losing the pattern. Store it
             // text-only instead, as search already does for such vectors; the vector
             // is filled later by `aqe learning backfill-embeddings`.
-            const hasProvenance = this.embeddingProvenance.has(embedding)
-                || Boolean(getActiveEmbeddingSpaceIdentity()?.spaceId);
+            const hasProvenance = embedding !== null && (this.embeddingProvenance.has(embedding)
+                || Boolean(getActiveEmbeddingSpaceIdentity()?.spaceId));
             if (hasProvenance) {
                 options = { ...options, embedding };
             }
@@ -389,10 +400,10 @@ export class QEReasoningBank {
                 searchQuery = '';
             }
             else {
-                const embedding = await this.embed(query);
+                const embedding = await this.embedForPatterns(query);
                 // Hash fallback and resized vectors do not belong to the active provider.
                 // Keep their text searchable without claiming semantic comparability.
-                searchQuery = this.embeddingProvenance.has(embedding) ? embedding : query;
+                searchQuery = embedding && this.embeddingProvenance.has(embedding) ? embedding : query;
                 useVectorSearch = Array.isArray(searchQuery);
             }
         }
@@ -501,9 +512,9 @@ export class QEReasoningBank {
                 detectedDomains.push('test-generation'); // Default
             }
             // 2. Search for similar patterns
-            const embedding = await this.embed(request.task);
-            const embeddingSpaceId = this.embeddingProvenance.get(embedding);
-            const patternResults = await this.patternStore.search(embeddingSpaceId ? embedding : request.task, {
+            const embedding = await this.embedForPatterns(request.task);
+            const embeddingSpaceId = embedding ? this.embeddingProvenance.get(embedding) : undefined;
+            const patternResults = await this.patternStore.search(embedding && embeddingSpaceId ? embedding : request.task, {
                 limit: this.config.maxRoutingCandidates,
                 domain: detectedDomains[0],
                 useVectorSearch: Boolean(embeddingSpaceId),
@@ -695,6 +706,41 @@ export class QEReasoningBank {
         // Hash-based fallback (always works, including ARM64)
         return hashEmbedding(text, this.config.embeddingDimension);
     }
+    /**
+     * #1187: embed for pattern store, search and routing, or null when the
+     * embedder endpoint fails at runtime. Callers then store the pattern
+     * text-only and search by text, as they do without an embedding runtime.
+     * ADR-097 still holds: no hash vector stands in for an endpoint vector.
+     * #1204: after a failure the embedder is not called again until a
+     * background retry, due `embedderRetryIntervalMs` later, finds it working.
+     */
+    async embedForPatterns(text) {
+        if (!this.embedderRecovery.isAvailable())
+            return null;
+        try {
+            return await this.embed(text);
+        }
+        catch (error) {
+            const reason = toErrorMessage(error);
+            const message = 'Embedder failed: storing and searching patterns by text only';
+            if (this.embedderRecovery.markUnavailable(reason)) {
+                logger.warn(message, { error: reason });
+            }
+            else {
+                logger.debug(message, { error: reason });
+            }
+            return null;
+        }
+    }
+    /**
+     * #1204: why the embedder failed, while patterns are stored and searched
+     * by text only because of that failure; null while it works or was never
+     * needed. A missing embedding runtime with no endpoint configured is not a
+     * failure: there embed() returns a hash vector, which is stored text-only.
+     */
+    getEmbedderFailure() {
+        return this.embedderRecovery.unavailableReason;
+    }
     /**
      * Get statistics
      */

package/dist/learning/real-embeddings.d.ts

@@ -75,6 +75,29 @@ export declare function getEmbeddingDimension(): number;
  * Reset initialization state (for testing)
  */
 export declare function resetInitialization(): void;
+/**
+ * #1187: allow a new initialization attempt after a failed one. A failed
+ * initialization is otherwise remembered for the whole process, so a short
+ * embedding-endpoint outage at startup would disable embeddings until the
+ * process restarts. Callers rate-limit retries. No-op unless the last
+ * initialization failed.
+ */
+export declare function allowEmbedderInitRetry(): void;
+/**
+ * #1204: check whether a failed embedder works again, for callers that retry
+ * it after a cooldown. Resolves when the embedder is usable; throws otherwise.
+ * Other callers keep failing fast while the check runs, so a host that never
+ * answers stalls only the (background) check, never them:
+ * - after a failed initialization, initialize again while the failure stays
+ *   recorded; concurrent rechecks share that attempt;
+ * - after runtime failures that opened the endpoint circuit breaker, probe the
+ *   endpoint as the breaker's single trial request, so recovery does not wait
+ *   for the breaker window as well as the caller's retry interval. The probe
+ *   must find the identity this process bound to (ADR-097): a changed
+ *   fingerprint means vectors would not be comparable.
+ * Callers rate-limit rechecks, then embed a canary to confirm.
+ */
+export declare function recheckEmbedder(config?: Partial<EmbeddingConfig>): Promise<void>;
 /**
  * ADR-097: Returns true when the embedding pipeline is the external endpoint path,
  * false when it's the in-process transformer (or uninitialized).

package/dist/learning/real-embeddings.js

@@ -17,6 +17,8 @@ let embeddingModel = null;
 let initPromise = null;
 let initializationFailed = false;
 let failureReason = '';
+/** #1204: an in-flight recheckEmbedder() re-initialization, shared by callers. */
+let initRetryPromise = null;
 /**
  * Set when ADR-097 endpoint path is active. Tracked separately so callers
  * (tests, observability) can verify endpoint vs in-process unambiguously.
@@ -78,8 +80,10 @@ function isNonSemanticText(text) {
 /**
  * Initialize the transformer model
  */
-async function initializeModel(config = {}) {
-    if (initializationFailed) {
+async function initializeModel(config = {}, retryingAfterFailure = false) {
+    // #1204: a recheckEmbedder() retry runs while the failure is still recorded,
+    // so other callers keep failing fast instead of waiting on a dead host.
+    if (initializationFailed && !retryingAfterFailure) {
         throw new Error(`Transformer initialization previously failed: ${failureReason}`);
     }
     if (initPromise) {
@@ -134,6 +138,8 @@ async function initializeModel(config = {}) {
                     console.warn(`[RealEmbeddings] Could not persist endpoint identity: ${toErrorMessage(persistErr)}`);
                 }
                 embeddingModel = createEndpointPipeline(endpointClient);
+                initializationFailed = false;
+                failureReason = '';
                 return;
             }
             // In-process branch — only here do we import the transformers package.
@@ -179,6 +185,8 @@ async function initializeModel(config = {}) {
                 implementation: `transformers:${fullConfig.quantized ? 'quantized' : 'full'}`,
                 runtimeFingerprint: fingerprintEmbeddingRuntime(canaryVector),
             });
+            initializationFailed = false;
+            failureReason = '';
             const loadTime = performance.now() - startTime;
             console.log(`[RealEmbeddings] Model loaded in ${loadTime.toFixed(0)}ms`);
         }
@@ -393,6 +401,65 @@ export function resetInitialization() {
     // AQE_MEMORY_PATH between runs don't write into a stale handle.
     resetEmbedderIdentityStore();
 }
+/**
+ * #1187: allow a new initialization attempt after a failed one. A failed
+ * initialization is otherwise remembered for the whole process, so a short
+ * embedding-endpoint outage at startup would disable embeddings until the
+ * process restarts. Callers rate-limit retries. No-op unless the last
+ * initialization failed.
+ */
+export function allowEmbedderInitRetry() {
+    if (!initializationFailed)
+        return;
+    initializationFailed = false;
+    failureReason = '';
+    initPromise = null;
+    if (endpointClient) {
+        endpointClient.close();
+    }
+    endpointClient = null;
+    endpointIdentity = null;
+}
+/**
+ * #1204: check whether a failed embedder works again, for callers that retry
+ * it after a cooldown. Resolves when the embedder is usable; throws otherwise.
+ * Other callers keep failing fast while the check runs, so a host that never
+ * answers stalls only the (background) check, never them:
+ * - after a failed initialization, initialize again while the failure stays
+ *   recorded; concurrent rechecks share that attempt;
+ * - after runtime failures that opened the endpoint circuit breaker, probe the
+ *   endpoint as the breaker's single trial request, so recovery does not wait
+ *   for the breaker window as well as the caller's retry interval. The probe
+ *   must find the identity this process bound to (ADR-097): a changed
+ *   fingerprint means vectors would not be comparable.
+ * Callers rate-limit rechecks, then embed a canary to confirm.
+ */
+export async function recheckEmbedder(config = {}) {
+    if (initializationFailed) {
+        initRetryPromise ??= retryInitialization(config).finally(() => {
+            initRetryPromise = null;
+        });
+        return initRetryPromise;
+    }
+    const client = endpointClient;
+    if (!client || !client.getBreakerState().open)
+        return;
+    const identity = await client.probeThroughOpenBreaker();
+    if (endpointIdentity && identity.fingerprint !== endpointIdentity.fingerprint) {
+        throw new Error(`[RealEmbeddings] endpoint identity changed since this process bound to it ` +
+            `(fingerprint ${endpointIdentity.fingerprint} -> ${identity.fingerprint}); ` +
+            `restart to use the new embedding space`);
+    }
+}
+async function retryInitialization(config) {
+    if (endpointClient) {
+        endpointClient.close();
+    }
+    endpointClient = null;
+    endpointIdentity = null;
+    initPromise = null;
+    await initializeModel(config, true);
+}
 /**
  * ADR-097: Returns true when the embedding pipeline is the external endpoint path,
  * false when it's the in-process transformer (or uninitialized).

package/dist/learning/real-qe-reasoning-bank.d.ts

@@ -28,6 +28,11 @@ export interface RealQEReasoningBankConfig {
     };
     /** Coherence energy threshold for pattern promotion (ADR-052) */
     coherenceThreshold?: number;
+    /**
+     * #1187: after the embedder fails (at startup or at runtime), wait this long
+     * before trying it again. Until then pattern search is lexical-only.
+     */
+    embedderRetryIntervalMs?: number;
 }
 export declare const DEFAULT_REAL_CONFIG: RealQEReasoningBankConfig;
 /** Ranking evidence keeps lexical relevance separate from vector reuse safety. */
@@ -42,7 +47,8 @@ export interface RealQEPatternSearchResult {
  * #1022: how pattern search works in this process. Without an embedding
  * runtime the bank stores patterns text-only (vectors pending, filled later by
  * `aqe learning backfill-embeddings`) and searches by SQLite FTS text matching
- * only. No similarity score is computed in that mode.
+ * only. No similarity score is computed in that mode. #1187: the same mode
+ * applies while an embedder that failed at runtime is unavailable.
  */
 export interface PatternSearchStatus {
     mode: 'vector' | 'lexical-only';
@@ -132,8 +138,11 @@ export declare class RealQEReasoningBank {
     private hnswIndex;
     private patternIdMap;
     private initialized;
-    /** #1022: set when no embedding runtime is available; null when vectors work. */
-    private embedderUnavailableReason;
+    /**
+     * #1022/#1187/#1204: whether the embedder works. While it does not, patterns
+     * are stored text-only and searched by text; it is retried in the background.
+     */
+    private readonly embedderRecovery;
     private stats;
     private routingLatencies;
     private readonly asymmetricEngine;
@@ -145,6 +154,17 @@ export declare class RealQEReasoningBank {
     initialize(): Promise<void>;
     /** #1022: whether pattern search uses vectors or text matching only. */
     getPatternSearchStatus(): PatternSearchStatus;
+    /**
+     * #1187: embed `text`, or return undefined when the embedder is unavailable.
+     * A runtime failure switches this bank to lexical-only (reported by
+     * getPatternSearchStatus): callers then store patterns text-only and search
+     * by text. The embedder is tried again once `embedderRetryIntervalMs` has
+     * passed, so a short outage, at startup or later, does not last until restart.
+     * #1204: that retry runs in the background; this call stays text-only.
+     */
+    private embedOrUndefined;
+    /** #1187: the retry probe; vector search needs the HNSW index as well. */
+    private probeEmbedderForRetry;
     /**
      * Initialize HNSW index for fast similarity search
      */
@@ -170,7 +190,7 @@ export declare class RealQEReasoningBank {
         domain?: QEDomain;
         minSimilarity?: number;
     }): Promise<Result<RealQEPatternSearchResult[]>>;
-    /** Append HNSW vector matches for `query` to `patterns`. */
+    /** Append HNSW vector matches for the query embedding to `patterns`. */
     private collectVectorMatches;
     /**
      * Route a task to optimal agent

package/dist/learning/real-qe-reasoning-bank.js

@@ -23,6 +23,7 @@ import { blendFtsScore, DEFAULT_PATTERN_STORE_CONFIG } from './pattern-store.js'
 import { EMBEDDING_SPACE_CANARY } from './embedding-space.js';
 const logger = LoggerFactory.create('RealQEReasoningBank');
 import { computeRealEmbedding, isTransformerAvailable, getEmbeddingDimension, clearEmbeddingCache, getActiveEmbeddingSpaceIdentity, } from './real-embeddings.js';
+import { EmbedderRecovery, probeEmbedder } from './embedder-recovery.js';
 import { createSQLitePatternStore, } from './sqlite-persistence.js';
 import { detectQEDomain, detectQEDomains, mapQEDomainToAQE, QE_DOMAIN_LIST, shouldPromotePattern, } from './qe-patterns.js';
 import { getGuidance, getCombinedGuidance, generateGuidanceContext, checkAntiPatterns, } from './qe-guidance.js';
@@ -59,6 +60,7 @@ export const DEFAULT_REAL_CONFIG = {
         capabilities: 0.3,
     },
     coherenceThreshold: 0.4, // ADR-052: Coherence gate threshold
+    embedderRetryIntervalMs: 60_000,
 };
 // ============================================================================
 // Real QE ReasoningBank Implementation
@@ -80,8 +82,11 @@ export class RealQEReasoningBank {
     hnswIndex = null;
     patternIdMap = new Map();
     initialized = false;
-    /** #1022: set when no embedding runtime is available; null when vectors work. */
-    embedderUnavailableReason = null;
+    /**
+     * #1022/#1187/#1204: whether the embedder works. While it does not, patterns
+     * are stored text-only and searched by text; it is retried in the background.
+     */
+    embedderRecovery;
     // Statistics tracking - using CircularBuffer for O(1) operations (ADR memory leak fix)
     stats = {
         routingRequests: 0,
@@ -157,6 +162,12 @@ export class RealQEReasoningBank {
         this.qeConfig = { ...DEFAULT_REAL_CONFIG, ...config };
         this.sqliteStore = createSQLitePatternStore(this.qeConfig.sqlite);
         this.asymmetricEngine = new AsymmetricLearningEngine();
+        this.embedderRecovery = new EmbedderRecovery({
+            retryIntervalMs: this.qeConfig.embedderRetryIntervalMs,
+            probe: () => this.probeEmbedderForRetry(),
+            onRecovered: () => logger.info('Embedder available again: pattern search uses vectors'),
+            onRetryFailed: (error) => logger.debug('Embedder still unavailable', { error: toErrorMessage(error) }),
+        });
     }
     /**
      * Initialize the QE ReasoningBank
@@ -176,11 +187,11 @@ export class RealQEReasoningBank {
             await computeRealEmbedding(EMBEDDING_SPACE_CANARY, this.qeConfig.embeddings);
         }
         catch (error) {
-            this.embedderUnavailableReason = toErrorMessage(error);
+            this.embedderRecovery.markUnavailable(toErrorMessage(error));
             logger.warn('No embedding runtime: patterns are stored without vectors (pending ' +
-                '`aqe learning backfill-embeddings`) and searched by text matching only', { reason: this.embedderUnavailableReason });
+                '`aqe learning backfill-embeddings`) and searched by text matching only', { reason: this.embedderRecovery.unavailableReason });
         }
-        if (this.embedderUnavailableReason === null) {
+        if (this.embedderRecovery.unavailableReason === null) {
             // Initialize HNSW index
             await this.initializeHNSW();
             logger.info('HNSW index initialized');
@@ -209,9 +220,39 @@ export class RealQEReasoningBank {
     }
     /** #1022: whether pattern search uses vectors or text matching only. */
     getPatternSearchStatus() {
-        return this.embedderUnavailableReason === null
-            ? { mode: 'vector' }
-            : { mode: 'lexical-only', reason: this.embedderUnavailableReason };
+        const reason = this.embedderRecovery.unavailableReason;
+        return reason === null ? { mode: 'vector' } : { mode: 'lexical-only', reason };
+    }
+    /**
+     * #1187: embed `text`, or return undefined when the embedder is unavailable.
+     * A runtime failure switches this bank to lexical-only (reported by
+     * getPatternSearchStatus): callers then store patterns text-only and search
+     * by text. The embedder is tried again once `embedderRetryIntervalMs` has
+     * passed, so a short outage, at startup or later, does not last until restart.
+     * #1204: that retry runs in the background; this call stays text-only.
+     */
+    async embedOrUndefined(text) {
+        if (!this.embedderRecovery.isAvailable())
+            return undefined;
+        try {
+            return await computeRealEmbedding(text, this.qeConfig.embeddings);
+        }
+        catch (error) {
+            const reason = `Embedder failed at runtime: ${toErrorMessage(error)}`;
+            if (this.embedderRecovery.markUnavailable(reason)) {
+                logger.warn('Embedder failed: patterns are stored without vectors and searched by ' +
+                    'text matching only until it recovers', { reason });
+            }
+            return undefined;
+        }
+    }
+    /** #1187: the retry probe; vector search needs the HNSW index as well. */
+    async probeEmbedderForRetry() {
+        await probeEmbedder(this.qeConfig.embeddings);
+        if (!this.hnswIndex) {
+            await this.initializeHNSW();
+            await this.loadPatternsIntoHNSW();
+        }
     }
     /**
      * Initialize HNSW index for fast similarity search
@@ -331,12 +372,10 @@ export class RealQEReasoningBank {
     async storeQEPattern(options) {
         // Require explicit initialization - don't auto-initialize to avoid memory issues
         try {
-            // Generate REAL embedding using transformer model. #1022: without an
-            // embedding runtime the pattern is stored text-only (vector pending).
+            // Generate REAL embedding using transformer model. #1022/#1187: without
+            // a working embedder the pattern is stored text-only (vector pending).
             const textToEmbed = `${options.name} ${options.description || ''} ${options.context?.tags?.join(' ') || ''}`;
-            const embedding = this.embedderUnavailableReason === null
-                ? await computeRealEmbedding(textToEmbed, this.qeConfig.embeddings)
-                : undefined;
+            const embedding = await this.embedOrUndefined(textToEmbed);
             // Detect QE domain
             const qeDomain = detectQEDomain(textToEmbed) || 'test-generation';
             // ADR-058: Check MemoryWriteGate for contradictions before storing
@@ -435,13 +474,14 @@ export class RealQEReasoningBank {
         try {
             const limit = options.limit || 10;
             const patterns = [];
-            // #1022: without an embedding runtime only the lexical (FTS) branch
+            // #1022/#1187: without a working embedder only the lexical (FTS) branch
             // below runs; its hits are labelled matchType 'lexical'.
-            if (this.embedderUnavailableReason === null) {
+            const queryEmbedding = await this.embedOrUndefined(query);
+            if (queryEmbedding) {
                 if (!this.hnswIndex) {
                     return err(new Error('HNSW index not available'));
                 }
-                await this.collectVectorMatches(query, limit, options, patterns);
+                this.collectVectorMatches(queryEmbedding, limit, options, patterns);
             }
             const existing = new Map(patterns.map(hit => [hit.pattern.id, hit]));
             for (const lexical of this.sqliteStore.searchFTS(query, limit * 2)) {
@@ -471,10 +511,8 @@ export class RealQEReasoningBank {
             return err(toError(error));
         }
     }
-    /** Append HNSW vector matches for `query` to `patterns`. */
-    async collectVectorMatches(query, limit, options, patterns) {
-        // Generate REAL embedding for query
-        const queryEmbedding = await computeRealEmbedding(query, this.qeConfig.embeddings);
+    /** Append HNSW vector matches for the query embedding to `patterns`. */
+    collectVectorMatches(queryEmbedding, limit, options, patterns) {
         // Search HNSW index
         const results = this.hnswIndex.searchKnn(queryEmbedding, limit * 2); // Get extra for filtering
         for (let i = 0; i < results.neighbors.length; i++) {