agentic-qe 3.15.11 → 3.15.13

package/dist/learning/embedder-endpoint-client.js

@@ -125,6 +125,10 @@ class CircuitBreaker {
         }
         return true;
     }
+    /** #1204: open (or re-open) the breaker for a full window from now. */
+    trip() {
+        this.trippedAt = Date.now();
+    }
     /** For tests / observability */
     getState() {
         return { open: this.isOpen(), failures: this.failures.length, trippedAt: this.trippedAt };
@@ -266,6 +270,22 @@ export class EmbedderEndpointClient {
     getBreakerState() {
         return this.breaker.getState();
     }
+    /**
+     * #1204: probe the endpoint while the circuit breaker is open, as its single
+     * half-open trial. Other callers keep fast-failing meanwhile. A success
+     * closes the breaker and refreshes the identity; a failure keeps the
+     * breaker open for another full window. Lets a caller that rate-limits its
+     * own retries recover without also waiting out the breaker window.
+     */
+    async probeThroughOpenBreaker() {
+        try {
+            return await this.probe();
+        }
+        catch (err) {
+            this.breaker.trip();
+            throw err;
+        }
+    }
     /**
      * Close the underlying keep-alive agent. Tests should call this in afterEach.
      */

package/dist/learning/embedder-recovery.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Embedder recovery after a failure (#1187, #1204)
+ *
+ * A consumer of the embedder (a pattern bank, ExperienceReplay) runs without
+ * vectors while its embedder is down: patterns are stored text-only and
+ * searched by text. This class tracks that state and retries the embedder
+ * after a bounded interval, so a short outage does not last until restart.
+ *
+ * The retry runs in the background. The call that finds a retry due still
+ * gets "unavailable" and carries on without vectors, so a slow or unreachable
+ * embedder host never stalls it. The consumer becomes available again only
+ * after the retry's probe has succeeded: it never reports vectors it does not
+ * have. ADR-097 still holds: nothing here produces a substitute vector.
+ */
+import { type EmbeddingConfig } from './real-embeddings.js';
+export declare const DEFAULT_EMBEDDER_RETRY_INTERVAL_MS = 60000;
+export interface EmbedderRecoveryOptions {
+    /** Wait this long after a failure before trying the embedder again. */
+    retryIntervalMs?: number;
+    /**
+     * Resolve once the embedder works and the consumer is ready to use
+     * vectors again; throw otherwise. Defaults to probeEmbedder().
+     */
+    probe?: () => Promise<void>;
+    /** Called after a retry succeeded. */
+    onRecovered?: () => void;
+    /** Called after a retry failed. */
+    onRetryFailed?: (error: unknown) => void;
+}
+/**
+ * Probe the embedder after a failure: re-initialize it, or try the endpoint
+ * through its open circuit breaker (recheckEmbedder), then embed the canary,
+ * bypassing the cache — a cached vector proves nothing about the embedder now.
+ */
+export declare function probeEmbedder(config?: Partial<EmbeddingConfig>): Promise<void>;
+export declare class EmbedderRecovery {
+    private readonly options;
+    private reason;
+    private failedAt;
+    /** Counts failures, so a retry does not clear one recorded while it ran. */
+    private failureCount;
+    private retry;
+    private readonly retryIntervalMs;
+    private readonly probe;
+    constructor(options?: EmbedderRecoveryOptions);
+    /** Why the embedder is unavailable, or null while it works. */
+    get unavailableReason(): string | null;
+    /**
+     * True when the embedder can be used now. When it cannot and a retry is
+     * due, starts one in the background and still returns false.
+     */
+    isAvailable(): boolean;
+    /**
+     * Record an embedder failure. Returns true when the embedder was available
+     * until now, so the caller can warn once per outage rather than per call.
+     */
+    markUnavailable(reason: string): boolean;
+    /** Resolves once an in-flight retry has finished (tests, shutdown). */
+    whenRetrySettled(): Promise<void>;
+    private startRetryIfDue;
+    private runRetry;
+}
+//# sourceMappingURL=embedder-recovery.d.ts.map

package/dist/learning/embedder-recovery.js

@@ -0,0 +1,94 @@
+/**
+ * Embedder recovery after a failure (#1187, #1204)
+ *
+ * A consumer of the embedder (a pattern bank, ExperienceReplay) runs without
+ * vectors while its embedder is down: patterns are stored text-only and
+ * searched by text. This class tracks that state and retries the embedder
+ * after a bounded interval, so a short outage does not last until restart.
+ *
+ * The retry runs in the background. The call that finds a retry due still
+ * gets "unavailable" and carries on without vectors, so a slow or unreachable
+ * embedder host never stalls it. The consumer becomes available again only
+ * after the retry's probe has succeeded: it never reports vectors it does not
+ * have. ADR-097 still holds: nothing here produces a substitute vector.
+ */
+import { computeRealEmbedding, recheckEmbedder, } from './real-embeddings.js';
+import { EMBEDDING_SPACE_CANARY } from './embedding-space.js';
+export const DEFAULT_EMBEDDER_RETRY_INTERVAL_MS = 60_000;
+/**
+ * Probe the embedder after a failure: re-initialize it, or try the endpoint
+ * through its open circuit breaker (recheckEmbedder), then embed the canary,
+ * bypassing the cache — a cached vector proves nothing about the embedder now.
+ */
+export async function probeEmbedder(config = {}) {
+    await recheckEmbedder(config);
+    await computeRealEmbedding(EMBEDDING_SPACE_CANARY, { ...config, enableCache: false });
+}
+export class EmbedderRecovery {
+    options;
+    reason = null;
+    failedAt = 0;
+    /** Counts failures, so a retry does not clear one recorded while it ran. */
+    failureCount = 0;
+    retry = null;
+    retryIntervalMs;
+    probe;
+    constructor(options = {}) {
+        this.options = options;
+        this.retryIntervalMs = options.retryIntervalMs ?? DEFAULT_EMBEDDER_RETRY_INTERVAL_MS;
+        this.probe = options.probe ?? (() => probeEmbedder());
+    }
+    /** Why the embedder is unavailable, or null while it works. */
+    get unavailableReason() {
+        return this.reason;
+    }
+    /**
+     * True when the embedder can be used now. When it cannot and a retry is
+     * due, starts one in the background and still returns false.
+     */
+    isAvailable() {
+        if (this.reason === null)
+            return true;
+        this.startRetryIfDue();
+        return false;
+    }
+    /**
+     * Record an embedder failure. Returns true when the embedder was available
+     * until now, so the caller can warn once per outage rather than per call.
+     */
+    markUnavailable(reason) {
+        const wasAvailable = this.reason === null;
+        this.reason = reason;
+        this.failedAt = Date.now();
+        this.failureCount++;
+        return wasAvailable;
+    }
+    /** Resolves once an in-flight retry has finished (tests, shutdown). */
+    async whenRetrySettled() {
+        await this.retry;
+    }
+    startRetryIfDue() {
+        if (this.retry)
+            return;
+        if (Date.now() - this.failedAt < this.retryIntervalMs)
+            return;
+        this.retry = this.runRetry().finally(() => {
+            this.retry = null;
+        });
+    }
+    async runRetry() {
+        const failuresBefore = this.failureCount;
+        try {
+            await this.probe();
+            if (this.failureCount !== failuresBefore)
+                return;
+            this.reason = null;
+            this.options.onRecovered?.();
+        }
+        catch (error) {
+            this.failedAt = Date.now();
+            this.options.onRetryFailed?.(error);
+        }
+    }
+}
+//# sourceMappingURL=embedder-recovery.js.map

package/dist/learning/qe-reasoning-bank-types.d.ts

@@ -38,6 +38,11 @@ export interface QEReasoningBankConfig {
     coherenceThreshold?: number;
     /** Optional RVF dual-writer for vector replication (Phase 3) */
     rvfDualWriter?: RvfDualWriter;
+    /**
+     * #1204: after the embedder fails, wait this long before trying it again
+     * (in the background). Until then patterns are stored and searched by text.
+     */
+    embedderRetryIntervalMs?: number;
 }
 /**
  * Default configuration

package/dist/learning/qe-reasoning-bank.d.ts

@@ -50,6 +50,11 @@ export declare class QEReasoningBank implements IQEReasoningBank {
     private readonly coherenceService?;
     private readonly config;
     private readonly embeddingProvenance;
+    /**
+     * #1187/#1204: whether the embedder works. While it does not, patterns are
+     * stored text-only and searched by text; it is retried in the background.
+     */
+    private readonly embedderRecovery;
     private patternStore;
     private initialized;
     private sqliteStore;
@@ -168,6 +173,22 @@ export declare class QEReasoningBank implements IQEReasoningBank {
      * for ARM64 or when transformers module cannot be loaded.
      */
     embed(text: string): Promise<number[]>;
+    /**
+     * #1187: embed for pattern store, search and routing, or null when the
+     * embedder endpoint fails at runtime. Callers then store the pattern
+     * text-only and search by text, as they do without an embedding runtime.
+     * ADR-097 still holds: no hash vector stands in for an endpoint vector.
+     * #1204: after a failure the embedder is not called again until a
+     * background retry, due `embedderRetryIntervalMs` later, finds it working.
+     */
+    private embedForPatterns;
+    /**
+     * #1204: why the embedder failed, while patterns are stored and searched
+     * by text only because of that failure; null while it works or was never
+     * needed. A missing embedding runtime with no endpoint configured is not a
+     * failure: there embed() returns a hash vector, which is stored text-only.
+     */
+    getEmbedderFailure(): string | null;
     /**
      * Get statistics
      */

package/dist/learning/qe-reasoning-bank.js

@@ -23,6 +23,7 @@ import { createPatternStore, } from './pattern-store.js';
 import { createSQLitePatternStore, } from './sqlite-persistence.js';
 import { getWitnessChain } from '../audit/witness-chain.js';
 import { getActiveEmbeddingSpaceIdentity } from './real-embeddings.js';
+import { EmbedderRecovery } from './embedder-recovery.js';
 import { reconcileRvfWithSqlite } from './rvf-reconcile.js';
 import { PatternMutationError } from './pattern-mutation-error.js';
 // Import extracted modules
@@ -67,6 +68,11 @@ export class QEReasoningBank {
     coherenceService;
     config;
     embeddingProvenance = new WeakMap();
+    /**
+     * #1187/#1204: whether the embedder works. While it does not, patterns are
+     * stored text-only and searched by text; it is retried in the background.
+     */
+    embedderRecovery;
     patternStore;
     initialized = false;
     sqliteStore = null;
@@ -106,6 +112,11 @@ export class QEReasoningBank {
         this.eventBus = eventBus;
         this.coherenceService = coherenceService;
         this.config = { ...DEFAULT_QE_REASONING_BANK_CONFIG, ...config };
+        this.embedderRecovery = new EmbedderRecovery({
+            retryIntervalMs: this.config.embedderRetryIntervalMs,
+            onRecovered: () => logger.info('Embedder available again: patterns are stored and searched with vectors'),
+            onRetryFailed: (error) => logger.debug('Embedder still unavailable', { error: toErrorMessage(error) }),
+        });
         if (config.rvfDualWriter) {
             this.rvfDualWriter = config.rvfDualWriter;
         }
@@ -318,14 +329,14 @@ export class QEReasoningBank {
         }
         // Generate embedding if not provided
         if (!options.embedding) {
-            const embedding = await this.embed(`${options.name} ${options.description} ${options.context?.tags?.join(' ') || ''}`);
+            const embedding = await this.embedForPatterns(`${options.name} ${options.description} ${options.context?.tags?.join(' ') || ''}`);
             // #981: with no embedding runtime, embed() returns a hash-fallback vector
             // that has no embedding-space provenance, and the pattern store refuses
             // to persist it (VECTOR_SPACE_UNVERIFIED), losing the pattern. Store it
             // text-only instead, as search already does for such vectors; the vector
             // is filled later by `aqe learning backfill-embeddings`.
-            const hasProvenance = this.embeddingProvenance.has(embedding)
-                || Boolean(getActiveEmbeddingSpaceIdentity()?.spaceId);
+            const hasProvenance = embedding !== null && (this.embeddingProvenance.has(embedding)
+                || Boolean(getActiveEmbeddingSpaceIdentity()?.spaceId));
             if (hasProvenance) {
                 options = { ...options, embedding };
             }
@@ -389,10 +400,10 @@ export class QEReasoningBank {
                 searchQuery = '';
             }
             else {
-                const embedding = await this.embed(query);
+                const embedding = await this.embedForPatterns(query);
                 // Hash fallback and resized vectors do not belong to the active provider.
                 // Keep their text searchable without claiming semantic comparability.
-                searchQuery = this.embeddingProvenance.has(embedding) ? embedding : query;
+                searchQuery = embedding && this.embeddingProvenance.has(embedding) ? embedding : query;
                 useVectorSearch = Array.isArray(searchQuery);
             }
         }
@@ -501,9 +512,9 @@ export class QEReasoningBank {
                 detectedDomains.push('test-generation'); // Default
             }
             // 2. Search for similar patterns
-            const embedding = await this.embed(request.task);
-            const embeddingSpaceId = this.embeddingProvenance.get(embedding);
-            const patternResults = await this.patternStore.search(embeddingSpaceId ? embedding : request.task, {
+            const embedding = await this.embedForPatterns(request.task);
+            const embeddingSpaceId = embedding ? this.embeddingProvenance.get(embedding) : undefined;
+            const patternResults = await this.patternStore.search(embedding && embeddingSpaceId ? embedding : request.task, {
                 limit: this.config.maxRoutingCandidates,
                 domain: detectedDomains[0],
                 useVectorSearch: Boolean(embeddingSpaceId),
@@ -695,6 +706,41 @@ export class QEReasoningBank {
         // Hash-based fallback (always works, including ARM64)
         return hashEmbedding(text, this.config.embeddingDimension);
     }
+    /**
+     * #1187: embed for pattern store, search and routing, or null when the
+     * embedder endpoint fails at runtime. Callers then store the pattern
+     * text-only and search by text, as they do without an embedding runtime.
+     * ADR-097 still holds: no hash vector stands in for an endpoint vector.
+     * #1204: after a failure the embedder is not called again until a
+     * background retry, due `embedderRetryIntervalMs` later, finds it working.
+     */
+    async embedForPatterns(text) {
+        if (!this.embedderRecovery.isAvailable())
+            return null;
+        try {
+            return await this.embed(text);
+        }
+        catch (error) {
+            const reason = toErrorMessage(error);
+            const message = 'Embedder failed: storing and searching patterns by text only';
+            if (this.embedderRecovery.markUnavailable(reason)) {
+                logger.warn(message, { error: reason });
+            }
+            else {
+                logger.debug(message, { error: reason });
+            }
+            return null;
+        }
+    }
+    /**
+     * #1204: why the embedder failed, while patterns are stored and searched
+     * by text only because of that failure; null while it works or was never
+     * needed. A missing embedding runtime with no endpoint configured is not a
+     * failure: there embed() returns a hash vector, which is stored text-only.
+     */
+    getEmbedderFailure() {
+        return this.embedderRecovery.unavailableReason;
+    }
     /**
      * Get statistics
      */

package/dist/learning/real-embeddings.d.ts

@@ -75,6 +75,29 @@ export declare function getEmbeddingDimension(): number;
  * Reset initialization state (for testing)
  */
 export declare function resetInitialization(): void;
+/**
+ * #1187: allow a new initialization attempt after a failed one. A failed
+ * initialization is otherwise remembered for the whole process, so a short
+ * embedding-endpoint outage at startup would disable embeddings until the
+ * process restarts. Callers rate-limit retries. No-op unless the last
+ * initialization failed.
+ */
+export declare function allowEmbedderInitRetry(): void;
+/**
+ * #1204: check whether a failed embedder works again, for callers that retry
+ * it after a cooldown. Resolves when the embedder is usable; throws otherwise.
+ * Other callers keep failing fast while the check runs, so a host that never
+ * answers stalls only the (background) check, never them:
+ * - after a failed initialization, initialize again while the failure stays
+ *   recorded; concurrent rechecks share that attempt;
+ * - after runtime failures that opened the endpoint circuit breaker, probe the
+ *   endpoint as the breaker's single trial request, so recovery does not wait
+ *   for the breaker window as well as the caller's retry interval. The probe
+ *   must find the identity this process bound to (ADR-097): a changed
+ *   fingerprint means vectors would not be comparable.
+ * Callers rate-limit rechecks, then embed a canary to confirm.
+ */
+export declare function recheckEmbedder(config?: Partial<EmbeddingConfig>): Promise<void>;
 /**
  * ADR-097: Returns true when the embedding pipeline is the external endpoint path,
  * false when it's the in-process transformer (or uninitialized).

package/dist/learning/real-embeddings.js

@@ -17,6 +17,8 @@ let embeddingModel = null;
 let initPromise = null;
 let initializationFailed = false;
 let failureReason = '';
+/** #1204: an in-flight recheckEmbedder() re-initialization, shared by callers. */
+let initRetryPromise = null;
 /**
  * Set when ADR-097 endpoint path is active. Tracked separately so callers
  * (tests, observability) can verify endpoint vs in-process unambiguously.
@@ -78,8 +80,10 @@ function isNonSemanticText(text) {
 /**
  * Initialize the transformer model
  */
-async function initializeModel(config = {}) {
-    if (initializationFailed) {
+async function initializeModel(config = {}, retryingAfterFailure = false) {
+    // #1204: a recheckEmbedder() retry runs while the failure is still recorded,
+    // so other callers keep failing fast instead of waiting on a dead host.
+    if (initializationFailed && !retryingAfterFailure) {
         throw new Error(`Transformer initialization previously failed: ${failureReason}`);
     }
     if (initPromise) {
@@ -134,6 +138,8 @@ async function initializeModel(config = {}) {
                     console.warn(`[RealEmbeddings] Could not persist endpoint identity: ${toErrorMessage(persistErr)}`);
                 }
                 embeddingModel = createEndpointPipeline(endpointClient);
+                initializationFailed = false;
+                failureReason = '';
                 return;
             }
             // In-process branch — only here do we import the transformers package.
@@ -179,6 +185,8 @@ async function initializeModel(config = {}) {
                 implementation: `transformers:${fullConfig.quantized ? 'quantized' : 'full'}`,
                 runtimeFingerprint: fingerprintEmbeddingRuntime(canaryVector),
             });
+            initializationFailed = false;
+            failureReason = '';
             const loadTime = performance.now() - startTime;
             console.log(`[RealEmbeddings] Model loaded in ${loadTime.toFixed(0)}ms`);
         }
@@ -393,6 +401,65 @@ export function resetInitialization() {
     // AQE_MEMORY_PATH between runs don't write into a stale handle.
     resetEmbedderIdentityStore();
 }
+/**
+ * #1187: allow a new initialization attempt after a failed one. A failed
+ * initialization is otherwise remembered for the whole process, so a short
+ * embedding-endpoint outage at startup would disable embeddings until the
+ * process restarts. Callers rate-limit retries. No-op unless the last
+ * initialization failed.
+ */
+export function allowEmbedderInitRetry() {
+    if (!initializationFailed)
+        return;
+    initializationFailed = false;
+    failureReason = '';
+    initPromise = null;
+    if (endpointClient) {
+        endpointClient.close();
+    }
+    endpointClient = null;
+    endpointIdentity = null;
+}
+/**
+ * #1204: check whether a failed embedder works again, for callers that retry
+ * it after a cooldown. Resolves when the embedder is usable; throws otherwise.
+ * Other callers keep failing fast while the check runs, so a host that never
+ * answers stalls only the (background) check, never them:
+ * - after a failed initialization, initialize again while the failure stays
+ *   recorded; concurrent rechecks share that attempt;
+ * - after runtime failures that opened the endpoint circuit breaker, probe the
+ *   endpoint as the breaker's single trial request, so recovery does not wait
+ *   for the breaker window as well as the caller's retry interval. The probe
+ *   must find the identity this process bound to (ADR-097): a changed
+ *   fingerprint means vectors would not be comparable.
+ * Callers rate-limit rechecks, then embed a canary to confirm.
+ */
+export async function recheckEmbedder(config = {}) {
+    if (initializationFailed) {
+        initRetryPromise ??= retryInitialization(config).finally(() => {
+            initRetryPromise = null;
+        });
+        return initRetryPromise;
+    }
+    const client = endpointClient;
+    if (!client || !client.getBreakerState().open)
+        return;
+    const identity = await client.probeThroughOpenBreaker();
+    if (endpointIdentity && identity.fingerprint !== endpointIdentity.fingerprint) {
+        throw new Error(`[RealEmbeddings] endpoint identity changed since this process bound to it ` +
+            `(fingerprint ${endpointIdentity.fingerprint} -> ${identity.fingerprint}); ` +
+            `restart to use the new embedding space`);
+    }
+}
+async function retryInitialization(config) {
+    if (endpointClient) {
+        endpointClient.close();
+    }
+    endpointClient = null;
+    endpointIdentity = null;
+    initPromise = null;
+    await initializeModel(config, true);
+}
 /**
  * ADR-097: Returns true when the embedding pipeline is the external endpoint path,
  * false when it's the in-process transformer (or uninitialized).

package/dist/learning/real-qe-reasoning-bank.d.ts

@@ -28,6 +28,11 @@ export interface RealQEReasoningBankConfig {
     };
     /** Coherence energy threshold for pattern promotion (ADR-052) */
     coherenceThreshold?: number;
+    /**
+     * #1187: after the embedder fails (at startup or at runtime), wait this long
+     * before trying it again. Until then pattern search is lexical-only.
+     */
+    embedderRetryIntervalMs?: number;
 }
 export declare const DEFAULT_REAL_CONFIG: RealQEReasoningBankConfig;
 /** Ranking evidence keeps lexical relevance separate from vector reuse safety. */
@@ -42,7 +47,8 @@ export interface RealQEPatternSearchResult {
  * #1022: how pattern search works in this process. Without an embedding
  * runtime the bank stores patterns text-only (vectors pending, filled later by
  * `aqe learning backfill-embeddings`) and searches by SQLite FTS text matching
- * only. No similarity score is computed in that mode.
+ * only. No similarity score is computed in that mode. #1187: the same mode
+ * applies while an embedder that failed at runtime is unavailable.
  */
 export interface PatternSearchStatus {
     mode: 'vector' | 'lexical-only';
@@ -132,8 +138,11 @@ export declare class RealQEReasoningBank {
     private hnswIndex;
     private patternIdMap;
     private initialized;
-    /** #1022: set when no embedding runtime is available; null when vectors work. */
-    private embedderUnavailableReason;
+    /**
+     * #1022/#1187/#1204: whether the embedder works. While it does not, patterns
+     * are stored text-only and searched by text; it is retried in the background.
+     */
+    private readonly embedderRecovery;
     private stats;
     private routingLatencies;
     private readonly asymmetricEngine;
@@ -145,6 +154,17 @@ export declare class RealQEReasoningBank {
     initialize(): Promise<void>;
     /** #1022: whether pattern search uses vectors or text matching only. */
     getPatternSearchStatus(): PatternSearchStatus;
+    /**
+     * #1187: embed `text`, or return undefined when the embedder is unavailable.
+     * A runtime failure switches this bank to lexical-only (reported by
+     * getPatternSearchStatus): callers then store patterns text-only and search
+     * by text. The embedder is tried again once `embedderRetryIntervalMs` has
+     * passed, so a short outage, at startup or later, does not last until restart.
+     * #1204: that retry runs in the background; this call stays text-only.
+     */
+    private embedOrUndefined;
+    /** #1187: the retry probe; vector search needs the HNSW index as well. */
+    private probeEmbedderForRetry;
     /**
      * Initialize HNSW index for fast similarity search
      */
@@ -170,7 +190,7 @@ export declare class RealQEReasoningBank {
         domain?: QEDomain;
         minSimilarity?: number;
     }): Promise<Result<RealQEPatternSearchResult[]>>;
-    /** Append HNSW vector matches for `query` to `patterns`. */
+    /** Append HNSW vector matches for the query embedding to `patterns`. */
     private collectVectorMatches;
     /**
      * Route a task to optimal agent