agentic-qe 3.15.10 → 3.15.12

package/dist/learning/real-qe-reasoning-bank.js

@@ -23,6 +23,7 @@ import { blendFtsScore, DEFAULT_PATTERN_STORE_CONFIG } from './pattern-store.js'
 import { EMBEDDING_SPACE_CANARY } from './embedding-space.js';
 const logger = LoggerFactory.create('RealQEReasoningBank');
 import { computeRealEmbedding, isTransformerAvailable, getEmbeddingDimension, clearEmbeddingCache, getActiveEmbeddingSpaceIdentity, } from './real-embeddings.js';
+import { EmbedderRecovery, probeEmbedder } from './embedder-recovery.js';
 import { createSQLitePatternStore, } from './sqlite-persistence.js';
 import { detectQEDomain, detectQEDomains, mapQEDomainToAQE, QE_DOMAIN_LIST, shouldPromotePattern, } from './qe-patterns.js';
 import { getGuidance, getCombinedGuidance, generateGuidanceContext, checkAntiPatterns, } from './qe-guidance.js';
@@ -59,6 +60,7 @@ export const DEFAULT_REAL_CONFIG = {
         capabilities: 0.3,
     },
     coherenceThreshold: 0.4, // ADR-052: Coherence gate threshold
+    embedderRetryIntervalMs: 60_000,
 };
 // ============================================================================
 // Real QE ReasoningBank Implementation
@@ -80,6 +82,11 @@ export class RealQEReasoningBank {
     hnswIndex = null;
     patternIdMap = new Map();
     initialized = false;
+    /**
+     * #1022/#1187/#1204: whether the embedder works. While it does not, patterns
+     * are stored text-only and searched by text; it is retried in the background.
+     */
+    embedderRecovery;
     // Statistics tracking - using CircularBuffer for O(1) operations (ADR memory leak fix)
     stats = {
         routingRequests: 0,
@@ -155,6 +162,12 @@ export class RealQEReasoningBank {
         this.qeConfig = { ...DEFAULT_REAL_CONFIG, ...config };
         this.sqliteStore = createSQLitePatternStore(this.qeConfig.sqlite);
         this.asymmetricEngine = new AsymmetricLearningEngine();
+        this.embedderRecovery = new EmbedderRecovery({
+            retryIntervalMs: this.qeConfig.embedderRetryIntervalMs,
+            probe: () => this.probeEmbedderForRetry(),
+            onRecovered: () => logger.info('Embedder available again: pattern search uses vectors'),
+            onRetryFailed: (error) => logger.debug('Embedder still unavailable', { error: toErrorMessage(error) }),
+        });
     }
     /**
      * Initialize the QE ReasoningBank
@@ -167,12 +180,24 @@ export class RealQEReasoningBank {
         await this.sqliteStore.initialize();
         logger.info('SQLite persistence initialized');
         // Establish executable provenance before any persisted vectors enter ANN.
-        await computeRealEmbedding(EMBEDDING_SPACE_CANARY, this.qeConfig.embeddings);
-        // Initialize HNSW index
-        await this.initializeHNSW();
-        logger.info('HNSW index initialized');
-        // Load existing patterns into HNSW
-        await this.loadPatternsIntoHNSW();
+        // #1022: without an embedding runtime (no @huggingface/transformers and no
+        // AQE_EMBEDDER_ENDPOINT) run text-only instead of failing: no vector is
+        // computed, stored or searched (ADR-097: no hash-embedding fallback).
+        try {
+            await computeRealEmbedding(EMBEDDING_SPACE_CANARY, this.qeConfig.embeddings);
+        }
+        catch (error) {
+            this.embedderRecovery.markUnavailable(toErrorMessage(error));
+            logger.warn('No embedding runtime: patterns are stored without vectors (pending ' +
+                '`aqe learning backfill-embeddings`) and searched by text matching only', { reason: this.embedderRecovery.unavailableReason });
+        }
+        if (this.embedderRecovery.unavailableReason === null) {
+            // Initialize HNSW index
+            await this.initializeHNSW();
+            logger.info('HNSW index initialized');
+            // Load existing patterns into HNSW
+            await this.loadPatternsIntoHNSW();
+        }
         // Load foundational patterns ONLY if this is a brand-new database (no tables or genuinely empty).
         // CRITICAL: Do NOT overwrite existing data. If patterns were unexpectedly wiped,
         // loading 22 seed patterns would mask the data loss. Log a warning instead.
@@ -193,6 +218,42 @@ export class RealQEReasoningBank {
         const initTime = performance.now() - startTime;
         logger.info('Fully initialized', { durationMs: Math.round(initTime) });
     }
+    /** #1022: whether pattern search uses vectors or text matching only. */
+    getPatternSearchStatus() {
+        const reason = this.embedderRecovery.unavailableReason;
+        return reason === null ? { mode: 'vector' } : { mode: 'lexical-only', reason };
+    }
+    /**
+     * #1187: embed `text`, or return undefined when the embedder is unavailable.
+     * A runtime failure switches this bank to lexical-only (reported by
+     * getPatternSearchStatus): callers then store patterns text-only and search
+     * by text. The embedder is tried again once `embedderRetryIntervalMs` has
+     * passed, so a short outage, at startup or later, does not last until restart.
+     * #1204: that retry runs in the background; this call stays text-only.
+     */
+    async embedOrUndefined(text) {
+        if (!this.embedderRecovery.isAvailable())
+            return undefined;
+        try {
+            return await computeRealEmbedding(text, this.qeConfig.embeddings);
+        }
+        catch (error) {
+            const reason = `Embedder failed at runtime: ${toErrorMessage(error)}`;
+            if (this.embedderRecovery.markUnavailable(reason)) {
+                logger.warn('Embedder failed: patterns are stored without vectors and searched by ' +
+                    'text matching only until it recovers', { reason });
+            }
+            return undefined;
+        }
+    }
+    /** #1187: the retry probe; vector search needs the HNSW index as well. */
+    async probeEmbedderForRetry() {
+        await probeEmbedder(this.qeConfig.embeddings);
+        if (!this.hnswIndex) {
+            await this.initializeHNSW();
+            await this.loadPatternsIntoHNSW();
+        }
+    }
     /**
      * Initialize HNSW index for fast similarity search
      */
@@ -311,9 +372,10 @@ export class RealQEReasoningBank {
     async storeQEPattern(options) {
         // Require explicit initialization - don't auto-initialize to avoid memory issues
         try {
-            // Generate REAL embedding using transformer model
+            // Generate REAL embedding using transformer model. #1022/#1187: without
+            // a working embedder the pattern is stored text-only (vector pending).
             const textToEmbed = `${options.name} ${options.description || ''} ${options.context?.tags?.join(' ') || ''}`;
-            const embedding = await computeRealEmbedding(textToEmbed, this.qeConfig.embeddings);
+            const embedding = await this.embedOrUndefined(textToEmbed);
             // Detect QE domain
             const qeDomain = detectQEDomain(textToEmbed) || 'test-generation';
             // ADR-058: Check MemoryWriteGate for contradictions before storing
@@ -374,7 +436,7 @@ export class RealQEReasoningBank {
             // Store in SQLite
             this.sqliteStore.storePattern(pattern, embedding);
             // Add to HNSW index
-            if (this.hnswIndex) {
+            if (this.hnswIndex && embedding) {
                 const index = this.hnswIndex.getCurrentCount();
                 this.hnswIndex.addPoint(embedding, index);
                 this.patternIdMap.set(index, patternId);
@@ -410,38 +472,16 @@ export class RealQEReasoningBank {
         // Require explicit initialization - don't auto-initialize to avoid memory issues
         const startTime = performance.now();
         try {
-            if (!this.hnswIndex) {
-                return err(new Error('HNSW index not available'));
-            }
-            // Generate REAL embedding for query
-            const queryEmbedding = await computeRealEmbedding(query, this.qeConfig.embeddings);
-            // Search HNSW index
             const limit = options.limit || 10;
-            const results = this.hnswIndex.searchKnn(queryEmbedding, limit * 2); // Get extra for filtering
             const patterns = [];
-            for (let i = 0; i < results.neighbors.length; i++) {
-                const hnswIndex = results.neighbors[i];
-                const distance = results.distances[i];
-                const similarity = 1 - distance; // Convert distance to similarity
-                if (options.minSimilarity !== undefined && similarity < options.minSimilarity) {
-                    continue;
+            // #1022/#1187: without a working embedder only the lexical (FTS) branch
+            // below runs; its hits are labelled matchType 'lexical'.
+            const queryEmbedding = await this.embedOrUndefined(query);
+            if (queryEmbedding) {
+                if (!this.hnswIndex) {
+                    return err(new Error('HNSW index not available'));
                 }
-                const patternId = this.patternIdMap.get(hnswIndex);
-                if (!patternId)
-                    continue;
-                const pattern = this.sqliteStore.getPattern(patternId);
-                if (!pattern)
-                    continue;
-                // Filter by domain if specified
-                if (options.domain && pattern.qeDomain !== options.domain) {
-                    continue;
-                }
-                const reuse = DEFAULT_PATTERN_STORE_CONFIG.reuseOptimization;
-                const ageDays = (Date.now() - new Date(pattern.lastUsedAt).getTime()) / 86_400_000;
-                patterns.push({ pattern, similarity, score: similarity, matchType: 'vector',
-                    canReuse: Boolean(pattern.reusable && similarity >= reuse.minSimilarityForReuse
-                        && pattern.successRate >= reuse.minSuccessRateForReuse && ageDays <= reuse.maxAgeForReuse),
-                });
+                this.collectVectorMatches(queryEmbedding, limit, options, patterns);
             }
             const existing = new Map(patterns.map(hit => [hit.pattern.id, hit]));
             for (const lexical of this.sqliteStore.searchFTS(query, limit * 2)) {
@@ -471,6 +511,35 @@ export class RealQEReasoningBank {
             return err(toError(error));
         }
     }
+    /** Append HNSW vector matches for the query embedding to `patterns`. */
+    collectVectorMatches(queryEmbedding, limit, options, patterns) {
+        // Search HNSW index
+        const results = this.hnswIndex.searchKnn(queryEmbedding, limit * 2); // Get extra for filtering
+        for (let i = 0; i < results.neighbors.length; i++) {
+            const hnswIndex = results.neighbors[i];
+            const distance = results.distances[i];
+            const similarity = 1 - distance; // Convert distance to similarity
+            if (options.minSimilarity !== undefined && similarity < options.minSimilarity) {
+                continue;
+            }
+            const patternId = this.patternIdMap.get(hnswIndex);
+            if (!patternId)
+                continue;
+            const pattern = this.sqliteStore.getPattern(patternId);
+            if (!pattern)
+                continue;
+            // Filter by domain if specified
+            if (options.domain && pattern.qeDomain !== options.domain) {
+                continue;
+            }
+            const reuse = DEFAULT_PATTERN_STORE_CONFIG.reuseOptimization;
+            const ageDays = (Date.now() - new Date(pattern.lastUsedAt).getTime()) / 86_400_000;
+            patterns.push({ pattern, similarity, score: similarity, matchType: 'vector',
+                canReuse: Boolean(pattern.reusable && similarity >= reuse.minSimilarityForReuse
+                    && pattern.successRate >= reuse.minSuccessRateForReuse && ageDays <= reuse.maxAgeForReuse),
+            });
+        }
+    }
     /**
      * Route a task to optimal agent
      */
@@ -701,6 +770,7 @@ export class RealQEReasoningBank {
                 : 0,
             embeddingCacheSize: 0, // Would need to track this in real-embeddings
             transformerAvailable: isTransformerAvailable(),
+            patternSearch: this.getPatternSearchStatus(),
             embeddingDimension: getEmbeddingDimension(),
             sqliteDbPath: this.qeConfig.sqlite.dbPath || '.agentic-qe/memory.db',
             // ADR-061: Asymmetric learning stats

package/dist/logging/redaction.js

@@ -21,26 +21,126 @@ const SENSITIVE_SUFFIXES = [
     'accesskey', 'secretkey', 'credential', 'credentials', 'cookie', 'authorization',
     'sessionid', 'connectionstring',
 ];
+/** Basic credentials are base64("user:password"); prose such as "Basic authentication" is not. */
+function looksLikeBasicCredential(value) {
+    const decoded = Buffer.from(value, 'base64').toString('latin1');
+    return decoded.includes(':') && /^[\x20-\x7e]+$/.test(decoded);
+}
+/**
+ * Bearer tokens are opaque, so prose ("use Bearer authorization") is told apart
+ * by shape: a short lowercase or Capitalized word, optionally ending a sentence.
+ * Anything with digits, mixed case, '_', '-', '/', '+', '=' or over 16 letters
+ * is treated as a token: when unsure, redact.
+ */
+function looksLikeProseWord(value) {
+    return /^[A-Z]?[a-z]+\.?$/.test(value) && value.replace(/\.$/, '').length <= 16;
+}
 /** Log-specific credential shapes, applied before the shared advisor patterns. */
 const LOG_PATTERNS = [
     // Anchored on '://' rather than a scheme prefix: `\b[a-z][a-z0-9+.-]*://`
     // backtracks quadratically on inputs such as 'a.a.a.…' (seconds per call at
     // 100 KB). The scheme is left in place; an empty user (://:pw@) is covered.
     [/(:\/\/)[^\s/@:"']*:[^\s/@"']+@/g, '$1<REDACTED:credentials>@'],
-    [/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}/gi, '$1 <REDACTED:credential>'],
+    // In a header any Basic value is a credential; outside one it must decode as user:password.
+    [/\b(authorization:\s*Basic)\s+[^\s"']+/gi, '$1 <REDACTED:credential>'],
+    [/\b(Basic)\s+([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9._~+/=-])/gi,
+        (match, scheme, value) => looksLikeBasicCredential(value) ? `${scheme} <REDACTED:credential>` : match],
+    // Same split for Bearer (#1196): any value in a header; outside one, only prose words are kept.
+    [/\b(authorization:\s*Bearer)\s+[^\s"']+/gi, '$1 <REDACTED:credential>'],
+    [/\b(Bearer)\s+([A-Za-z0-9._~+/=-]{8,})/gi,
+        (match, scheme, value) => looksLikeProseWord(value) ? match : `${scheme} <REDACTED:credential>`],
     [/\bsk-(?:ant-|proj-)?[A-Za-z0-9_-]{16,}/g, '<REDACTED:api_key>'],
     [/\b(set-cookie|cookie):\s*[^\n]+/gi, '$1: <REDACTED:cookie>'],
     // Compound query/assignment keys the shared pattern misses: access_token=, client_secret=
     [/\b([A-Za-z0-9_-]{0,64}?[_-](?:token|secret|password|passwd))=([^\s&"'#]+)/gi, '$1=<REDACTED:credential>'],
 ];
 /**
- * JSON / repr serialized fields ("apiKey":"…", 'password': 123). The key is
- * checked with isSensitiveLogKey() so text and structured values agree.
- * Unterminated strings are redacted to end of line (fail closed).
+ * JSON / repr serialized fields ("apiKey":"…", 'password': 123, token: {…}).
+ * Keys are quoted (JSON, Python repr) or bare (util.inspect); a bare key must
+ * start a word. The key is checked with isSensitiveLogKey() so text and
+ * structured values agree. Values are strings, numbers, or an opening '{' / '['
+ * whose whole object/array is then redacted. JSON embedded in a JSON string
+ * ({\"password\":\"…\"}) uses escaped quotes; its string value runs to the next
+ * escaped quote. Unterminated strings are redacted to end of line and
+ * unbalanced objects/arrays to end of text (fail closed).
  */
-const SERIALIZED_FIELD = /(["'])([A-Za-z0-9_.-]{1,64})\1(\s*:\s*)("(?:[^"\\\n]|\\.)*"?|'(?:[^'\\\n]|\\.)*'?|-?\d[\d.eE+-]*)/g;
+const SERIALIZED_FIELD = /(?:(\\?["'])([A-Za-z0-9_.-]{1,64})\1|(?<![A-Za-z0-9_.$-])([A-Za-z_$][A-Za-z0-9_.$-]{0,63}))(\s*:\s*)("(?:[^"\\\n]|\\.)*"?|'(?:[^'\\\n]|\\.)*'?|\\["']|-?\d[\d.eE+-]*|[{[])/g;
 function redactSerializedFields(text) {
-    return text.replace(SERIALIZED_FIELD, (match, q, key, sep) => isSensitiveLogKey(key) ? `${q}${key}${q}${sep}${q}<REDACTED:sensitive-key>${q}` : match);
+    const pattern = new RegExp(SERIALIZED_FIELD.source, SERIALIZED_FIELD.flags);
+    let out = '';
+    let copied = 0;
+    for (let m = pattern.exec(text); m !== null; m = pattern.exec(text)) {
+        const [match, quote = '', quotedKey, bareKey, sep, value] = m;
+        const key = quotedKey ?? bareKey;
+        const valueStart = m.index + match.length - value.length;
+        if (!isSensitiveLogKey(key)) {
+            // Not sensitive: resume inside the value, so keys nested in an object or
+            // serialized into a string ("body={\"password\":…}") are still checked.
+            pattern.lastIndex = valueStart + 1;
+            continue;
+        }
+        const end = value === '{' || value === '['
+            ? compoundValueEnd(text, valueStart, quote.length === 2)
+            : value.length === 2 && value[0] === '\\'
+                ? escapedStringEnd(text, valueStart + 2, value)
+                : m.index + match.length;
+        out += `${text.slice(copied, m.index)}${quote}${key}${quote}${sep}${quote}<REDACTED:sensitive-key>${quote}`;
+        copied = end;
+        // Resume after the redacted value, which is never scanned again.
+        pattern.lastIndex = end;
+    }
+    return copied === 0 ? text : out + text.slice(copied);
+}
+/**
+ * Index just past the escaped closing quote (\" or \'), or the end of the line
+ * when there is none. Each \X pair decodes to X; a decoded backslash escapes
+ * the next decoded character, so an inner \\\" does not end the string.
+ */
+function escapedStringEnd(text, start, quote) {
+    let innerEscape = false;
+    for (let i = start; i < text.length; i++) {
+        const pair = text[i] === '\\';
+        if (pair)
+            i++;
+        const decoded = text[i];
+        if (decoded === '\n' || decoded === undefined)
+            return i;
+        if (innerEscape)
+            innerEscape = false;
+        else if (pair && decoded === '\\')
+            innerEscape = true;
+        else if (pair && decoded === quote[1])
+            return i + 1;
+    }
+    return text.length;
+}
+/**
+ * Index just past the object/array opened at `start`, skipping quoted strings
+ * (delimited by \" when the object is itself escaped inside a JSON string).
+ * An unbalanced value runs to the end of the text, so it is redacted rather than printed.
+ */
+function compoundValueEnd(text, start, escaped) {
+    let depth = 0;
+    for (let i = start; i < text.length; i++) {
+        const ch = text[i];
+        if (escaped && ch === '\\' && (text[i + 1] === '"' || text[i + 1] === "'")) {
+            i = escapedStringEnd(text, i + 2, text.slice(i, i + 2)) - 1;
+        }
+        else if (!escaped && (ch === '"' || ch === "'")) {
+            // A newline also ends a string: neither JSON nor repr strings contain a raw one.
+            for (i++; i < text.length && text[i] !== ch && text[i] !== '\n'; i++) {
+                if (text[i] === '\\')
+                    i++;
+            }
+        }
+        else if (ch === '{' || ch === '[') {
+            depth++;
+        }
+        else if ((ch === '}' || ch === ']') && --depth === 0) {
+            return i + 1;
+        }
+    }
+    return text.length;
 }
 export function isSensitiveLogKey(key) {
     const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, '');
@@ -55,8 +155,11 @@ export function redactLogText(text, maxLength = DEFAULTS.maxStringLength) {
     if (text.length > MAX_SCAN_LENGTH)
         return `[REDACTED:size-limit ${text.length} chars]`;
     let result = text;
-    for (const [regex, replacement] of LOG_PATTERNS)
-        result = result.replace(regex, replacement);
+    for (const [regex, replacement] of LOG_PATTERNS) {
+        result = typeof replacement === 'string'
+            ? result.replace(regex, replacement)
+            : result.replace(regex, replacement);
+    }
     result = redactSerializedFields(result);
     result = redact(result, 'balanced').text;
     return result.length > maxLength
@@ -71,11 +174,11 @@ export function redactLogValue(value, options = {}) {
 export function formatErrorForLog(error) {
     try {
         if (error instanceof Error) {
-            let text = redactLogText(error.stack || `${error.name}: ${error.message}`);
+            let text = formatErrorText(error);
             let cause = error.cause;
             for (let hops = 0; cause !== undefined && hops < 3; hops++) {
                 text += `\nCaused by: ${cause instanceof Error
-                    ? redactLogText(cause.stack || `${cause.name}: ${cause.message}`)
+                    ? formatErrorText(cause)
                     : JSON.stringify(redactLogValue(cause, { maxDepth: 3 }))}`;
                 cause = cause instanceof Error ? cause.cause : undefined;
             }
@@ -89,6 +192,31 @@ export function formatErrorForLog(error) {
         return '[REDACTED:sanitization-failed]';
     }
 }
+/** Fields kept from an error besides its message and stack: they identify the failure, not its data. */
+const SAFE_ERROR_FIELDS = ['name', 'code', 'status'];
+const MAX_ERROR_FIELD_LENGTH = 200;
+/** name/code/status as a redacted record; strings are redacted, numbers kept, anything else dropped. */
+function safeErrorFields(error) {
+    const fields = {};
+    for (const field of SAFE_ERROR_FIELDS) {
+        const descriptor = Object.getOwnPropertyDescriptor(error, field)
+            ?? (field === 'name' ? { value: error.name } : undefined);
+        const value = descriptor && 'value' in descriptor ? descriptor.value : undefined;
+        if (typeof value === 'string')
+            fields[field] = redactLogText(value, MAX_ERROR_FIELD_LENGTH);
+        else if (typeof value === 'number' && Number.isFinite(value))
+            fields[field] = value;
+    }
+    return fields;
+}
+/** Redacted stack (or name: message) followed by the safe structured fields, unless they add nothing. */
+function formatErrorText(error) {
+    const text = redactLogText(error.stack || `${error.name}: ${error.message}`);
+    const fields = safeErrorFields(error);
+    const keys = Object.keys(fields);
+    const nameOnly = keys.length === 0 || (keys.length === 1 && text.startsWith(`${fields.name}`));
+    return nameOnly ? text : `${text} ${JSON.stringify(fields)}`;
+}
 function sanitize(value, opts, depth, ancestors) {
     try {
         if (value === null || value === undefined || typeof value === 'number' || typeof value === 'boolean')
@@ -136,9 +264,11 @@ function sanitizeError(error, opts, depth, path) {
     };
     if (typeof error.stack === 'string')
         out.stack = redactLogText(error.stack, opts.maxStringLength);
-    const code = error.code;
-    if (typeof code === 'string' || typeof code === 'number')
+    const { code, status } = safeErrorFields(error);
+    if (code !== undefined)
         out.code = code;
+    if (status !== undefined)
+        out.status = status;
     const cause = error.cause;
     if (cause !== undefined)
         out.cause = sanitize(cause, opts, depth + 1, path);