agentic-qe 3.15.11 → 3.15.13

package/dist/learning/real-qe-reasoning-bank.js

@@ -23,6 +23,7 @@ import { blendFtsScore, DEFAULT_PATTERN_STORE_CONFIG } from './pattern-store.js'
 import { EMBEDDING_SPACE_CANARY } from './embedding-space.js';
 const logger = LoggerFactory.create('RealQEReasoningBank');
 import { computeRealEmbedding, isTransformerAvailable, getEmbeddingDimension, clearEmbeddingCache, getActiveEmbeddingSpaceIdentity, } from './real-embeddings.js';
+import { EmbedderRecovery, probeEmbedder } from './embedder-recovery.js';
 import { createSQLitePatternStore, } from './sqlite-persistence.js';
 import { detectQEDomain, detectQEDomains, mapQEDomainToAQE, QE_DOMAIN_LIST, shouldPromotePattern, } from './qe-patterns.js';
 import { getGuidance, getCombinedGuidance, generateGuidanceContext, checkAntiPatterns, } from './qe-guidance.js';
@@ -59,6 +60,7 @@ export const DEFAULT_REAL_CONFIG = {
         capabilities: 0.3,
     },
     coherenceThreshold: 0.4, // ADR-052: Coherence gate threshold
+    embedderRetryIntervalMs: 60_000,
 };
 // ============================================================================
 // Real QE ReasoningBank Implementation
@@ -80,8 +82,11 @@ export class RealQEReasoningBank {
     hnswIndex = null;
     patternIdMap = new Map();
     initialized = false;
-    /** #1022: set when no embedding runtime is available; null when vectors work. */
-    embedderUnavailableReason = null;
+    /**
+     * #1022/#1187/#1204: whether the embedder works. While it does not, patterns
+     * are stored text-only and searched by text; it is retried in the background.
+     */
+    embedderRecovery;
     // Statistics tracking - using CircularBuffer for O(1) operations (ADR memory leak fix)
     stats = {
         routingRequests: 0,
@@ -157,6 +162,12 @@ export class RealQEReasoningBank {
         this.qeConfig = { ...DEFAULT_REAL_CONFIG, ...config };
         this.sqliteStore = createSQLitePatternStore(this.qeConfig.sqlite);
         this.asymmetricEngine = new AsymmetricLearningEngine();
+        this.embedderRecovery = new EmbedderRecovery({
+            retryIntervalMs: this.qeConfig.embedderRetryIntervalMs,
+            probe: () => this.probeEmbedderForRetry(),
+            onRecovered: () => logger.info('Embedder available again: pattern search uses vectors'),
+            onRetryFailed: (error) => logger.debug('Embedder still unavailable', { error: toErrorMessage(error) }),
+        });
     }
     /**
      * Initialize the QE ReasoningBank
@@ -176,11 +187,11 @@ export class RealQEReasoningBank {
             await computeRealEmbedding(EMBEDDING_SPACE_CANARY, this.qeConfig.embeddings);
         }
         catch (error) {
-            this.embedderUnavailableReason = toErrorMessage(error);
+            this.embedderRecovery.markUnavailable(toErrorMessage(error));
             logger.warn('No embedding runtime: patterns are stored without vectors (pending ' +
-                '`aqe learning backfill-embeddings`) and searched by text matching only', { reason: this.embedderUnavailableReason });
+                '`aqe learning backfill-embeddings`) and searched by text matching only', { reason: this.embedderRecovery.unavailableReason });
         }
-        if (this.embedderUnavailableReason === null) {
+        if (this.embedderRecovery.unavailableReason === null) {
             // Initialize HNSW index
             await this.initializeHNSW();
             logger.info('HNSW index initialized');
@@ -209,9 +220,39 @@ export class RealQEReasoningBank {
     }
     /** #1022: whether pattern search uses vectors or text matching only. */
     getPatternSearchStatus() {
-        return this.embedderUnavailableReason === null
-            ? { mode: 'vector' }
-            : { mode: 'lexical-only', reason: this.embedderUnavailableReason };
+        const reason = this.embedderRecovery.unavailableReason;
+        return reason === null ? { mode: 'vector' } : { mode: 'lexical-only', reason };
+    }
+    /**
+     * #1187: embed `text`, or return undefined when the embedder is unavailable.
+     * A runtime failure switches this bank to lexical-only (reported by
+     * getPatternSearchStatus): callers then store patterns text-only and search
+     * by text. The embedder is tried again once `embedderRetryIntervalMs` has
+     * passed, so a short outage, at startup or later, does not last until restart.
+     * #1204: that retry runs in the background; this call stays text-only.
+     */
+    async embedOrUndefined(text) {
+        if (!this.embedderRecovery.isAvailable())
+            return undefined;
+        try {
+            return await computeRealEmbedding(text, this.qeConfig.embeddings);
+        }
+        catch (error) {
+            const reason = `Embedder failed at runtime: ${toErrorMessage(error)}`;
+            if (this.embedderRecovery.markUnavailable(reason)) {
+                logger.warn('Embedder failed: patterns are stored without vectors and searched by ' +
+                    'text matching only until it recovers', { reason });
+            }
+            return undefined;
+        }
+    }
+    /** #1187: the retry probe; vector search needs the HNSW index as well. */
+    async probeEmbedderForRetry() {
+        await probeEmbedder(this.qeConfig.embeddings);
+        if (!this.hnswIndex) {
+            await this.initializeHNSW();
+            await this.loadPatternsIntoHNSW();
+        }
     }
     /**
      * Initialize HNSW index for fast similarity search
@@ -331,12 +372,10 @@ export class RealQEReasoningBank {
     async storeQEPattern(options) {
         // Require explicit initialization - don't auto-initialize to avoid memory issues
         try {
-            // Generate REAL embedding using transformer model. #1022: without an
-            // embedding runtime the pattern is stored text-only (vector pending).
+            // Generate REAL embedding using transformer model. #1022/#1187: without
+            // a working embedder the pattern is stored text-only (vector pending).
             const textToEmbed = `${options.name} ${options.description || ''} ${options.context?.tags?.join(' ') || ''}`;
-            const embedding = this.embedderUnavailableReason === null
-                ? await computeRealEmbedding(textToEmbed, this.qeConfig.embeddings)
-                : undefined;
+            const embedding = await this.embedOrUndefined(textToEmbed);
             // Detect QE domain
             const qeDomain = detectQEDomain(textToEmbed) || 'test-generation';
             // ADR-058: Check MemoryWriteGate for contradictions before storing
@@ -435,13 +474,14 @@ export class RealQEReasoningBank {
         try {
             const limit = options.limit || 10;
             const patterns = [];
-            // #1022: without an embedding runtime only the lexical (FTS) branch
+            // #1022/#1187: without a working embedder only the lexical (FTS) branch
             // below runs; its hits are labelled matchType 'lexical'.
-            if (this.embedderUnavailableReason === null) {
+            const queryEmbedding = await this.embedOrUndefined(query);
+            if (queryEmbedding) {
                 if (!this.hnswIndex) {
                     return err(new Error('HNSW index not available'));
                 }
-                await this.collectVectorMatches(query, limit, options, patterns);
+                this.collectVectorMatches(queryEmbedding, limit, options, patterns);
             }
             const existing = new Map(patterns.map(hit => [hit.pattern.id, hit]));
             for (const lexical of this.sqliteStore.searchFTS(query, limit * 2)) {
@@ -471,10 +511,8 @@ export class RealQEReasoningBank {
             return err(toError(error));
         }
     }
-    /** Append HNSW vector matches for `query` to `patterns`. */
-    async collectVectorMatches(query, limit, options, patterns) {
-        // Generate REAL embedding for query
-        const queryEmbedding = await computeRealEmbedding(query, this.qeConfig.embeddings);
+    /** Append HNSW vector matches for the query embedding to `patterns`. */
+    collectVectorMatches(queryEmbedding, limit, options, patterns) {
         // Search HNSW index
         const results = this.hnswIndex.searchKnn(queryEmbedding, limit * 2); // Get extra for filtering
         for (let i = 0; i < results.neighbors.length; i++) {

package/dist/logging/redaction.js

@@ -21,26 +21,126 @@ const SENSITIVE_SUFFIXES = [
     'accesskey', 'secretkey', 'credential', 'credentials', 'cookie', 'authorization',
     'sessionid', 'connectionstring',
 ];
+/** Basic credentials are base64("user:password"); prose such as "Basic authentication" is not. */
+function looksLikeBasicCredential(value) {
+    const decoded = Buffer.from(value, 'base64').toString('latin1');
+    return decoded.includes(':') && /^[\x20-\x7e]+$/.test(decoded);
+}
+/**
+ * Bearer tokens are opaque, so prose ("use Bearer authorization") is told apart
+ * by shape: a short lowercase or Capitalized word, optionally ending a sentence.
+ * Anything with digits, mixed case, '_', '-', '/', '+', '=' or over 16 letters
+ * is treated as a token: when unsure, redact.
+ */
+function looksLikeProseWord(value) {
+    return /^[A-Z]?[a-z]+\.?$/.test(value) && value.replace(/\.$/, '').length <= 16;
+}
 /** Log-specific credential shapes, applied before the shared advisor patterns. */
 const LOG_PATTERNS = [
     // Anchored on '://' rather than a scheme prefix: `\b[a-z][a-z0-9+.-]*://`
     // backtracks quadratically on inputs such as 'a.a.a.…' (seconds per call at
     // 100 KB). The scheme is left in place; an empty user (://:pw@) is covered.
     [/(:\/\/)[^\s/@:"']*:[^\s/@"']+@/g, '$1<REDACTED:credentials>@'],
-    [/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}/gi, '$1 <REDACTED:credential>'],
+    // In a header any Basic value is a credential; outside one it must decode as user:password.
+    [/\b(authorization:\s*Basic)\s+[^\s"']+/gi, '$1 <REDACTED:credential>'],
+    [/\b(Basic)\s+([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9._~+/=-])/gi,
+        (match, scheme, value) => looksLikeBasicCredential(value) ? `${scheme} <REDACTED:credential>` : match],
+    // Same split for Bearer (#1196): any value in a header; outside one, only prose words are kept.
+    [/\b(authorization:\s*Bearer)\s+[^\s"']+/gi, '$1 <REDACTED:credential>'],
+    [/\b(Bearer)\s+([A-Za-z0-9._~+/=-]{8,})/gi,
+        (match, scheme, value) => looksLikeProseWord(value) ? match : `${scheme} <REDACTED:credential>`],
     [/\bsk-(?:ant-|proj-)?[A-Za-z0-9_-]{16,}/g, '<REDACTED:api_key>'],
     [/\b(set-cookie|cookie):\s*[^\n]+/gi, '$1: <REDACTED:cookie>'],
     // Compound query/assignment keys the shared pattern misses: access_token=, client_secret=
     [/\b([A-Za-z0-9_-]{0,64}?[_-](?:token|secret|password|passwd))=([^\s&"'#]+)/gi, '$1=<REDACTED:credential>'],
 ];
 /**
- * JSON / repr serialized fields ("apiKey":"…", 'password': 123). The key is
- * checked with isSensitiveLogKey() so text and structured values agree.
- * Unterminated strings are redacted to end of line (fail closed).
+ * JSON / repr serialized fields ("apiKey":"…", 'password': 123, token: {…}).
+ * Keys are quoted (JSON, Python repr) or bare (util.inspect); a bare key must
+ * start a word. The key is checked with isSensitiveLogKey() so text and
+ * structured values agree. Values are strings, numbers, or an opening '{' / '['
+ * whose whole object/array is then redacted. JSON embedded in a JSON string
+ * ({\"password\":\"…\"}) uses escaped quotes; its string value runs to the next
+ * escaped quote. Unterminated strings are redacted to end of line and
+ * unbalanced objects/arrays to end of text (fail closed).
  */
-const SERIALIZED_FIELD = /(["'])([A-Za-z0-9_.-]{1,64})\1(\s*:\s*)("(?:[^"\\\n]|\\.)*"?|'(?:[^'\\\n]|\\.)*'?|-?\d[\d.eE+-]*)/g;
+const SERIALIZED_FIELD = /(?:(\\?["'])([A-Za-z0-9_.-]{1,64})\1|(?<![A-Za-z0-9_.$-])([A-Za-z_$][A-Za-z0-9_.$-]{0,63}))(\s*:\s*)("(?:[^"\\\n]|\\.)*"?|'(?:[^'\\\n]|\\.)*'?|\\["']|-?\d[\d.eE+-]*|[{[])/g;
 function redactSerializedFields(text) {
-    return text.replace(SERIALIZED_FIELD, (match, q, key, sep) => isSensitiveLogKey(key) ? `${q}${key}${q}${sep}${q}<REDACTED:sensitive-key>${q}` : match);
+    const pattern = new RegExp(SERIALIZED_FIELD.source, SERIALIZED_FIELD.flags);
+    let out = '';
+    let copied = 0;
+    for (let m = pattern.exec(text); m !== null; m = pattern.exec(text)) {
+        const [match, quote = '', quotedKey, bareKey, sep, value] = m;
+        const key = quotedKey ?? bareKey;
+        const valueStart = m.index + match.length - value.length;
+        if (!isSensitiveLogKey(key)) {
+            // Not sensitive: resume inside the value, so keys nested in an object or
+            // serialized into a string ("body={\"password\":…}") are still checked.
+            pattern.lastIndex = valueStart + 1;
+            continue;
+        }
+        const end = value === '{' || value === '['
+            ? compoundValueEnd(text, valueStart, quote.length === 2)
+            : value.length === 2 && value[0] === '\\'
+                ? escapedStringEnd(text, valueStart + 2, value)
+                : m.index + match.length;
+        out += `${text.slice(copied, m.index)}${quote}${key}${quote}${sep}${quote}<REDACTED:sensitive-key>${quote}`;
+        copied = end;
+        // Resume after the redacted value, which is never scanned again.
+        pattern.lastIndex = end;
+    }
+    return copied === 0 ? text : out + text.slice(copied);
+}
+/**
+ * Index just past the escaped closing quote (\" or \'), or the end of the line
+ * when there is none. Each \X pair decodes to X; a decoded backslash escapes
+ * the next decoded character, so an inner \\\" does not end the string.
+ */
+function escapedStringEnd(text, start, quote) {
+    let innerEscape = false;
+    for (let i = start; i < text.length; i++) {
+        const pair = text[i] === '\\';
+        if (pair)
+            i++;
+        const decoded = text[i];
+        if (decoded === '\n' || decoded === undefined)
+            return i;
+        if (innerEscape)
+            innerEscape = false;
+        else if (pair && decoded === '\\')
+            innerEscape = true;
+        else if (pair && decoded === quote[1])
+            return i + 1;
+    }
+    return text.length;
+}
+/**
+ * Index just past the object/array opened at `start`, skipping quoted strings
+ * (delimited by \" when the object is itself escaped inside a JSON string).
+ * An unbalanced value runs to the end of the text, so it is redacted rather than printed.
+ */
+function compoundValueEnd(text, start, escaped) {
+    let depth = 0;
+    for (let i = start; i < text.length; i++) {
+        const ch = text[i];
+        if (escaped && ch === '\\' && (text[i + 1] === '"' || text[i + 1] === "'")) {
+            i = escapedStringEnd(text, i + 2, text.slice(i, i + 2)) - 1;
+        }
+        else if (!escaped && (ch === '"' || ch === "'")) {
+            // A newline also ends a string: neither JSON nor repr strings contain a raw one.
+            for (i++; i < text.length && text[i] !== ch && text[i] !== '\n'; i++) {
+                if (text[i] === '\\')
+                    i++;
+            }
+        }
+        else if (ch === '{' || ch === '[') {
+            depth++;
+        }
+        else if ((ch === '}' || ch === ']') && --depth === 0) {
+            return i + 1;
+        }
+    }
+    return text.length;
 }
 export function isSensitiveLogKey(key) {
     const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, '');
@@ -55,8 +155,11 @@ export function redactLogText(text, maxLength = DEFAULTS.maxStringLength) {
     if (text.length > MAX_SCAN_LENGTH)
         return `[REDACTED:size-limit ${text.length} chars]`;
     let result = text;
-    for (const [regex, replacement] of LOG_PATTERNS)
-        result = result.replace(regex, replacement);
+    for (const [regex, replacement] of LOG_PATTERNS) {
+        result = typeof replacement === 'string'
+            ? result.replace(regex, replacement)
+            : result.replace(regex, replacement);
+    }
     result = redactSerializedFields(result);
     result = redact(result, 'balanced').text;
     return result.length > maxLength
@@ -71,11 +174,11 @@ export function redactLogValue(value, options = {}) {
 export function formatErrorForLog(error) {
     try {
         if (error instanceof Error) {
-            let text = redactLogText(error.stack || `${error.name}: ${error.message}`);
+            let text = formatErrorText(error);
             let cause = error.cause;
             for (let hops = 0; cause !== undefined && hops < 3; hops++) {
                 text += `\nCaused by: ${cause instanceof Error
-                    ? redactLogText(cause.stack || `${cause.name}: ${cause.message}`)
+                    ? formatErrorText(cause)
                     : JSON.stringify(redactLogValue(cause, { maxDepth: 3 }))}`;
                 cause = cause instanceof Error ? cause.cause : undefined;
             }
@@ -89,6 +192,31 @@ export function formatErrorForLog(error) {
         return '[REDACTED:sanitization-failed]';
     }
 }
+/** Fields kept from an error besides its message and stack: they identify the failure, not its data. */
+const SAFE_ERROR_FIELDS = ['name', 'code', 'status'];
+const MAX_ERROR_FIELD_LENGTH = 200;
+/** name/code/status as a redacted record; strings are redacted, numbers kept, anything else dropped. */
+function safeErrorFields(error) {
+    const fields = {};
+    for (const field of SAFE_ERROR_FIELDS) {
+        const descriptor = Object.getOwnPropertyDescriptor(error, field)
+            ?? (field === 'name' ? { value: error.name } : undefined);
+        const value = descriptor && 'value' in descriptor ? descriptor.value : undefined;
+        if (typeof value === 'string')
+            fields[field] = redactLogText(value, MAX_ERROR_FIELD_LENGTH);
+        else if (typeof value === 'number' && Number.isFinite(value))
+            fields[field] = value;
+    }
+    return fields;
+}
+/** Redacted stack (or name: message) followed by the safe structured fields, unless they add nothing. */
+function formatErrorText(error) {
+    const text = redactLogText(error.stack || `${error.name}: ${error.message}`);
+    const fields = safeErrorFields(error);
+    const keys = Object.keys(fields);
+    const nameOnly = keys.length === 0 || (keys.length === 1 && text.startsWith(`${fields.name}`));
+    return nameOnly ? text : `${text} ${JSON.stringify(fields)}`;
+}
 function sanitize(value, opts, depth, ancestors) {
     try {
         if (value === null || value === undefined || typeof value === 'number' || typeof value === 'boolean')
@@ -136,9 +264,11 @@ function sanitizeError(error, opts, depth, path) {
     };
     if (typeof error.stack === 'string')
         out.stack = redactLogText(error.stack, opts.maxStringLength);
-    const code = error.code;
-    if (typeof code === 'string' || typeof code === 'number')
+    const { code, status } = safeErrorFields(error);
+    if (code !== undefined)
         out.code = code;
+    if (status !== undefined)
+        out.status = status;
     const cause = error.cause;
     if (cause !== undefined)
         out.cause = sanitize(cause, opts, depth + 1, path);