agentic-qe 3.9.35 → 3.10.0

package/dist/learning/embedder-identity-store.js

@@ -0,0 +1,136 @@
+/**
+ * Persistent store for ADR-097 embedder endpoint identity.
+ *
+ * Identity is written into the unified memory.db `kv_store` table under
+ * namespace `_system`, key `embedder_identity:<safe_endpoint_url>`. This lets
+ * us detect cross-run model drift: if the fingerprint changes between AQE
+ * processes, vectors written before the change may not be comparable to vectors
+ * written after, and the operator should know.
+ *
+ * Conforms to the project's unified-persistence rule (no new .db files).
+ *
+ * Persistence failures are non-fatal — callers wrap us in try/catch and log.
+ * We never throw out of these helpers under expected conditions; failure
+ * reasons are surfaced as thrown errors only for genuine I/O problems.
+ */
+import { join } from 'node:path';
+import { existsSync, mkdirSync } from 'node:fs';
+import { createRequire } from 'node:module';
+// `require` is unavailable in pure ESM; createRequire gives us synchronous
+// access to CommonJS modules like better-sqlite3 from the ESM build output.
+const nodeRequire = createRequire(import.meta.url);
+/**
+ * Path to the unified memory.db. Honors AQE_MEMORY_PATH env override (used by
+ * tests and platform installers), otherwise falls back to the project default.
+ */
+function getMemoryDbPath() {
+    const env = process.env.AQE_MEMORY_PATH;
+    if (env && env.length > 0)
+        return env;
+    return join(process.cwd(), '.agentic-qe', 'memory.db');
+}
+const NAMESPACE = '_system';
+const KEY_PREFIX = 'embedder_identity:';
+let cachedDb = null;
+let openFailed = false;
+/**
+ * Lazily open memory.db. We use a dedicated read/write connection (better-sqlite3
+ * supports concurrent connections to the same file under WAL mode, which the
+ * init wizard enables). Failure to open is cached so we don't retry every call.
+ */
+function openDb() {
+    if (cachedDb)
+        return cachedDb;
+    if (openFailed)
+        return null;
+    try {
+        // Dynamic require (via createRequire — `require` is undefined in ESM) so
+        // this module doesn't fail to import when better-sqlite3 is absent in
+        // some test environments. The endpoint feature is optional; identity
+        // persistence is a nice-to-have on top of it.
+        const Database = nodeRequire('better-sqlite3');
+        const path = getMemoryDbPath();
+        const dir = join(path, '..');
+        if (!existsSync(dir)) {
+            mkdirSync(dir, { recursive: true });
+        }
+        const db = new Database(path);
+        db.pragma('journal_mode = WAL');
+        db.pragma('busy_timeout = 5000');
+        db.exec(`
+      CREATE TABLE IF NOT EXISTS kv_store (
+        key TEXT NOT NULL,
+        namespace TEXT NOT NULL,
+        value TEXT NOT NULL,
+        expires_at INTEGER,
+        created_at INTEGER DEFAULT (strftime('%s', 'now') * 1000),
+        PRIMARY KEY (namespace, key)
+      );
+      CREATE INDEX IF NOT EXISTS idx_kv_namespace ON kv_store(namespace);
+    `);
+        cachedDb = db;
+        return db;
+    }
+    catch {
+        openFailed = true;
+        return null;
+    }
+}
+/**
+ * Load the last persisted identity for the given endpoint URL.
+ * Returns null when nothing has been stored yet OR when the kv store is unavailable.
+ */
+export function loadEmbedderIdentity(endpointUrl) {
+    const db = openDb();
+    if (!db)
+        return null;
+    try {
+        const row = db
+            .prepare('SELECT value FROM kv_store WHERE namespace = ? AND key = ?')
+            .get(NAMESPACE, KEY_PREFIX + endpointUrl);
+        if (!row)
+            return null;
+        const parsed = JSON.parse(row.value);
+        if (!parsed.fingerprint || typeof parsed.dim !== 'number')
+            return null;
+        return {
+            fingerprint: parsed.fingerprint,
+            dim: parsed.dim,
+            endpoint: parsed.endpoint,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Persist the current identity. Overwrites any prior entry for the same endpoint.
+ */
+export function saveEmbedderIdentity(identity) {
+    const db = openDb();
+    if (!db)
+        return;
+    const row = {
+        fingerprint: identity.fingerprint,
+        dim: identity.dim,
+        endpoint: identity.endpoint,
+        updatedAt: Date.now(),
+    };
+    db.prepare('INSERT OR REPLACE INTO kv_store (key, namespace, value) VALUES (?, ?, ?)').run(KEY_PREFIX + identity.endpoint, NAMESPACE, JSON.stringify(row));
+}
+/**
+ * For tests: close the cached connection so a fresh DB path can be used.
+ */
+export function resetEmbedderIdentityStore() {
+    if (cachedDb) {
+        try {
+            cachedDb.close();
+        }
+        catch {
+            // ignore
+        }
+    }
+    cachedDb = null;
+    openFailed = false;
+}
+//# sourceMappingURL=embedder-identity-store.js.map

package/dist/learning/qe-reasoning-bank.js

@@ -25,7 +25,7 @@ import { getWitnessChain } from '../audit/witness-chain.js';
 // Import extracted modules
 import { DEFAULT_QE_REASONING_BANK_CONFIG } from './qe-reasoning-bank-types.js';
 import { PRETRAINED_PATTERNS } from './pretrained-patterns.js';
-import { AGENT_CAPABILITIES, calculateAgentScores, applyExplorationPolicy, resolveExplorationRate, deriveTaskType, buildRoutingStateKey, deriveComplexityBucket, } from './agent-routing.js';
+import { AGENT_CAPABILITIES, calculateAgentScores, applyExplorationPolicy, resolveExplorationRate, deriveTaskType, buildRoutingStateKey, } from './agent-routing.js';
 import { resolveTopologyCriticalFromSharedMincut } from './routing-topology-gate.js';
 import { getUnifiedMemory } from '../kernel/unified-memory.js';
 import { resizeEmbedding, hashEmbedding } from './embedding-utils.js';
@@ -417,11 +417,13 @@ export class QEReasoningBank {
             // ADR-095: build the Q-value lookup closure from the task's state_key.
             // The state_key MUST match what `updateHookRouterQValue` writes from
             // post-task — buildRoutingStateKey enforces that.
+            // ADR-096 (issue #499): state_key dropped `complexityBucket` — the
+            // length-keyed bucket fragmented identical tasks across cells so
+            // QWEIGHT_RAMP_VISITS=20 was never reached. 3-dim key now.
             const stateKey = buildRoutingStateKey({
                 taskType: deriveTaskType(request.task),
                 priority: 'normal',
                 domain: detectedDomains[0],
-                complexityBucket: deriveComplexityBucket(request.task),
             });
             const qValueLookup = this.buildQValueLookup(stateKey);
             // 4. Calculate agent scores (now Q-blended when lookup available)
@@ -551,7 +553,7 @@ export class QEReasoningBank {
      * for ARM64 or when transformers module cannot be loaded.
      */
     async embed(text) {
-        // Try ONNX embeddings if enabled
+        // Try ONNX / endpoint embeddings if enabled
         if (this.config.useONNXEmbeddings) {
             try {
                 const { computeRealEmbedding } = await import('./real-embeddings.js');
@@ -563,8 +565,16 @@ export class QEReasoningBank {
                 return embedding;
             }
             catch (error) {
-                // ARM64 ONNX compatibility issue or module not available
-                // Fall through to hash-based embedding silently
+                // ADR-097: if the external embedder endpoint is configured, we MUST NOT
+                // silently fall back to hash embeddings — they would poison the HNSW
+                // index by mixing non-comparable vectors. Propagate the throw so the
+                // caller can decide (retry, skip the write, etc).
+                const { isUsingEndpoint } = await import('./real-embeddings.js');
+                if (isUsingEndpoint()) {
+                    throw error;
+                }
+                // In-process ARM64 ONNX compatibility issue or module not available —
+                // fall through to hash-based embedding (existing pre-ADR-097 behavior).
                 if (process.env.DEBUG) {
                     logger.warn('ONNX embeddings unavailable, using hash fallback', {
                         error: toErrorMessage(error),

package/dist/learning/real-embeddings.d.ts

@@ -6,6 +6,7 @@
  * Model: all-MiniLM-L6-v2 (384-dimensional sentence embeddings)
  */
 export { cosineSimilarity } from '../shared/utils/vector-math.js';
+import { type EndpointIdentity } from './embedder-endpoint-client.js';
 /**
  * Embedding model configuration
  */
@@ -18,6 +19,24 @@ export interface EmbeddingConfig {
     enableCache: boolean;
     /** Maximum cache size */
     maxCacheSize: number;
+    /**
+     * Optional external embedder endpoint (ADR-097).
+     *
+     * When set, AQE routes feature-extraction to an OpenAI-compatible
+     * `/v1/embeddings` service instead of loading `@huggingface/transformers`
+     * in-process. Forms:
+     *   - `http(s)://host:port`  — HTTP transport
+     *   - `unix:/absolute/path`  — HTTP-over-Unix-socket transport
+     *
+     * Defaults to `process.env.AQE_EMBEDDER_ENDPOINT` when unset.
+     * When neither is set, behavior is identical to pre-ADR-097.
+     */
+    endpoint?: string;
+    /**
+     * Bearer token for the embedder endpoint (ADR-097).
+     * Defaults to `process.env.AQE_EMBEDDER_TOKEN`. Env-only — never in config files.
+     */
+    endpointToken?: string;
 }
 export declare const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig;
 /**
@@ -55,4 +74,13 @@ export declare function getEmbeddingDimension(): number;
  * Reset initialization state (for testing)
  */
 export declare function resetInitialization(): void;
+/**
+ * ADR-097: Returns true when the embedding pipeline is the external endpoint path,
+ * false when it's the in-process transformer (or uninitialized).
+ */
+export declare function isUsingEndpoint(): boolean;
+/**
+ * ADR-097: Returns the endpoint identity fingerprint, or null when not using an endpoint.
+ */
+export declare function getEndpointIdentity(): EndpointIdentity | null;
 //# sourceMappingURL=real-embeddings.d.ts.map

package/dist/learning/real-embeddings.js

@@ -8,21 +8,50 @@
 // Re-export cosineSimilarity from shared utility for backward compatibility
 export { cosineSimilarity } from '../shared/utils/vector-math.js';
 import { toErrorMessage } from '../shared/error-utils.js';
+import { EmbedderEndpointClient, redactEndpoint, } from './embedder-endpoint-client.js';
+import { loadEmbedderIdentity, resetEmbedderIdentityStore, saveEmbedderIdentity, } from './embedder-identity-store.js';
 // Lazy-loaded transformer pipeline
 let pipeline = null;
 let embeddingModel = null;
 let initPromise = null;
 let initializationFailed = false;
 let failureReason = '';
+/**
+ * Set when ADR-097 endpoint path is active. Tracked separately so callers
+ * (tests, observability) can verify endpoint vs in-process unambiguously.
+ */
+let endpointClient = null;
+let endpointIdentity = null;
 export const DEFAULT_EMBEDDING_CONFIG = {
     modelName: 'Xenova/all-MiniLM-L6-v2',
     quantized: true,
     enableCache: true,
     maxCacheSize: 10000,
+    endpoint: process.env.AQE_EMBEDDER_ENDPOINT,
+    endpointToken: process.env.AQE_EMBEDDER_TOKEN,
 };
-// Embedding cache (LRU)
+// Embedding cache (LRU). Keys are namespaced by embedding mode to prevent
+// cross-mode collisions (e.g. flipping endpoint on/off mid-process must not
+// return a vector computed under the other mode). See `cacheKey()`.
 const embeddingCache = new Map();
 const CACHE_TTL_MS = 3600000; // 1 hour
+/**
+ * Build the cache key for a given text under the current embedding mode.
+ * Endpoint mode: `endpoint:<fingerprint>:<text>` — different fingerprints
+ *   (e.g. after a model swap) get separate cache entries.
+ * In-process mode: `inproc:<text>`.
+ *
+ * Until the endpoint probe completes, calls land under `inproc:` — these get
+ * evicted naturally by TTL/LRU after init. Callers should not interleave
+ * endpoint-enabled and endpoint-disabled `computeRealEmbedding` against the
+ * same text within the cache TTL.
+ */
+function cacheKey(text) {
+    if (endpointIdentity) {
+        return `endpoint:${endpointIdentity.fingerprint}:${text}`;
+    }
+    return `inproc:${text}`;
+}
 /**
  * Detect if text is a JSON metrics/internal data string that shouldn't be embedded.
  * These strings have no semantic value for vector search and waste compute.
@@ -57,7 +86,45 @@ async function initializeModel(config = {}) {
     const fullConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
     initPromise = (async () => {
         try {
-            // Dynamic import to avoid issues if transformers not available
+            // ADR-097: Endpoint branch — when an external embedder endpoint is configured,
+            // we MUST NOT import @huggingface/transformers in this process. The cold-load
+            // elimination depends on the dynamic import being inside the else branch only.
+            if (fullConfig.endpoint) {
+                const safeUrl = redactEndpoint(fullConfig.endpoint);
+                console.log(`[RealEmbeddings] Using external embedder endpoint: ${safeUrl}`);
+                endpointClient = new EmbedderEndpointClient({
+                    endpoint: fullConfig.endpoint,
+                    token: fullConfig.endpointToken,
+                    model: fullConfig.modelName,
+                    expectedDim: getEmbeddingDimension(),
+                });
+                // Probe + identity fingerprint. Fails loud on dim mismatch.
+                endpointIdentity = await endpointClient.probe();
+                console.log(`[RealEmbeddings] Endpoint identity: dim=${endpointIdentity.dim} fingerprint=${endpointIdentity.fingerprint}`);
+                // Cross-run drift detection: compare against the last-known identity
+                // for this endpoint (memory.db kv_store, namespace _system). If the
+                // fingerprint changed since last run, log a loud warning — the operator
+                // either intentionally swapped models (fine, but should know) or the
+                // remote silently changed (poisoning risk for the existing index).
+                try {
+                    const previous = loadEmbedderIdentity(endpointIdentity.endpoint);
+                    if (previous && previous.fingerprint !== endpointIdentity.fingerprint) {
+                        console.warn(`[RealEmbeddings] WARNING: endpoint identity changed since last run. ` +
+                            `Previous fingerprint=${previous.fingerprint} dim=${previous.dim}, ` +
+                            `current fingerprint=${endpointIdentity.fingerprint} dim=${endpointIdentity.dim}. ` +
+                            `Vectors written before this change may no longer be comparable.`);
+                    }
+                    saveEmbedderIdentity(endpointIdentity);
+                }
+                catch (persistErr) {
+                    // Persistence failure is non-fatal — we still have the in-memory identity
+                    // for this run. Log so operators see it.
+                    console.warn(`[RealEmbeddings] Could not persist endpoint identity: ${toErrorMessage(persistErr)}`);
+                }
+                embeddingModel = createEndpointPipeline(endpointClient);
+                return;
+            }
+            // In-process branch — only here do we import the transformers package.
             const transformers = await import('@huggingface/transformers');
             pipeline = transformers.pipeline;
             console.log(`[RealEmbeddings] Loading model: ${fullConfig.modelName}`);
@@ -77,6 +144,27 @@ async function initializeModel(config = {}) {
     })();
     return initPromise;
 }
+/**
+ * Build a FeatureExtractionPipeline-shaped wrapper around the endpoint client.
+ * The shape matches `pipeline('feature-extraction', ...)` so the rest of
+ * computeRealEmbedding / computeBatchEmbeddings remains unchanged.
+ *
+ * The endpoint already returns L2-normalized vectors (and we re-normalize on
+ * receive inside the client), so the `pooling` / `normalize` options from
+ * upstream callers are no-ops here — that's the OpenAI contract.
+ */
+function createEndpointPipeline(client) {
+    return async (input) => {
+        const texts = Array.isArray(input) ? input : [input];
+        const vectors = await client.embed(texts);
+        const dim = vectors[0]?.length ?? getEmbeddingDimension();
+        const data = new Float32Array(texts.length * dim);
+        for (let i = 0; i < vectors.length; i++) {
+            data.set(vectors[i], i * dim);
+        }
+        return { data, dims: [texts.length, dim] };
+    };
+}
 /**
  * Compute real embedding using transformer model
  *
@@ -90,20 +178,21 @@ export async function computeRealEmbedding(text, config = {}) {
     if (isNonSemanticText(text)) {
         return new Array(getEmbeddingDimension()).fill(0);
     }
-    // Check cache first
-    if (fullConfig.enableCache) {
-        const cached = embeddingCache.get(text);
-        if (cached && Date.now() - cached.timestamp < CACHE_TTL_MS) {
-            return cached.embedding;
-        }
-    }
-    // Ensure model is initialized
+    // Ensure model is initialized BEFORE checking cache so the cache key
+    // namespace (which depends on endpoint identity) is stable.
     if (!embeddingModel) {
         await initializeModel(config);
     }
     if (!embeddingModel) {
         throw new Error('Embedding model failed to initialize');
     }
+    const key = cacheKey(text);
+    if (fullConfig.enableCache) {
+        const cached = embeddingCache.get(key);
+        if (cached && Date.now() - cached.timestamp < CACHE_TTL_MS) {
+            return cached.embedding;
+        }
+    }
     // Compute embedding
     const startTime = performance.now();
     const output = await embeddingModel(text, { pooling: 'mean', normalize: true });
@@ -121,7 +210,7 @@ export async function computeRealEmbedding(text, config = {}) {
             if (oldestKey)
                 embeddingCache.delete(oldestKey);
         }
-        embeddingCache.set(text, { embedding, timestamp: Date.now() });
+        embeddingCache.set(key, { embedding, timestamp: Date.now() });
     }
     return embedding;
 }
@@ -134,9 +223,16 @@ export async function computeBatchEmbeddings(texts, config = {}) {
     const uncachedTexts = [];
     const uncachedIndices = [];
     const results = new Array(texts.length).fill(null);
+    // Init BEFORE cache lookup so the cache key namespace is stable.
+    if (!embeddingModel) {
+        await initializeModel(config);
+    }
+    if (!embeddingModel) {
+        throw new Error('Embedding model failed to initialize');
+    }
     if (fullConfig.enableCache) {
         for (let i = 0; i < texts.length; i++) {
-            const cached = embeddingCache.get(texts[i]);
+            const cached = embeddingCache.get(cacheKey(texts[i]));
             if (cached && Date.now() - cached.timestamp < CACHE_TTL_MS) {
                 results[i] = cached.embedding;
             }
@@ -154,12 +250,6 @@ export async function computeBatchEmbeddings(texts, config = {}) {
     }
     // Compute uncached embeddings
     if (uncachedTexts.length > 0) {
-        if (!embeddingModel) {
-            await initializeModel(config);
-        }
-        if (!embeddingModel) {
-            throw new Error('Embedding model failed to initialize');
-        }
         const startTime = performance.now();
         // Process in batches of 32 for memory efficiency
         const BATCH_SIZE = 32;
@@ -184,7 +274,7 @@ export async function computeBatchEmbeddings(texts, config = {}) {
                         if (oldestKey)
                             embeddingCache.delete(oldestKey);
                     }
-                    embeddingCache.set(uncachedTexts[textIndex], { embedding, timestamp: Date.now() });
+                    embeddingCache.set(cacheKey(uncachedTexts[textIndex]), { embedding, timestamp: Date.now() });
                 }
             }
         }
@@ -231,5 +321,26 @@ export function resetInitialization() {
     initializationFailed = false;
     failureReason = '';
     embeddingCache.clear();
+    if (endpointClient) {
+        endpointClient.close();
+    }
+    endpointClient = null;
+    endpointIdentity = null;
+    // Drop the identity-store's cached DB connection so tests that flip cwd or
+    // AQE_MEMORY_PATH between runs don't write into a stale handle.
+    resetEmbedderIdentityStore();
+}
+/**
+ * ADR-097: Returns true when the embedding pipeline is the external endpoint path,
+ * false when it's the in-process transformer (or uninitialized).
+ */
+export function isUsingEndpoint() {
+    return endpointClient !== null;
+}
+/**
+ * ADR-097: Returns the endpoint identity fingerprint, or null when not using an endpoint.
+ */
+export function getEndpointIdentity() {
+    return endpointIdentity;
 }
 //# sourceMappingURL=real-embeddings.js.map

package/dist/learning/sqlite-persistence.js

@@ -16,7 +16,7 @@ import { v4 as uuidv4 } from 'uuid';
 import { safeJsonParse } from '../shared/safe-json.js';
 import { toErrorMessage } from '../shared/error-utils.js';
 import { getUnifiedMemory } from '../kernel/unified-memory.js';
-import { computeBatchEmbeddings, getEmbeddingDimension } from './real-embeddings.js';
+import { computeBatchEmbeddings, getEmbeddingDimension, isUsingEndpoint, } from './real-embeddings.js';
 import { recordPatternUsage } from './pattern-usage-recorder.js';
 export const DEFAULT_SQLITE_CONFIG = {
     // LEGACY: Ignored when useUnified=true (the default). All data goes to memory.db
@@ -975,13 +975,26 @@ export class SQLitePatternStore {
             }
             if (textsWithIds.length === 0)
                 continue;
-            // Compute real embeddings via all-MiniLM-L6-v2 (async ONNX inference)
+            // Compute real embeddings via all-MiniLM-L6-v2 (async ONNX inference) or
+            // the ADR-097 endpoint. We MUST NOT silently fall back to hash embeddings
+            // when the endpoint is configured — mixing hash and transformer vectors in
+            // the same HNSW index poisons it (vectors are not comparable). When the
+            // endpoint is active we propagate the failure: a future backfill will
+            // retry; the existing hash-fallback path is preserved only for the
+            // in-process transformer-load failure case.
             let embeddings;
             try {
                 embeddings = await computeBatchEmbeddings(textsWithIds.map(t => t.text));
             }
             catch (e) {
-                // Transformer unavailable — fall back to hash embeddings for remaining patterns
+                if (isUsingEndpoint()) {
+                    // Endpoint failed (or breaker open). Skip this batch entirely — do NOT
+                    // poison the index. Re-throw so caller knows the run was incomplete.
+                    throw new Error(`[SQLitePatternStore] external embedder endpoint failed during backfill; ` +
+                        `aborting to protect HNSW index. Original error: ${toErrorMessage(e)}`);
+                }
+                // In-process transformer path: hash fallback is acceptable because the
+                // entire index in this mode is hash-based anyway.
                 console.warn(`[SQLitePatternStore] Transformer unavailable, falling back to hash embeddings: ${toErrorMessage(e)}`);
                 method = 'hash-fallback';
                 embeddings = textsWithIds.map(t => hashEmbedding(t.text, dimension));