agentic-qe 3.9.36 → 3.10.0

package/dist/learning/real-embeddings.js

@@ -8,21 +8,50 @@
 // Re-export cosineSimilarity from shared utility for backward compatibility
 export { cosineSimilarity } from '../shared/utils/vector-math.js';
 import { toErrorMessage } from '../shared/error-utils.js';
+import { EmbedderEndpointClient, redactEndpoint, } from './embedder-endpoint-client.js';
+import { loadEmbedderIdentity, resetEmbedderIdentityStore, saveEmbedderIdentity, } from './embedder-identity-store.js';
 // Lazy-loaded transformer pipeline
 let pipeline = null;
 let embeddingModel = null;
 let initPromise = null;
 let initializationFailed = false;
 let failureReason = '';
+/**
+ * Set when ADR-097 endpoint path is active. Tracked separately so callers
+ * (tests, observability) can verify endpoint vs in-process unambiguously.
+ */
+let endpointClient = null;
+let endpointIdentity = null;
 export const DEFAULT_EMBEDDING_CONFIG = {
     modelName: 'Xenova/all-MiniLM-L6-v2',
     quantized: true,
     enableCache: true,
     maxCacheSize: 10000,
+    endpoint: process.env.AQE_EMBEDDER_ENDPOINT,
+    endpointToken: process.env.AQE_EMBEDDER_TOKEN,
 };
-// Embedding cache (LRU)
+// Embedding cache (LRU). Keys are namespaced by embedding mode to prevent
+// cross-mode collisions (e.g. flipping endpoint on/off mid-process must not
+// return a vector computed under the other mode). See `cacheKey()`.
 const embeddingCache = new Map();
 const CACHE_TTL_MS = 3600000; // 1 hour
+/**
+ * Build the cache key for a given text under the current embedding mode.
+ * Endpoint mode: `endpoint:<fingerprint>:<text>` — different fingerprints
+ *   (e.g. after a model swap) get separate cache entries.
+ * In-process mode: `inproc:<text>`.
+ *
+ * Until the endpoint probe completes, calls land under `inproc:` — these get
+ * evicted naturally by TTL/LRU after init. Callers should not interleave
+ * endpoint-enabled and endpoint-disabled `computeRealEmbedding` against the
+ * same text within the cache TTL.
+ */
+function cacheKey(text) {
+    if (endpointIdentity) {
+        return `endpoint:${endpointIdentity.fingerprint}:${text}`;
+    }
+    return `inproc:${text}`;
+}
 /**
  * Detect if text is a JSON metrics/internal data string that shouldn't be embedded.
  * These strings have no semantic value for vector search and waste compute.
@@ -57,7 +86,45 @@ async function initializeModel(config = {}) {
     const fullConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
     initPromise = (async () => {
         try {
-            // Dynamic import to avoid issues if transformers not available
+            // ADR-097: Endpoint branch — when an external embedder endpoint is configured,
+            // we MUST NOT import @huggingface/transformers in this process. The cold-load
+            // elimination depends on the dynamic import being inside the else branch only.
+            if (fullConfig.endpoint) {
+                const safeUrl = redactEndpoint(fullConfig.endpoint);
+                console.log(`[RealEmbeddings] Using external embedder endpoint: ${safeUrl}`);
+                endpointClient = new EmbedderEndpointClient({
+                    endpoint: fullConfig.endpoint,
+                    token: fullConfig.endpointToken,
+                    model: fullConfig.modelName,
+                    expectedDim: getEmbeddingDimension(),
+                });
+                // Probe + identity fingerprint. Fails loud on dim mismatch.
+                endpointIdentity = await endpointClient.probe();
+                console.log(`[RealEmbeddings] Endpoint identity: dim=${endpointIdentity.dim} fingerprint=${endpointIdentity.fingerprint}`);
+                // Cross-run drift detection: compare against the last-known identity
+                // for this endpoint (memory.db kv_store, namespace _system). If the
+                // fingerprint changed since last run, log a loud warning — the operator
+                // either intentionally swapped models (fine, but should know) or the
+                // remote silently changed (poisoning risk for the existing index).
+                try {
+                    const previous = loadEmbedderIdentity(endpointIdentity.endpoint);
+                    if (previous && previous.fingerprint !== endpointIdentity.fingerprint) {
+                        console.warn(`[RealEmbeddings] WARNING: endpoint identity changed since last run. ` +
+                            `Previous fingerprint=${previous.fingerprint} dim=${previous.dim}, ` +
+                            `current fingerprint=${endpointIdentity.fingerprint} dim=${endpointIdentity.dim}. ` +
+                            `Vectors written before this change may no longer be comparable.`);
+                    }
+                    saveEmbedderIdentity(endpointIdentity);
+                }
+                catch (persistErr) {
+                    // Persistence failure is non-fatal — we still have the in-memory identity
+                    // for this run. Log so operators see it.
+                    console.warn(`[RealEmbeddings] Could not persist endpoint identity: ${toErrorMessage(persistErr)}`);
+                }
+                embeddingModel = createEndpointPipeline(endpointClient);
+                return;
+            }
+            // In-process branch — only here do we import the transformers package.
             const transformers = await import('@huggingface/transformers');
             pipeline = transformers.pipeline;
             console.log(`[RealEmbeddings] Loading model: ${fullConfig.modelName}`);
@@ -77,6 +144,27 @@ async function initializeModel(config = {}) {
     })();
     return initPromise;
 }
+/**
+ * Build a FeatureExtractionPipeline-shaped wrapper around the endpoint client.
+ * The shape matches `pipeline('feature-extraction', ...)` so the rest of
+ * computeRealEmbedding / computeBatchEmbeddings remains unchanged.
+ *
+ * The endpoint already returns L2-normalized vectors (and we re-normalize on
+ * receive inside the client), so the `pooling` / `normalize` options from
+ * upstream callers are no-ops here — that's the OpenAI contract.
+ */
+function createEndpointPipeline(client) {
+    return async (input) => {
+        const texts = Array.isArray(input) ? input : [input];
+        const vectors = await client.embed(texts);
+        const dim = vectors[0]?.length ?? getEmbeddingDimension();
+        const data = new Float32Array(texts.length * dim);
+        for (let i = 0; i < vectors.length; i++) {
+            data.set(vectors[i], i * dim);
+        }
+        return { data, dims: [texts.length, dim] };
+    };
+}
 /**
  * Compute real embedding using transformer model
  *
@@ -90,20 +178,21 @@ export async function computeRealEmbedding(text, config = {}) {
     if (isNonSemanticText(text)) {
         return new Array(getEmbeddingDimension()).fill(0);
     }
-    // Check cache first
-    if (fullConfig.enableCache) {
-        const cached = embeddingCache.get(text);
-        if (cached && Date.now() - cached.timestamp < CACHE_TTL_MS) {
-            return cached.embedding;
-        }
-    }
-    // Ensure model is initialized
+    // Ensure model is initialized BEFORE checking cache so the cache key
+    // namespace (which depends on endpoint identity) is stable.
     if (!embeddingModel) {
         await initializeModel(config);
     }
     if (!embeddingModel) {
         throw new Error('Embedding model failed to initialize');
     }
+    const key = cacheKey(text);
+    if (fullConfig.enableCache) {
+        const cached = embeddingCache.get(key);
+        if (cached && Date.now() - cached.timestamp < CACHE_TTL_MS) {
+            return cached.embedding;
+        }
+    }
     // Compute embedding
     const startTime = performance.now();
     const output = await embeddingModel(text, { pooling: 'mean', normalize: true });
@@ -121,7 +210,7 @@ export async function computeRealEmbedding(text, config = {}) {
             if (oldestKey)
                 embeddingCache.delete(oldestKey);
         }
-        embeddingCache.set(text, { embedding, timestamp: Date.now() });
+        embeddingCache.set(key, { embedding, timestamp: Date.now() });
     }
     return embedding;
 }
@@ -134,9 +223,16 @@ export async function computeBatchEmbeddings(texts, config = {}) {
     const uncachedTexts = [];
     const uncachedIndices = [];
     const results = new Array(texts.length).fill(null);
+    // Init BEFORE cache lookup so the cache key namespace is stable.
+    if (!embeddingModel) {
+        await initializeModel(config);
+    }
+    if (!embeddingModel) {
+        throw new Error('Embedding model failed to initialize');
+    }
     if (fullConfig.enableCache) {
         for (let i = 0; i < texts.length; i++) {
-            const cached = embeddingCache.get(texts[i]);
+            const cached = embeddingCache.get(cacheKey(texts[i]));
             if (cached && Date.now() - cached.timestamp < CACHE_TTL_MS) {
                 results[i] = cached.embedding;
             }
@@ -154,12 +250,6 @@ export async function computeBatchEmbeddings(texts, config = {}) {
     }
     // Compute uncached embeddings
     if (uncachedTexts.length > 0) {
-        if (!embeddingModel) {
-            await initializeModel(config);
-        }
-        if (!embeddingModel) {
-            throw new Error('Embedding model failed to initialize');
-        }
         const startTime = performance.now();
         // Process in batches of 32 for memory efficiency
         const BATCH_SIZE = 32;
@@ -184,7 +274,7 @@ export async function computeBatchEmbeddings(texts, config = {}) {
                         if (oldestKey)
                             embeddingCache.delete(oldestKey);
                     }
-                    embeddingCache.set(uncachedTexts[textIndex], { embedding, timestamp: Date.now() });
+                    embeddingCache.set(cacheKey(uncachedTexts[textIndex]), { embedding, timestamp: Date.now() });
                 }
             }
         }
@@ -231,5 +321,26 @@ export function resetInitialization() {
     initializationFailed = false;
     failureReason = '';
     embeddingCache.clear();
+    if (endpointClient) {
+        endpointClient.close();
+    }
+    endpointClient = null;
+    endpointIdentity = null;
+    // Drop the identity-store's cached DB connection so tests that flip cwd or
+    // AQE_MEMORY_PATH between runs don't write into a stale handle.
+    resetEmbedderIdentityStore();
+}
+/**
+ * ADR-097: Returns true when the embedding pipeline is the external endpoint path,
+ * false when it's the in-process transformer (or uninitialized).
+ */
+export function isUsingEndpoint() {
+    return endpointClient !== null;
+}
+/**
+ * ADR-097: Returns the endpoint identity fingerprint, or null when not using an endpoint.
+ */
+export function getEndpointIdentity() {
+    return endpointIdentity;
 }
 //# sourceMappingURL=real-embeddings.js.map

package/dist/learning/sqlite-persistence.js

@@ -16,7 +16,7 @@ import { v4 as uuidv4 } from 'uuid';
 import { safeJsonParse } from '../shared/safe-json.js';
 import { toErrorMessage } from '../shared/error-utils.js';
 import { getUnifiedMemory } from '../kernel/unified-memory.js';
-import { computeBatchEmbeddings, getEmbeddingDimension } from './real-embeddings.js';
+import { computeBatchEmbeddings, getEmbeddingDimension, isUsingEndpoint, } from './real-embeddings.js';
 import { recordPatternUsage } from './pattern-usage-recorder.js';
 export const DEFAULT_SQLITE_CONFIG = {
     // LEGACY: Ignored when useUnified=true (the default). All data goes to memory.db
@@ -975,13 +975,26 @@ export class SQLitePatternStore {
             }
             if (textsWithIds.length === 0)
                 continue;
-            // Compute real embeddings via all-MiniLM-L6-v2 (async ONNX inference)
+            // Compute real embeddings via all-MiniLM-L6-v2 (async ONNX inference) or
+            // the ADR-097 endpoint. We MUST NOT silently fall back to hash embeddings
+            // when the endpoint is configured — mixing hash and transformer vectors in
+            // the same HNSW index poisons it (vectors are not comparable). When the
+            // endpoint is active we propagate the failure: a future backfill will
+            // retry; the existing hash-fallback path is preserved only for the
+            // in-process transformer-load failure case.
             let embeddings;
             try {
                 embeddings = await computeBatchEmbeddings(textsWithIds.map(t => t.text));
             }
             catch (e) {
-                // Transformer unavailable — fall back to hash embeddings for remaining patterns
+                if (isUsingEndpoint()) {
+                    // Endpoint failed (or breaker open). Skip this batch entirely — do NOT
+                    // poison the index. Re-throw so caller knows the run was incomplete.
+                    throw new Error(`[SQLitePatternStore] external embedder endpoint failed during backfill; ` +
+                        `aborting to protect HNSW index. Original error: ${toErrorMessage(e)}`);
+                }
+                // In-process transformer path: hash fallback is acceptable because the
+                // entire index in this mode is hash-based anyway.
                 console.warn(`[SQLitePatternStore] Transformer unavailable, falling back to hash embeddings: ${toErrorMessage(e)}`);
                 method = 'hash-fallback';
                 embeddings = textsWithIds.map(t => hashEmbedding(t.text, dimension));