agentic-qe 3.9.32 → 3.9.34

package/dist/integrations/ruvector/shared-rvf-adapter.js

@@ -12,6 +12,34 @@
  */
 let sharedAdapter = null;
 let initAttempted = false;
+// ---------------------------------------------------------------------------
+// Test seam: allow tests to install a fake adapter without going through the
+// native binding loader. NOT part of the public API — used by tests only.
+// ---------------------------------------------------------------------------
+/** @internal test-only seam */
+export function __setSharedRvfAdapterForTests(adapter) {
+    sharedAdapter = adapter;
+    initAttempted = adapter !== null;
+}
+// ----------------------------------------------------------------------------
+// Compaction tunables
+//
+// `compact()` is best-effort and idempotent on a clean file, so the thresholds
+// are deliberately conservative. They can be tightened via env vars without a
+// rebuild — useful in field debugging when an installation is already large.
+// ----------------------------------------------------------------------------
+/** File size above which we run a boot-time compact (default: 256 MB). */
+const SIZE_GUARD_BYTES = (() => {
+    const env = process.env.AQE_RVF_SIZE_GUARD_BYTES;
+    const parsed = env ? Number(env) : NaN;
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : 256 * 1024 * 1024;
+})();
+/** Dead-space ratio above which we run a compact (default: 0.30 = 30%). */
+const DEAD_RATIO_THRESHOLD = (() => {
+    const env = process.env.AQE_RVF_DEAD_RATIO_THRESHOLD;
+    const parsed = env ? Number(env) : NaN;
+    return Number.isFinite(parsed) && parsed > 0 && parsed <= 1 ? parsed : 0.30;
+})();
 /**
  * Get or create the shared RvfNativeAdapter singleton for patterns.rvf.
  *
@@ -48,6 +76,10 @@ export function getSharedRvfAdapter(dataDir = '.agentic-qe', dimensions = 384) {
         //     FsyncFailed regardless. The try-ladder degrades to whichever
         //     path the OS actually permits.
         sharedAdapter = openOrCreateRvf(openRvfStore, createRvfStore, rvfPath, dimensions);
+        // Boot-time size guard: an oversized or fragmented patterns.rvf gets a
+        // best-effort compaction on first open. Synchronous — runs once per
+        // process. Errors logged, never thrown (compaction must not block boot).
+        runBootCompactGuard(sharedAdapter, rvfPath);
         return sharedAdapter;
     }
     catch (error) {
@@ -149,6 +181,111 @@ function openOrCreateRvf(openFn, createFn, rvfPath, dimensions) {
         }
     }
 }
+/**
+ * Pure decision function: given an adapter status, decide whether compaction
+ * should run and which threshold triggered it. Exported so tests can verify
+ * the decision logic without exercising the singleton or native binding.
+ */
+export function decideCompactionFromStatus(status, opts) {
+    if (opts?.force)
+        return { shouldCompact: true, trigger: 'force' };
+    const sizeThreshold = opts?.sizeGuardBytes ?? SIZE_GUARD_BYTES;
+    const deadThreshold = opts?.deadRatioThreshold ?? DEAD_RATIO_THRESHOLD;
+    if (status.fileSizeBytes >= sizeThreshold) {
+        return { shouldCompact: true, trigger: 'size-guard' };
+    }
+    if ((status.deadSpaceRatio ?? 0) >= deadThreshold) {
+        return { shouldCompact: true, trigger: 'dead-ratio' };
+    }
+    return { shouldCompact: false, trigger: 'none' };
+}
+/**
+ * Run `compact()` against the shared patterns.rvf adapter when dead-space or
+ * file-size thresholds are exceeded. Best-effort — returns the reclaim stats
+ * if compaction ran, or `null` if it was skipped or failed.
+ *
+ * Safe to call from steady-state code (e.g. after a dream cycle). The native
+ * binding's compact() is documented as exclusive against writers, so callers
+ * should run this in idle windows when possible.
+ *
+ * Thresholds can be overridden via:
+ *   AQE_RVF_SIZE_GUARD_BYTES        (default 256 MB)
+ *   AQE_RVF_DEAD_RATIO_THRESHOLD    (default 0.30)
+ */
+export function compactSharedRvfAdapter(opts) {
+    if (!sharedAdapter)
+        return null;
+    let status;
+    try {
+        status = sharedAdapter.status();
+    }
+    catch {
+        return null;
+    }
+    const decision = decideCompactionFromStatus(status, opts);
+    if (!decision.shouldCompact)
+        return null;
+    const before = status.fileSizeBytes;
+    let result;
+    try {
+        result = sharedAdapter.compact();
+    }
+    catch {
+        // The thin native wrapper already catches and returns null on native
+        // errors, but be defensive in case a caller installs a non-wrapped
+        // adapter (e.g. tests, custom adapters).
+        return null;
+    }
+    if (!result)
+        return null;
+    // Log only when there's something interesting to report — keeps idle logs quiet.
+    if (result.bytesReclaimed > 0 || result.segmentsCompacted > 0) {
+        console.log(`[RVF] compacted patterns.rvf: reclaimed ${formatBytes(result.bytesReclaimed)} ` +
+            `(${result.segmentsCompacted} segments, fileSize ${formatBytes(before)} → ` +
+            `${formatBytes(Math.max(0, before - result.bytesReclaimed))}, ` +
+            `trigger: ${decision.trigger})`);
+    }
+    return result;
+}
+/**
+ * One-shot at boot. Runs a best-effort `compact()` against the freshly-opened
+ * adapter when fileSize or deadSpaceRatio exceed configured thresholds. Lives
+ * here (not at module load) so a never-opened adapter doesn't trigger native
+ * binding init. Exported for tests; production callers go via
+ * `getSharedRvfAdapter()` which invokes it internally.
+ */
+export function runBootCompactGuard(adapter, rvfPath) {
+    try {
+        const status = adapter.status();
+        const decision = decideCompactionFromStatus(status);
+        if (!decision.shouldCompact)
+            return null;
+        const result = adapter.compact();
+        if (result && (result.bytesReclaimed > 0 || result.segmentsCompacted > 0)) {
+            console.log(`[RVF] boot-time compact (${rvfPath}): reclaimed ${formatBytes(result.bytesReclaimed)} ` +
+                `from ${result.segmentsCompacted} segments (trigger: ${decision.trigger})`);
+        }
+        return result;
+    }
+    catch (error) {
+        if (process.env.DEBUG) {
+            console.debug('[RVF] boot-time compact guard skipped:', error instanceof Error ? error.message : error);
+        }
+        return null;
+    }
+}
+function formatBytes(n) {
+    if (!Number.isFinite(n) || n < 0)
+        return `${n}B`;
+    const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+    let i = 0;
+    let v = n;
+    while (v >= 1024 && i < units.length - 1) {
+        v /= 1024;
+        i++;
+    }
+    return `${v.toFixed(i === 0 ? 0 : 1)}${units[i]}`;
+}
 /** Close the shared adapter and reset the singleton. */
 export function resetSharedRvfAdapter() {
     if (sharedAdapter) {

package/dist/kernel/hybrid-backend.d.ts

@@ -71,9 +71,20 @@ export declare class HybridMemoryBackend implements MemoryBackend {
     private initialized;
     constructor(config?: Partial<HybridBackendConfig>);
     /**
-     * Initialize the unified backend
-     */
-    initialize(): Promise<void>;
+     * Initialize the unified backend.
+     *
+     * `options.signal` bounds the init: if a caller-side timeout aborts the
+     * signal, `signal.throwIfAborted()` checks between awaited steps cause
+     * initialize() to reject promptly instead of leaking the init promise
+     * past the timeout (issue #495 — sibling to #478's fix for
+     * ReasoningBank.initialize). The previous Promise.race-based timeout in
+     * `createHybridBackendWithTimeout()` could not stop the underlying work,
+     * so a stuck unified-memory init would keep running for 14+ min while
+     * the hook subprocess held patterns.rvf open.
+     */
+    initialize(options?: {
+        signal?: AbortSignal;
+    }): Promise<void>;
     /**
      * Dispose of the backend
      */

package/dist/kernel/hybrid-backend.js

@@ -50,12 +50,23 @@ export class HybridMemoryBackend {
         };
     }
     /**
-     * Initialize the unified backend
+     * Initialize the unified backend.
+     *
+     * `options.signal` bounds the init: if a caller-side timeout aborts the
+     * signal, `signal.throwIfAborted()` checks between awaited steps cause
+     * initialize() to reject promptly instead of leaking the init promise
+     * past the timeout (issue #495 — sibling to #478's fix for
+     * ReasoningBank.initialize). The previous Promise.race-based timeout in
+     * `createHybridBackendWithTimeout()` could not stop the underlying work,
+     * so a stuck unified-memory init would keep running for 14+ min while
+     * the hook subprocess held patterns.rvf open.
      */
-    async initialize() {
+    async initialize(options) {
         if (this.initialized) {
             return;
         }
+        const signal = options?.signal;
+        signal?.throwIfAborted();
         // Get unified memory manager with our config
         const unifiedConfig = {
             dbPath: this.config.sqlite.path ?? DEFAULT_UNIFIED_MEMORY_CONFIG.dbPath,
@@ -63,7 +74,9 @@ export class HybridMemoryBackend {
             busyTimeout: this.config.sqlite.busyTimeout ?? MEMORY_CONSTANTS.BUSY_TIMEOUT_MS,
         };
         this.unifiedMemory = getUnifiedMemory(unifiedConfig);
-        await this.unifiedMemory.initialize();
+        signal?.throwIfAborted();
+        await this.unifiedMemory.initialize({ signal });
+        signal?.throwIfAborted();
         // Start cleanup interval (unref so it doesn't block process exit)
         this.cleanupInterval = setInterval(() => this.cleanup(), this.config.cleanupInterval);
         if (this.cleanupInterval.unref) {

package/dist/kernel/unified-memory.d.ts

@@ -92,7 +92,9 @@ export declare class UnifiedMemoryManager {
     static getInstance(config?: Partial<UnifiedMemoryConfig>): UnifiedMemoryManager;
     static getInstanceAsync(config?: Partial<UnifiedMemoryConfig>): Promise<UnifiedMemoryManager>;
     static resetInstance(): void;
-    initialize(): Promise<void>;
+    initialize(options?: {
+        signal?: AbortSignal;
+    }): Promise<void>;
     private _doInitialize;
     /**
      * Find recent backup files for the memory database, sorted newest first.

package/dist/kernel/unified-memory.js

@@ -218,17 +218,18 @@ export class UnifiedMemoryManager {
         // opening the production database instead of the test database.
         clearProjectRootCache();
     }
-    async initialize() {
+    async initialize(options) {
         if (this.initialized)
             return;
         if (!this.initPromise) {
-            this.initPromise = this._doInitialize();
+            this.initPromise = this._doInitialize(options?.signal);
         }
         return this.initPromise;
     }
-    async _doInitialize() {
+    async _doInitialize(signal) {
         if (this.initialized)
             return;
+        signal?.throwIfAborted();
         try {
             // SAFETY: Detect test processes trying to open the production DB.
             // If AQE_PROJECT_ROOT is set to a temp dir (vitest isolation) but the
@@ -282,6 +283,9 @@ export class UnifiedMemoryManager {
                     dbSizeBefore = newest.size;
                 }
             }
+            // Bail before opening the DB if the caller already gave up — saves a
+            // file handle and avoids racing the busy-timeout under contention.
+            signal?.throwIfAborted();
             this.db = openSafeDatabase(this.config.dbPath, {
                 walMode: this.config.walMode,
                 busyTimeout: this.config.busyTimeout,
@@ -289,12 +293,26 @@ export class UnifiedMemoryManager {
             this.db.pragma(`mmap_size = ${this.config.mmapSize}`);
             this.db.pragma(`cache_size = ${this.config.cacheSize}`);
             this.db.pragma('foreign_keys = ON');
+            // Migrations are the longest-running step under contention (#495's
+            // primary stall point). If abort fires after the DB opens, drop the
+            // handle before throwing so we don't leak the WAL/SHM in the background.
+            if (signal?.aborted) {
+                try {
+                    this.db.close();
+                }
+                catch { /* best-effort */ }
+                this.db = null;
+                signal.throwIfAborted();
+            }
             await this.runMigrations();
+            signal?.throwIfAborted();
             // DATA LOSS PREVENTION: After migration, if the DB existed before and was
             // large (>1MB = has real data), but now qe_patterns is empty, something
             // went wrong. Refuse to proceed with an empty DB over real data.
             if (dbExistedBefore && dbSizeBefore > 1_000_000) {
                 try {
+                    // Non-null: post-runMigrations, this.db is guaranteed open (the
+                    // abort-cleanup branch above already throws before reaching here).
                     const row = this.db.prepare('SELECT COUNT(*) as cnt FROM qe_patterns').get();
                     if (row && row.cnt === 0) {
                         const currentSize = fs.statSync(this.config.dbPath).size;
@@ -318,6 +336,13 @@ export class UnifiedMemoryManager {
         }
         catch (error) {
             this.initPromise = null;
+            // Preserve AbortError so callers can distinguish cancellation from a
+            // genuine init failure. Wrapping it in a generic "Failed to initialize"
+            // message would hide the signal cause (#495).
+            if (error instanceof Error &&
+                (error.name === 'AbortError' || (signal?.aborted ?? false))) {
+                throw error;
+            }
             throw new Error(`Failed to initialize UnifiedMemoryManager: ${toErrorMessage(error)}`);
         }
     }

package/dist/learning/dream/dream-scheduler.d.ts

@@ -258,6 +258,24 @@ export declare class DreamScheduler {
      * Auto-apply insights that meet the confidence threshold.
      */
     private autoApplyInsights;
+    /**
+     * Reclaim dead space in patterns.rvf and brain.rvf after a dream cycle.
+     *
+     * Background: RVF is append-only — every ingest/delete writes a new segment.
+     * Without periodic compaction these files grow monotonically (field reports
+     * have seen 59 GB on a fresh clone). The shared adapter exposes a helper
+     * that runs `compact()` only when configured thresholds are exceeded, so
+     * idle cycles are essentially free.
+     *
+     * Two files are compacted:
+     *   • patterns.rvf — primary HNSW vector store (every storePattern() append)
+     *   • brain.rvf — dual-write target (one append per storePattern() via
+     *     RvfDualWriter.writePattern)
+     *
+     * Best-effort throughout: any error is logged and swallowed. The dream
+     * cycle's success is not contingent on compaction completing.
+     */
+    private maybeCompactPatternsRvf;
     /**
      * Collect a metrics snapshot and, if enough have accumulated, run a
      * meta-learning analysis cycle to detect token-waste, quality plateaus,

package/dist/learning/dream/dream-scheduler.js

@@ -357,6 +357,17 @@ export class DreamScheduler {
             }
             // Publish dream completed event
             await this.publishDreamCompletedEvent(result);
+            // Reclaim dead space in patterns.rvf after each dream cycle. Best-effort
+            // and threshold-gated (only runs when deadSpaceRatio or fileSize cross
+            // configured thresholds), so idle cycles are nearly free. Without this
+            // call patterns.rvf grows monotonically — see the field reports of
+            // 59 GB regrowth on a fresh clone.
+            //
+            // AWAITED on purpose: native compact() takes an exclusive lock against
+            // the RVF file. Running concurrently with the meta-learning analysis
+            // (which reads from RVF) would risk lock contention. The threshold
+            // gate keeps the wall-clock cost near zero on healthy installations.
+            await this.maybeCompactPatternsRvf();
             // ADR-062: Run meta-learning analysis after dream cycle
             if (process.env.AQE_META_LEARNING_ENABLED === 'true') {
                 try {
@@ -411,6 +422,63 @@ export class DreamScheduler {
             }
         }
     }
+    /**
+     * Reclaim dead space in patterns.rvf and brain.rvf after a dream cycle.
+     *
+     * Background: RVF is append-only — every ingest/delete writes a new segment.
+     * Without periodic compaction these files grow monotonically (field reports
+     * have seen 59 GB on a fresh clone). The shared adapter exposes a helper
+     * that runs `compact()` only when configured thresholds are exceeded, so
+     * idle cycles are essentially free.
+     *
+     * Two files are compacted:
+     *   • patterns.rvf — primary HNSW vector store (every storePattern() append)
+     *   • brain.rvf — dual-write target (one append per storePattern() via
+     *     RvfDualWriter.writePattern)
+     *
+     * Best-effort throughout: any error is logged and swallowed. The dream
+     * cycle's success is not contingent on compaction completing.
+     */
+    async maybeCompactPatternsRvf() {
+        // patterns.rvf — threshold-gated compaction via the shared singleton.
+        try {
+            const mod = await import('../../integrations/ruvector/shared-rvf-adapter.js');
+            const result = mod.compactSharedRvfAdapter();
+            if (result && result.bytesReclaimed > 0) {
+                logger.info('patterns.rvf compacted', {
+                    bytesReclaimed: result.bytesReclaimed,
+                    segmentsCompacted: result.segmentsCompacted,
+                });
+            }
+        }
+        catch (err) {
+            logger.warn('patterns.rvf compaction skipped (non-critical)', {
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+        // brain.rvf — separate file, separate writer (RvfDualWriter). The
+        // dual-writer does not gate on thresholds (it has no status surface that
+        // exposes deadSpaceRatio), so we call compact() unconditionally. Native
+        // compact() is a no-op on a clean file, so the cost is bounded.
+        try {
+            const mod = await import('../../integrations/ruvector/shared-rvf-dual-writer.js');
+            const writer = mod.getSharedRvfDualWriterSync();
+            if (writer) {
+                const result = writer.compact();
+                if (result && result.bytesReclaimed > 0) {
+                    logger.info('brain.rvf compacted', {
+                        bytesReclaimed: result.bytesReclaimed,
+                        segmentsCompacted: result.segmentsCompacted,
+                    });
+                }
+            }
+        }
+        catch (err) {
+            logger.warn('brain.rvf compaction skipped (non-critical)', {
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
     // ==========================================================================
     // Private: Meta-Learning (ADR-062)
     // ==========================================================================

package/dist/learning/qe-reasoning-bank.js

@@ -179,7 +179,19 @@ export class QEReasoningBank {
             const adapter = this.patternStore.getAdapter?.();
             if (adapter && (adapter.status()?.totalVectors ?? 0) === 0) {
                 try {
-                    const embeddings = this.getSqliteStore().getAllEmbeddings();
+                    // Cap the backfill to the most-recently-used N embeddings. Prevents
+                    // a truncated patterns.rvf (e.g. `: > patterns.rvf` for disk
+                    // recovery) from triggering an unbounded re-ingest of every
+                    // historical embedding — the path that reproduces "regrew to 59 GB"
+                    // in the field.
+                    //
+                    // Override via AQE_RVF_BACKFILL_LIMIT (default: 1000).
+                    const limit = (() => {
+                        const env = process.env.AQE_RVF_BACKFILL_LIMIT;
+                        const parsed = env ? Number(env) : NaN;
+                        return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : 1000;
+                    })();
+                    const embeddings = this.getSqliteStore().getRecentEmbeddings(limit);
                     const vectors = embeddings
                         .filter(({ embedding }) => embedding && embedding.length > 0)
                         .map(({ patternId, embedding }) => ({
@@ -190,7 +202,12 @@ export class QEReasoningBank {
                     }));
                     if (vectors.length > 0) {
                         const { accepted, rejected } = adapter.ingest(vectors);
-                        logger.info('Backfilled RVF from SQLite', { count: vectors.length, accepted });
+                        logger.info('Backfilled RVF from SQLite', {
+                            requested: vectors.length,
+                            accepted,
+                            cap: limit,
+                            truncated: vectors.length >= limit,
+                        });
                         if (rejected > 0) {
                             logger.warn('RVF backfill partially rejected', { accepted, rejected });
                         }

package/dist/learning/real-embeddings.js

@@ -58,7 +58,7 @@ async function initializeModel(config = {}) {
     initPromise = (async () => {
         try {
             // Dynamic import to avoid issues if transformers not available
-            const transformers = await import('@xenova/transformers');
+            const transformers = await import('@huggingface/transformers');
             pipeline = transformers.pipeline;
             console.log(`[RealEmbeddings] Loading model: ${fullConfig.modelName}`);
             const startTime = performance.now();

package/dist/learning/sqlite-persistence.d.ts

@@ -114,6 +114,17 @@ export declare class SQLitePatternStore {
         patternId: string;
         embedding: number[];
     }>;
+    /**
+     * Get up to `limit` embeddings, most-recently-used first.
+     *
+     * Used by the RVF backfill path so an operator-truncated patterns.rvf does
+     * not trigger an unbounded re-ingest of every historical embedding (which
+     * is what reproduces the "regrew to 59 GB" pathology after `: > patterns.rvf`).
+     */
+    getRecentEmbeddings(limit: number): Array<{
+        patternId: string;
+        embedding: number[];
+    }>;
     /**
      * Record pattern usage.
      *

package/dist/learning/sqlite-persistence.js

@@ -361,6 +361,16 @@ export class SQLitePatternStore {
     `));
         this.prepared.set('getAllEmbeddings', this.db.prepare(`
       SELECT pattern_id, embedding, dimension FROM qe_pattern_embeddings
+    `));
+        // Recent-first variant used by the RVF backfill path. Order by
+        // qe_patterns.last_used_at (then created_at) so the bounded backfill keeps
+        // the most actively-used patterns when the cap is hit.
+        this.prepared.set('getRecentEmbeddings', this.db.prepare(`
+      SELECT e.pattern_id, e.embedding, e.dimension
+        FROM qe_pattern_embeddings e
+        LEFT JOIN qe_patterns p ON p.id = e.pattern_id
+       ORDER BY COALESCE(p.last_used_at, p.created_at, e.created_at) DESC
+       LIMIT ?
     `));
         this.prepared.set('countPatterns', this.db.prepare(`
       SELECT COUNT(*) as count FROM qe_patterns
@@ -518,6 +528,26 @@ export class SQLitePatternStore {
             embedding: Array.from(new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.dimension)),
         }));
     }
+    /**
+     * Get up to `limit` embeddings, most-recently-used first.
+     *
+     * Used by the RVF backfill path so an operator-truncated patterns.rvf does
+     * not trigger an unbounded re-ingest of every historical embedding (which
+     * is what reproduces the "regrew to 59 GB" pathology after `: > patterns.rvf`).
+     */
+    getRecentEmbeddings(limit) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        const cap = Number.isFinite(limit) && limit > 0 ? Math.floor(limit) : 1000;
+        const stmt = this.prepared.get('getRecentEmbeddings');
+        if (!stmt)
+            throw new Error('Prepared statement not ready');
+        const rows = stmt.all(cap);
+        return rows.map(row => ({
+            patternId: row.pattern_id,
+            embedding: Array.from(new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.dimension)),
+        }));
+    }
     /**
      * Record pattern usage.
      *