agentic-qe 3.9.20 → 3.9.22

package/dist/integrations/ruvector/shared-rvf-adapter.js

@@ -26,14 +26,28 @@ export function getSharedRvfAdapter(dataDir = '.agentic-qe', dimensions = 384) {
     try {
         // Dynamic require to match the bundled build pattern used elsewhere
         // eslint-disable-next-line @typescript-eslint/no-require-imports
-        const { isRvfNativeAvailable, createRvfStore } = require('./rvf-native-adapter.js');
+        const { isRvfNativeAvailable, createRvfStore, openRvfStore } = require('./rvf-native-adapter.js');
         if (!isRvfNativeAvailable()) {
             console.warn('[RVF] Native bindings unavailable — agent branching and dream COW disabled. ' +
                 'Install @ruvector/rvf-node to enable.');
             return null;
         }
+        // eslint-disable-next-line @typescript-eslint/no-require-imports
         const path = require('path');
-        sharedAdapter = createRvfStore(path.join(dataDir, 'patterns.rvf'), dimensions);
+        const rvfPath = path.join(dataDir, 'patterns.rvf');
+        // Open-or-create with a try-ladder rather than `existsSync` gate.
+        // Reasons:
+        //  1. RVF native's `RvfDatabase.create()` throws `0x0303 FsyncFailed`
+        //     when the target file already exists (verified against both 0.1.7
+        //     and 0.1.8 native binaries on linux-arm64). Earlier init phases
+        //     legitimately produce patterns.rvf, so subsequent CLI / MCP boot
+        //     would crash without this guard. (Jordi #439 / RUFLO P020.)
+        //  2. A bare `existsSync(...) ? open : create` is racy across
+        //     processes — two parallel `aqe` invocations during init can both
+        //     observe absent and both call `create`, with the second hitting
+        //     FsyncFailed regardless. The try-ladder degrades to whichever
+        //     path the OS actually permits.
+        sharedAdapter = openOrCreateRvf(openRvfStore, createRvfStore, rvfPath, dimensions);
         return sharedAdapter;
     }
     catch (error) {
@@ -41,6 +55,100 @@ export function getSharedRvfAdapter(dataDir = '.agentic-qe', dimensions = 384) {
         return null;
     }
 }
+/**
+ * Race-tolerant open-or-create.
+ * Tries open first (cheap, succeeds for the common case where init has
+ * already produced the file). On open failure we attempt create. On the
+ * concurrent-create loser, create itself fails with `FsyncFailed`; we
+ * retry open once more. Any final exception bubbles up to the caller.
+ *
+ * Also asserts `dim()` matches the caller-requested `dimensions` after
+ * open so silent dimension drift between releases / configs is detected
+ * — a bad-dim file is closed and create() is attempted (fail-loud rather
+ * than corrupt-silently, which would manifest as wrong vector hits later).
+ */
+function openOrCreateRvf(openFn, createFn, rvfPath, dimensions) {
+    const tryOpen = () => {
+        try {
+            return { adapter: openFn(rvfPath), err: null };
+        }
+        catch (err) {
+            return { adapter: null, err };
+        }
+    };
+    const isLockHeld = (err) => {
+        const m = err instanceof Error ? err.message : String(err);
+        return m.includes('LockHeld') || m.includes('0x0300');
+    };
+    // Pass 1: try to open whatever's there.
+    let { adapter: opened, err: openErr } = tryOpen();
+    // Pass 1.5: stale-lock recovery. The native binding writes a `<rvfPath>.lock`
+    // file on open and removes it on close. `aqe init` and short-lived CLI
+    // processes routinely exit without an explicit close, leaving a stale
+    // .lock that subsequent invocations interpret as `LockHeld`. Since AQE
+    // CLI is overwhelmingly single-shot serial, the realistic case is "the
+    // prior process is dead and the lock is stale". We unlink the .lock and
+    // retry open exactly once. If a genuinely-live peer existed, it still
+    // holds the file open — but the binding's lock is content-based (lock
+    // file presence), so this is a best-effort cooperative recovery rather
+    // than an OS-level lock break.
+    if (!opened && isLockHeld(openErr)) {
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-require-imports
+            const fs = require('fs');
+            const lockPath = `${rvfPath}.lock`;
+            if (fs.existsSync(lockPath)) {
+                fs.unlinkSync(lockPath);
+                console.warn(`[RVF] Removed stale lock file at ${lockPath} (prior process exited without closing). ` +
+                    'Retrying open. If you see this repeatedly under live concurrency, file an issue.');
+                ({ adapter: opened, err: openErr } = tryOpen());
+            }
+        }
+        catch (recoveryErr) {
+            // Lock removal failed (permissions?) — fall through to error path.
+            if (process.env.DEBUG) {
+                console.debug('[RVF] stale-lock recovery failed:', recoveryErr instanceof Error ? recoveryErr.message : recoveryErr);
+            }
+        }
+    }
+    if (opened) {
+        const actualDim = opened.dimension();
+        if (actualDim === dimensions)
+            return opened;
+        console.warn(`[RVF] patterns.rvf dimension mismatch: file=${actualDim} requested=${dimensions} — ` +
+            'closing and degrading. Delete the .rvf file to recreate at the requested dim.');
+        try {
+            opened.close();
+        }
+        catch { /* best-effort */ }
+        // Don't auto-recreate over a dim-mismatched file. Surface to caller via
+        // throw so getSharedRvfAdapter logs and returns null (degrade to SQLite).
+        throw new Error(`RVF dimension mismatch (file=${actualDim}, requested=${dimensions})`);
+    }
+    // Pass 2: open failed even after stale-lock recovery → try create.
+    try {
+        return createFn(rvfPath, dimensions);
+    }
+    catch (createErr) {
+        // Pass 3: create failed (likely FsyncFailed because a peer process won
+        // the race). Try open one more time.
+        try {
+            const reopened = openFn(rvfPath);
+            if (reopened.dimension() !== dimensions) {
+                try {
+                    reopened.close();
+                }
+                catch { /* best-effort */ }
+                throw new Error(`RVF dimension mismatch after race (file=${reopened.dimension()}, requested=${dimensions})`);
+            }
+            return reopened;
+        }
+        catch {
+            // Fall through with the more informative original error.
+            throw createErr instanceof Error ? createErr : new Error(String(createErr));
+        }
+    }
+}
 /** Close the shared adapter and reset the singleton. */
 export function resetSharedRvfAdapter() {
     if (sharedAdapter) {

package/dist/learning/experience-consolidation.d.ts

@@ -5,6 +5,12 @@
  * - Phase 1: Clusters & merges similar experiences via HNSW
  * - Phase 2: Reinforces quality based on reuse success
  * - Phase 3: Archives truly valueless entries (soft-delete only)
+ * - Phase 4 (safety valve): Soft-archives oldest/lowest-quality rows when a
+ *   domain exceeds `hardThreshold`. Previously did `DELETE FROM` which
+ *   permanently destroyed ~16K rows in production (see release notes for
+ *   v3.9.22). Now strictly non-destructive: rows stay queryable and counted
+ *   by the statusline formula, only `consolidated_into` changes to
+ *   'archived'.
  *
  * Ensures the Exp counter is monotonically non-decreasing.
  */
@@ -14,9 +20,14 @@ export interface ConsolidationResult {
     merged: number;
     /** Experiences with quality recalculated */
     qualityUpdated: number;
-    /** Experiences soft-archived */
+    /** Experiences soft-archived (Phase 3 valueless + Phase 4 safety valve) */
     archived: number;
-    /** Hard-deleted (only when exceeding safety valve) */
+    /**
+     * Deprecated: previously the count of physically-deleted rows when a
+     * domain exceeded `hardThreshold`. Always 0 since v3.9.22 — the safety
+     * valve is now non-destructive (soft-archive only). Kept for callers that
+     * still read the field; new code should treat it as a permanent zero.
+     */
     hardDeleted: number;
     /** Total active experiences remaining */
     activeRemaining: number;
@@ -30,7 +41,12 @@ export interface ConsolidationConfig {
     maxMergesPerRun: number;
     /** Soft threshold: start consolidating when domain exceeds this */
     softThreshold: number;
-    /** Hard threshold: safety valve for hard-delete */
+    /**
+     * Hard threshold: safety valve. When a domain still exceeds this after
+     * Phases 1-3, the oldest/lowest-quality un-applied rows are
+     * soft-archived (consolidated_into = 'archived'). NOT hard-deleted —
+     * archived rows still count toward the statusline Exp formula.
+     */
     hardThreshold: number;
     /** Min age in days for archival eligibility */
     archiveMinAgeDays: number;

package/dist/learning/experience-consolidation.js

@@ -5,6 +5,12 @@
  * - Phase 1: Clusters & merges similar experiences via HNSW
  * - Phase 2: Reinforces quality based on reuse success
  * - Phase 3: Archives truly valueless entries (soft-delete only)
+ * - Phase 4 (safety valve): Soft-archives oldest/lowest-quality rows when a
+ *   domain exceeds `hardThreshold`. Previously did `DELETE FROM` which
+ *   permanently destroyed ~16K rows in production (see release notes for
+ *   v3.9.22). Now strictly non-destructive: rows stay queryable and counted
+ *   by the statusline formula, only `consolidated_into` changes to
+ *   'archived'.
  *
  * Ensures the Exp counter is monotonically non-decreasing.
  */
@@ -100,12 +106,17 @@ export class ExperienceConsolidator {
         result.qualityUpdated = this.reinforceQuality(domain);
         // Phase 3: Archive Valueless
         result.archived = this.archiveValueless(domain);
-        // Safety valve: hard-delete if still over hard threshold
+        // Safety valve: when the domain still exceeds hardThreshold after the
+        // soft phases, archive (not delete) the oldest low-quality excess.
+        // Result is attributed to `archived` — `hardDeleted` stays 0 since
+        // v3.9.22 (see ConsolidationResult docstring).
         const afterCount = this.db.prepare("SELECT COUNT(*) as cnt FROM captured_experiences WHERE domain = ? AND consolidated_into IS NULL").get(domain).cnt;
+        let safetyValveArchived = 0;
         if (afterCount > this.config.hardThreshold) {
-            result.hardDeleted = this.hardDeleteExcess(domain, afterCount);
+            safetyValveArchived = this.hardDeleteExcess(domain, afterCount);
+            result.archived += safetyValveArchived;
         }
-        result.activeRemaining = afterCount - result.hardDeleted;
+        result.activeRemaining = afterCount - safetyValveArchived;
         return result;
     }
     /**
@@ -342,41 +353,60 @@ export class ExperienceConsolidator {
         return result.changes;
     }
     // ============================================================================
-    // Safety Valve: Hard Delete
+    // Safety Valve: Soft Archive (formerly Hard Delete)
     // ============================================================================
+    //
+    // Pre-v3.9.22 this method ran `DELETE FROM captured_experiences`, which
+    // permanently destroyed ~16K rows in production once `code-intelligence`
+    // exceeded the threshold. The replacement keeps the same trigger condition
+    // (domain still over `hardThreshold` after Phases 1-3) but soft-archives
+    // the excess instead. Archived rows remain in the table and continue to
+    // contribute to the statusline Exp formula
+    // (`consolidated_into IS NULL OR consolidated_into = 'archived'`), so the
+    // counter stays monotonic.
+    //
+    // Method name and signature are preserved so the existing caller
+    // (`consolidateDomain`) keeps working unchanged. The returned number now
+    // represents rows transitioned active → archived, not rows deleted.
+    // ConsolidationResult.hardDeleted is kept zero for that reason; the
+    // archive count is rolled into Phase 3's `archived` total.
     hardDeleteExcess(domain, currentCount) {
         const excess = currentCount - this.config.hardThreshold;
         if (excess <= 0)
             return 0;
-        // Delete oldest archived entries first
-        const archivedDeleted = this.db.prepare(`
-      DELETE FROM captured_experiences
+        // Soft-archive oldest low-quality un-applied active rows.
+        // Same selection criteria as the previous hard-delete path so the
+        // memory pressure relief behavior is unchanged — only the operation
+        // (UPDATE vs DELETE) changes.
+        const result = this.db.prepare(`
+      UPDATE captured_experiences
+      SET consolidated_into = 'archived'
       WHERE id IN (
         SELECT id FROM captured_experiences
-        WHERE domain = ? AND consolidated_into = 'archived'
-        ORDER BY started_at ASC
+        WHERE domain = ? AND consolidated_into IS NULL AND application_count = 0
+        ORDER BY quality ASC, started_at ASC
         LIMIT ?
       )
     `).run(domain, excess);
-        let totalDeleted = archivedDeleted.changes;
-        // If still over, delete oldest low-quality active entries
-        if (totalDeleted < excess) {
-            const remaining = excess - totalDeleted;
-            const activeDeleted = this.db.prepare(`
-        DELETE FROM captured_experiences
-        WHERE id IN (
-          SELECT id FROM captured_experiences
-          WHERE domain = ? AND consolidated_into IS NULL AND application_count = 0
-          ORDER BY quality ASC, started_at ASC
-          LIMIT ?
-        )
-      `).run(domain, remaining);
-            totalDeleted += activeDeleted.changes;
-        }
-        if (totalDeleted > 0) {
-            console.warn(`[ExperienceConsolidator] Safety valve: hard-deleted ${totalDeleted} from ${domain}`);
+        const archived = result.changes;
+        if (archived > 0) {
+            try {
+                this.db.prepare(`
+          INSERT INTO experience_consolidation_log (id, domain, action, source_ids, details, created_at)
+          VALUES (?, ?, 'safety-valve-archive', '[]', ?, datetime('now'))
+        `).run(uuidv4(), domain, JSON.stringify({
+                    count: archived,
+                    currentCount,
+                    hardThreshold: this.config.hardThreshold,
+                    note: 'soft-archive replacement for legacy hard-delete safety valve',
+                }));
+            }
+            catch {
+                // Logging is best-effort; safety-valve effect is in the UPDATE above.
+            }
+            console.warn(`[ExperienceConsolidator] Safety valve: soft-archived ${archived} from ${domain} (was ${currentCount}, threshold ${this.config.hardThreshold})`);
         }
-        return totalDeleted;
+        return archived;
     }
     // ============================================================================
     // Utilities

package/dist/learning/pattern-store.js

@@ -1391,7 +1391,44 @@ export function createPatternStore(memory, config) {
                     // Shared adapter unavailable — fall back to direct create
                 }
                 if (!useSharedAdapter) {
-                    const store = new RvfPatternStore((path, dim) => _createRvfStore(path, dim), { rvfPath, base: mergedConfig });
+                    // Race-tolerant open-or-create with dim verification, mirroring
+                    // shared-rvf-adapter.ts. Bare `existsSync ? open : create` is
+                    // racy across processes (two parallel CLI invocations during init
+                    // both observe absent, second hits FsyncFailed regardless). Try
+                    // open first, fall back to create, retry open on create-race.
+                    // (Jordi #439 / RUFLO P020.)
+                    const store = new RvfPatternStore((path, dim) => {
+                        // eslint-disable-next-line @typescript-eslint/no-require-imports
+                        const { openRvfStore } = require('../integrations/ruvector/rvf-native-adapter.js');
+                        const tryOpen = () => {
+                            try {
+                                return openRvfStore(path);
+                            }
+                            catch {
+                                return null;
+                            }
+                        };
+                        let adapter = tryOpen();
+                        if (adapter) {
+                            if (adapter.dimension() !== dim) {
+                                try {
+                                    adapter.close();
+                                }
+                                catch { /* best-effort */ }
+                                throw new Error(`RVF dimension mismatch (file=${adapter.dimension()}, requested=${dim})`);
+                            }
+                            return adapter;
+                        }
+                        try {
+                            return _createRvfStore(path, dim);
+                        }
+                        catch (createErr) {
+                            adapter = tryOpen();
+                            if (adapter && adapter.dimension() === dim)
+                                return adapter;
+                            throw createErr;
+                        }
+                    }, { rvfPath, base: mergedConfig });
                     console.log('[PatternStore] Using RVF-backed store (ADR-066)');
                     return store;
                 }