agentic-qe 3.15.13 → 3.15.15

package/dist/learning/dream/dream-engine.js

@@ -19,14 +19,24 @@ import { toErrorMessage } from '../../shared/error-utils.js';
 import { PatternMutationError } from '../pattern-mutation-error.js';
 import { LoggerFactory } from '../../logging/index.js';
 const dreamLogger = LoggerFactory.create('dream-engine');
+/** #1239: an unreadable retention setting is reported once per process. */
+let retentionWarningsLogged = false;
+/**
+ * #1239: how long retention waits for a locked database. better-sqlite3 waits
+ * synchronously, and retention is optional (the next dream retries), so it
+ * gives up quickly instead of using the engine's 60 s commit wait.
+ */
+export const RETENTION_BUSY_TIMEOUT_MS = 1000;
 import { v4 as uuidv4 } from 'uuid';
 import { getUnifiedPersistence } from '../../kernel/unified-persistence.js';
 import { registerStoreCloser } from '../../kernel/process-lifecycle.js';
 import { ConceptGraph } from './concept-graph.js';
+import { DREAM_LOCK_WAIT_MS, DREAM_SHUTDOWN_LOCK_WAIT_MS, isBusyError, retryWhileBusy, withBusyTimeout, } from './dream-lock-retry.js';
 import { SpreadingActivation, } from './spreading-activation.js';
 import { InsightGenerator, } from './insight-generator.js';
 import { DEFAULT_VALIDATION_THRESHOLDS, } from './rvcow-branch-manager.js';
 import { commitDream, markAbandonedCycles, ABANDONED_CYCLE_GRACE_MS, } from './dream-commit.js';
+import { describeRetention, resolveDefaultRetention, runDreamRetention, } from '../learning-retention.js';
 /**
  * Default DreamEngine configuration
  */
@@ -268,16 +278,14 @@ export class DreamEngine {
                 await this.persistence.initialize();
             }
             this.db = this.persistence.getDatabase();
-            // ADR-001 Option C: dynamic-import paths in checkAndTriggerDream() open
-            // a fresh handle that the hook/worker busy_timeout pragmas never reach.
-            // 60s patient timeout here keeps dream cycles from failing under WAL
-            // contention from MCP workers (was 71% failure rate observed).
-            try {
-                this.db.pragma('busy_timeout = 60000');
-            }
-            catch { /* fail-soft */ }
-            // Migrate legacy schema: rename 'duration' → 'duration_ms' if needed
-            this.migrateSchema();
+            // #1253: no 60 s busy_timeout on this shared connection. better-sqlite3
+            // waits synchronously, so it froze the whole process (the MCP event loop)
+            // while another writer held the database. Dream writes instead wait
+            // briefly per attempt and retry asynchronously (dream-lock-retry.ts),
+            // which keeps the patience that avoided failures under contention (#1060).
+            // Migrate legacy schema: rename 'duration' → 'duration_ms' if needed.
+            // #1253: a short lock wait only; migrateSchema() logs and carries on.
+            withBusyTimeout(this.db, DREAM_LOCK_WAIT_MS, () => this.migrateSchema());
             // Initialize concept graph (shares same unified persistence)
             this.graph = new ConceptGraph();
             await this.graph.initialize();
@@ -287,9 +295,19 @@ export class DreamEngine {
                     'committed in one short transaction instead of a SAVEPOINT held across the dream');
             }
             // #1060: a process that exited mid-dream left its cycle 'running'.
-            const abandoned = markAbandonedCycles(this.db, Math.max(this.config.maxDurationMs, MAX_TOOL_DREAM_MS) + ABANDONED_CYCLE_GRACE_MS);
-            if (abandoned > 0) {
-                dreamLogger.info('Marked abandoned dream cycles as interrupted', { abandoned });
+            // #1253: housekeeping, so a locked database skips it after a short
+            // wait instead of blocking the process; the next initialize() retries.
+            const db = this.db;
+            try {
+                const abandoned = withBusyTimeout(db, DREAM_LOCK_WAIT_MS, () => markAbandonedCycles(db, Math.max(this.config.maxDurationMs, MAX_TOOL_DREAM_MS) + ABANDONED_CYCLE_GRACE_MS));
+                if (abandoned > 0) {
+                    dreamLogger.info('Marked abandoned dream cycles as interrupted', { abandoned });
+                }
+            }
+            catch (error) {
+                if (!isBusyError(error))
+                    throw error;
+                dreamLogger.warn('Abandoned dream cycle sweep skipped: database is locked', { error: toErrorMessage(error) });
             }
             this.initialized = true;
             this.closing = false;
@@ -424,7 +442,7 @@ export class DreamEngine {
         let committed = false;
         try {
             if (commitResults)
-                await this.saveCycle(cycle);
+                await this.saveCycle(cycle, live);
             if (!live())
                 throw new Error('Dream cycle cancelled');
             // 1. Check we have enough concepts
@@ -474,8 +492,10 @@ export class DreamEngine {
                 committed = true; // nothing to record: the caller decides
                 return { ...result, pending: computed };
             }
-            result.commitDurationMs = this.commitComputed(db, computed);
+            const commit = await this.commitComputed(db, computed, live);
             committed = true;
+            result.commitDurationMs = commit.durationMs;
+            result.retention = commit.retention;
             return result;
         }
         catch (error) {
@@ -488,7 +508,7 @@ export class DreamEngine {
                 cycle.durationMs = Date.now() - startTime;
                 if (commitResults && this.db === db) {
                     try {
-                        this.updateCycleSync(cycle);
+                        await retryWhileBusy(db, () => this.updateCycleSync(cycle), { shouldContinue: () => this.db === db });
                     }
                     catch { /* best effort */ }
                 }
@@ -517,9 +537,10 @@ export class DreamEngine {
      * #1060: write the results of a `dream(…, { commit: false })` run, in the
      * same single transaction a normal dream uses. Returns the commit duration
      * in ms. Throws (and writes nothing) if the engine is closed or the write
-     * fails.
+     * fails. #1253: async, because a locked database is retried without
+     * blocking the process.
      */
-    commitDreamResult(result) {
+    async commitDreamResult(result) {
         this.ensureInitialized();
         if (this.closing) {
             throw new Error('Dream engine is closing; nothing is committed');
@@ -527,14 +548,23 @@ export class DreamEngine {
         if (!result.pending) {
             throw new Error('Dream result has nothing pending to commit (it was already committed)');
         }
-        const duration = this.commitComputed(this.db, result.pending);
+        const db = this.db;
+        const commit = await this.commitComputed(db, result.pending, () => !this.closing && this.db === db);
         result.pending = undefined;
-        result.commitDurationMs = duration;
-        return duration;
+        result.commitDurationMs = commit.durationMs;
+        result.retention = commit.retention;
+        return commit.durationMs;
     }
-    /** Commit a computed dream, then record it in the witness chain (#1060). */
-    commitComputed(db, computed) {
-        const commit = commitDream(db, computed);
+    /**
+     * Commit a computed dream, then record it in the witness chain (#1060) and
+     * run the default retention steps (#1239).
+     */
+    async commitComputed(db, computed, live) {
+        // #1253: retry a locked database asynchronously; each attempt is short.
+        const commit = await retryWhileBusy(db, () => commitDream(db, computed), {
+            shouldContinue: live,
+            cancelledError: () => new Error('Dream cycle cancelled'),
+        });
         try {
             this._witnessChain?.append('DREAM_COMMIT', {
                 cycleId: computed.cycle.id,
@@ -544,7 +574,43 @@ export class DreamEngine {
             }, 'dream-engine');
         }
         catch { /* best-effort witness */ }
-        return commit.commitDurationMs;
+        return { durationMs: commit.commitDurationMs, retention: this.runDefaultRetention(db) };
+    }
+    /**
+     * #1239: after a committed dream, drop failed cycles past their retention
+     * window and outcome concepts beyond the cap when AQE_DREAM_RETENTION=on;
+     * otherwise only count what that would remove. The dream is already
+     * committed, so a failure here (a locked database, say) is only logged.
+     */
+    runDefaultRetention(db) {
+        const settings = resolveDefaultRetention();
+        if (settings.warnings.length > 0 && !retentionWarningsLogged) {
+            retentionWarningsLogged = true;
+            for (const warning of settings.warnings)
+                dreamLogger.warn(warning);
+        }
+        const previousBusyTimeout = db.pragma('busy_timeout', { simple: true });
+        try {
+            db.pragma(`busy_timeout = ${RETENTION_BUSY_TIMEOUT_MS}`);
+            const report = runDreamRetention(db, settings);
+            if (report.enabled) {
+                for (const step of report.steps) {
+                    if (step.affected > 0)
+                        dreamLogger.info(`Retention ${describeRetention(step)}`);
+                }
+            }
+            return report;
+        }
+        catch (error) {
+            dreamLogger.warn('Dream retention skipped after dream commit', { error: toErrorMessage(error) });
+            return { enabled: settings.enabled, steps: [], error: toErrorMessage(error) };
+        }
+        finally {
+            try {
+                db.pragma(`busy_timeout = ${previousBusyTimeout}`);
+            }
+            catch { /* connection gone */ }
+        }
     }
     /**
      * #1060: build a synchronous handler that stops a still-running dream and
@@ -562,7 +628,9 @@ export class DreamEngine {
             cycle.endTime = new Date();
             cycle.durationMs = Date.now() - startTime;
             try {
-                this.updateCycleSync(cycle);
+                // #1253: a short wait only; if the database stays locked, the
+                // abandoned-cycle sweep marks the cycle interrupted later.
+                withBusyTimeout(this.db, DREAM_SHUTDOWN_LOCK_WAIT_MS, () => this.updateCycleSync(cycle));
             }
             catch {
                 // Best effort during shutdown
@@ -861,16 +929,17 @@ export class DreamEngine {
     // ==========================================================================
     // Persistence
     // ==========================================================================
-    async saveCycle(cycle) {
+    async saveCycle(cycle, live) {
         if (!this.db)
             return;
+        const db = this.db;
         const stmt = this.db.prepare(`
       INSERT INTO dream_cycles
       (id, start_time, end_time, duration_ms, concepts_processed, associations_found,
        insights_generated, status, error, created_at)
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     `);
-        stmt.run(cycle.id, cycle.startTime.toISOString(), cycle.endTime?.toISOString() ?? null, cycle.durationMs ?? null, cycle.conceptsProcessed, cycle.associationsFound, cycle.insightsGenerated, cycle.status, cycle.error ?? null, cycle.startTime.toISOString());
+        await retryWhileBusy(db, () => stmt.run(cycle.id, cycle.startTime.toISOString(), cycle.endTime?.toISOString() ?? null, cycle.durationMs ?? null, cycle.conceptsProcessed, cycle.associationsFound, cycle.insightsGenerated, cycle.status, cycle.error ?? null, cycle.startTime.toISOString()), { shouldContinue: live, cancelledError: () => new Error('Dream cycle cancelled') });
     }
     updateCycleSync(cycle) {
         if (!this.db)

package/dist/learning/dream/dream-lock-retry.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Lock waits for dream writes (#1253).
+ *
+ * better-sqlite3 waits for a locked database synchronously, so a long
+ * busy_timeout freezes the whole process: in the MCP server that is the event
+ * loop and every tool call. Dream writes therefore wait only briefly per
+ * attempt and retry asynchronously, so other work runs between attempts.
+ */
+import type { Database as DatabaseType } from 'better-sqlite3';
+/** Longest a single dream write attempt blocks the process on a locked database. */
+export declare const DREAM_LOCK_WAIT_MS = 200;
+/** Total time a dream write keeps retrying a locked database before it fails. */
+export declare const DREAM_LOCK_RETRY_BUDGET_MS = 60000;
+/** True for SQLite "database is locked" / "busy" failures. */
+export declare function isBusyError(error: unknown): boolean;
+/** Run `fn` with a temporary busy_timeout, restoring the connection's previous value. */
+export declare function withBusyTimeout<T>(db: DatabaseType, timeoutMs: number, fn: () => T): T;
+export interface RetryWhileBusyOptions {
+    /** Synchronous wait per attempt (default DREAM_LOCK_WAIT_MS). */
+    waitMs?: number;
+    /** Total retry time before the busy error is thrown (default DREAM_LOCK_RETRY_BUDGET_MS). */
+    budgetMs?: number;
+    /** Checked before every attempt; when false, `cancelledError` is thrown instead of retrying. */
+    shouldContinue?: () => boolean;
+    cancelledError?: () => Error;
+    /** Injectable for tests. */
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => number;
+}
+/**
+ * Run a synchronous write, retrying while the database is locked. Each attempt
+ * blocks for at most `waitMs`; between attempts the event loop is free. Errors
+ * other than a busy database are thrown at once.
+ */
+export declare function retryWhileBusy<T>(db: DatabaseType, fn: () => T, options?: RetryWhileBusyOptions): Promise<T>;
+/** Wait for the synchronous best-effort write made while a dream is interrupted at shutdown. */
+export declare const DREAM_SHUTDOWN_LOCK_WAIT_MS = 1000;
+//# sourceMappingURL=dream-lock-retry.d.ts.map

package/dist/learning/dream/dream-lock-retry.js

@@ -0,0 +1,66 @@
+/**
+ * Lock waits for dream writes (#1253).
+ *
+ * better-sqlite3 waits for a locked database synchronously, so a long
+ * busy_timeout freezes the whole process: in the MCP server that is the event
+ * loop and every tool call. Dream writes therefore wait only briefly per
+ * attempt and retry asynchronously, so other work runs between attempts.
+ */
+/** Longest a single dream write attempt blocks the process on a locked database. */
+export const DREAM_LOCK_WAIT_MS = 200;
+/** Total time a dream write keeps retrying a locked database before it fails. */
+export const DREAM_LOCK_RETRY_BUDGET_MS = 60_000;
+/** Longest pause between attempts; the pause doubles from 25 ms up to this. */
+const MAX_BACKOFF_MS = 1000;
+/** True for SQLite "database is locked" / "busy" failures. */
+export function isBusyError(error) {
+    const code = error?.code;
+    if (typeof code === 'string' && /^SQLITE_(BUSY|LOCKED)/.test(code))
+        return true;
+    const message = error instanceof Error ? error.message : String(error);
+    return /database is locked|database table is locked|SQLITE_BUSY/i.test(message);
+}
+/** Run `fn` with a temporary busy_timeout, restoring the connection's previous value. */
+export function withBusyTimeout(db, timeoutMs, fn) {
+    const previous = db.pragma('busy_timeout', { simple: true });
+    db.pragma(`busy_timeout = ${timeoutMs}`);
+    try {
+        return fn();
+    }
+    finally {
+        try {
+            db.pragma(`busy_timeout = ${previous}`);
+        }
+        catch { /* connection closed */ }
+    }
+}
+/**
+ * Run a synchronous write, retrying while the database is locked. Each attempt
+ * blocks for at most `waitMs`; between attempts the event loop is free. Errors
+ * other than a busy database are thrown at once.
+ */
+export async function retryWhileBusy(db, fn, options = {}) {
+    const waitMs = options.waitMs ?? DREAM_LOCK_WAIT_MS;
+    const budgetMs = options.budgetMs ?? DREAM_LOCK_RETRY_BUDGET_MS;
+    const sleep = options.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    const now = options.now ?? Date.now;
+    const started = now();
+    let backoff = 25;
+    for (;;) {
+        if (options.shouldContinue && !options.shouldContinue()) {
+            throw options.cancelledError?.() ?? new Error('Cancelled while waiting for a locked database');
+        }
+        try {
+            return withBusyTimeout(db, waitMs, fn);
+        }
+        catch (error) {
+            if (!isBusyError(error) || now() - started + backoff >= budgetMs)
+                throw error;
+        }
+        await sleep(backoff);
+        backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);
+    }
+}
+/** Wait for the synchronous best-effort write made while a dream is interrupted at shutdown. */
+export const DREAM_SHUTDOWN_LOCK_WAIT_MS = 1000;
+//# sourceMappingURL=dream-lock-retry.js.map

package/dist/learning/dream/speculative-dreamer.js

@@ -117,7 +117,7 @@ export class SpeculativeDreamer {
         let commitDurationMs;
         if (winner?.result) {
             try {
-                commitDurationMs = this.engine.commitDreamResult(winner.result);
+                commitDurationMs = await this.engine.commitDreamResult(winner.result);
                 winner.selected = true;
             }
             catch (error) {

package/dist/learning/learning-retention.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Learning Retention (#1239)
+ *
+ * Bounds the learning tables that otherwise grow for as long as a project
+ * lives. Two kinds of step, kept apart on purpose:
+ *
+ * - Dream retention steps remove only rows that carry no learning value.
+ *   They are off by default. With `AQE_DREAM_RETENTION=on` they run after
+ *   each committed dream cycle; while off, the dream reports what they
+ *   would remove (COUNT queries only, nothing is written):
+ *     - failed/interrupted `dream_cycles` past a retention window. A cycle
+ *       that did not commit wrote no insights, and any cycle that still has
+ *       an insight is kept.
+ *     - `outcome` concept nodes beyond a cap, with their edges. They are
+ *       derived from `qe_pattern_usage`, which is not touched, and only the
+ *       newest 100 usage rows are ever loaded back, so a cap >= 100 does not
+ *       churn.
+ *
+ * - Opt-in steps drop data that could still matter to someone, so they run
+ *   only from `aqe learning prune`, which defaults to a dry run:
+ *     - compacting soft-archived and merged experiences: `result_json` and the
+ *       embedding are cleared, the row and its counters stay (the experience
+ *       count stays monotonic, v3.9.22).
+ *     - `routing_outcomes` older than a number of days.
+ *
+ * Every step reports the row counts it measured before and after, or, in a
+ * dry run, the counts it would change.
+ */
+import type { Database as DatabaseType } from 'better-sqlite3';
+export interface TableCount {
+    readonly table: string;
+    readonly before: number;
+    readonly after: number;
+}
+export interface RetentionStepResult {
+    readonly step: RetentionStep;
+    readonly dryRun: boolean;
+    /** Rows deleted, or for compaction rows whose payload was cleared. */
+    readonly affected: number;
+    /**
+     * Row counts per table. `after` is read back after the write; in a dry run
+     * it is the projection (`before` minus the rows the step would delete).
+     */
+    readonly tables: readonly TableCount[];
+    /** Bytes of payload removed (compaction only), measured with length(). */
+    readonly payloadBytes?: number;
+}
+export type RetentionStep = 'failed-dream-cycles' | 'outcome-concepts' | 'compact-experiences' | 'routing-outcomes';
+export declare const DEFAULT_FAILED_DREAM_CYCLE_RETENTION_DAYS = 7;
+export declare const DEFAULT_OUTCOME_CONCEPT_CAP = 2000;
+/** Below this the cap would delete outcome nodes that the next dream reloads. */
+export declare const MIN_OUTCOME_CONCEPT_CAP = 100;
+/** Failed or interrupted cycles older than `retentionDays` that have no insight. */
+export declare function pruneFailedDreamCycles(db: DatabaseType, options?: {
+    retentionDays?: number;
+    dryRun?: boolean;
+}): RetentionStepResult;
+/** Keep the newest `maxNodes` outcome concept nodes; delete the rest and their edges. */
+export declare function capOutcomeConcepts(db: DatabaseType, options?: {
+    maxNodes?: number;
+    dryRun?: boolean;
+}): RetentionStepResult;
+/** Clear `result_json` and the embedding of archived and merged experiences. */
+export declare function compactConsolidatedExperiences(db: DatabaseType, options?: {
+    dryRun?: boolean;
+}): RetentionStepResult;
+/** Delete routing outcomes older than `olderThanDays`. */
+export declare function pruneRoutingOutcomes(db: DatabaseType, options: {
+    olderThanDays: number;
+    dryRun?: boolean;
+}): RetentionStepResult;
+export interface DefaultRetentionSettings {
+    /** AQE_DREAM_RETENTION: false (the default) previews the steps instead of running them. */
+    readonly enabled: boolean;
+    /** null = off */
+    readonly failedDreamCycleRetentionDays: number | null;
+    /** null = off */
+    readonly outcomeConceptCap: number | null;
+    readonly warnings: readonly string[];
+}
+/**
+ * Read `AQE_DREAM_RETENTION` (off by default) and the per-step settings
+ * `AQE_FAILED_DREAM_CYCLE_RETENTION_DAYS` (default 7) and
+ * `AQE_OUTCOME_CONCEPT_CAP` (default 2000). Each step setting accepts "off".
+ */
+export declare function resolveDefaultRetention(env?: Readonly<Record<string, string | undefined>>): DefaultRetentionSettings;
+export interface DreamRetentionReport {
+    /** False: the steps ran as a dry run and wrote nothing. */
+    readonly enabled: boolean;
+    readonly steps: readonly RetentionStepResult[];
+    /** Set when retention could not run (a locked database, say); nothing was deleted. */
+    readonly error?: string;
+}
+/**
+ * The dream retention steps that are not switched off individually: run
+ * when retention is enabled, otherwise previewed as a dry run.
+ */
+export declare function runDreamRetention(db: DatabaseType, settings?: DefaultRetentionSettings): DreamRetentionReport;
+/** Lines for a dream's output: what retention removed, or would remove while off. */
+export declare function describeDreamRetention(report: DreamRetentionReport): string[];
+/** One line per step, for logs: "outcome-concepts: 3000 removed (concept_nodes 5000 -> 2000, ...)". */
+export declare function describeRetention(result: RetentionStepResult): string;
+//# sourceMappingURL=learning-retention.d.ts.map

package/dist/learning/learning-retention.js

@@ -0,0 +1,213 @@
+/**
+ * Learning Retention (#1239)
+ *
+ * Bounds the learning tables that otherwise grow for as long as a project
+ * lives. Two kinds of step, kept apart on purpose:
+ *
+ * - Dream retention steps remove only rows that carry no learning value.
+ *   They are off by default. With `AQE_DREAM_RETENTION=on` they run after
+ *   each committed dream cycle; while off, the dream reports what they
+ *   would remove (COUNT queries only, nothing is written):
+ *     - failed/interrupted `dream_cycles` past a retention window. A cycle
+ *       that did not commit wrote no insights, and any cycle that still has
+ *       an insight is kept.
+ *     - `outcome` concept nodes beyond a cap, with their edges. They are
+ *       derived from `qe_pattern_usage`, which is not touched, and only the
+ *       newest 100 usage rows are ever loaded back, so a cap >= 100 does not
+ *       churn.
+ *
+ * - Opt-in steps drop data that could still matter to someone, so they run
+ *   only from `aqe learning prune`, which defaults to a dry run:
+ *     - compacting soft-archived and merged experiences: `result_json` and the
+ *       embedding are cleared, the row and its counters stay (the experience
+ *       count stays monotonic, v3.9.22).
+ *     - `routing_outcomes` older than a number of days.
+ *
+ * Every step reports the row counts it measured before and after, or, in a
+ * dry run, the counts it would change.
+ */
+export const DEFAULT_FAILED_DREAM_CYCLE_RETENTION_DAYS = 7;
+export const DEFAULT_OUTCOME_CONCEPT_CAP = 2000;
+/** Below this the cap would delete outcome nodes that the next dream reloads. */
+export const MIN_OUTCOME_CONCEPT_CAP = 100;
+function tableExists(db, table) {
+    return !!db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`).get(table);
+}
+function count(db, sql, ...params) {
+    return db.prepare(sql).get(...params).n;
+}
+function rowCount(db, table) {
+    return count(db, `SELECT COUNT(*) AS n FROM ${table}`);
+}
+/**
+ * Measure, then apply, in one transaction, reading the table counts around
+ * it. A real run is IMMEDIATE (waits on busy_timeout instead of failing on
+ * lock upgrade); a dry run only reads.
+ */
+function runStep(db, step, tables, dryRun, measure, apply) {
+    if (!tables.every(t => tableExists(db, t))) {
+        return { step, dryRun, affected: 0, tables: [] };
+    }
+    const run = db.transaction(() => {
+        const before = tables.map(t => rowCount(db, t));
+        const { affected, deletes, payloadBytes } = measure();
+        if (!dryRun && affected > 0)
+            apply();
+        const after = dryRun
+            ? tables.map((t, i) => before[i] - (deletes[t] ?? 0))
+            : tables.map(t => rowCount(db, t));
+        return {
+            step,
+            dryRun,
+            affected,
+            tables: tables.map((table, i) => ({ table, before: before[i], after: after[i] })),
+            ...(payloadBytes !== undefined ? { payloadBytes } : {}),
+        };
+    });
+    return dryRun ? run() : run.immediate();
+}
+// ---------------------------------------------------------------------------
+// Default steps
+// ---------------------------------------------------------------------------
+/** Failed or interrupted cycles older than `retentionDays` that have no insight. */
+export function pruneFailedDreamCycles(db, options = {}) {
+    const retentionDays = options.retentionDays ?? DEFAULT_FAILED_DREAM_CYCLE_RETENTION_DAYS;
+    assertWholeNumber('retentionDays', retentionDays, 1);
+    // julianday() reads both the ISO strings the engine writes and SQLite's own
+    // datetime('now') format, which a plain string comparison would mix up.
+    const where = `status IN ('failed', 'interrupted')
+    AND julianday(COALESCE(end_time, start_time, created_at)) < julianday('now', ?)
+    AND NOT EXISTS (SELECT 1 FROM dream_insights i WHERE i.cycle_id = dream_cycles.id)`;
+    const window = `-${retentionDays} days`;
+    return runStep(db, 'failed-dream-cycles', ['dream_cycles', 'dream_insights'], options.dryRun ?? false, () => {
+        const n = count(db, `SELECT COUNT(*) AS n FROM dream_cycles WHERE ${where}`, window);
+        return { affected: n, deletes: { dream_cycles: n } };
+    }, () => { db.prepare(`DELETE FROM dream_cycles WHERE ${where}`).run(window); });
+}
+/** Keep the newest `maxNodes` outcome concept nodes; delete the rest and their edges. */
+export function capOutcomeConcepts(db, options = {}) {
+    const maxNodes = options.maxNodes ?? DEFAULT_OUTCOME_CONCEPT_CAP;
+    assertWholeNumber('maxNodes', maxNodes, MIN_OUTCOME_CONCEPT_CAP);
+    const doomed = `SELECT id FROM concept_nodes WHERE concept_type = 'outcome'
+    ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?`;
+    return runStep(db, 'outcome-concepts', ['concept_nodes', 'concept_edges'], options.dryRun ?? false, () => {
+        const nodes = count(db, `SELECT COUNT(*) AS n FROM (${doomed})`, maxNodes);
+        const edges = count(db, `SELECT COUNT(*) AS n FROM concept_edges WHERE source IN (${doomed}) OR target IN (${doomed})`, maxNodes, maxNodes);
+        return { affected: nodes, deletes: { concept_nodes: nodes, concept_edges: edges } };
+    }, () => {
+        // Edges first and explicitly: the ON DELETE CASCADE only fires when the
+        // connection has foreign_keys on, which not every caller sets.
+        db.prepare(`DELETE FROM concept_edges WHERE source IN (${doomed}) OR target IN (${doomed})`)
+            .run(maxNodes, maxNodes);
+        db.prepare(`DELETE FROM concept_nodes WHERE id IN (${doomed})`).run(maxNodes);
+    });
+}
+// ---------------------------------------------------------------------------
+// Opt-in steps (aqe learning prune)
+// ---------------------------------------------------------------------------
+/** Soft-archived ('archived') and merged (survivor id) rows; 'contradicted' is left alone. */
+const CONSOLIDATED = `consolidated_into IS NOT NULL AND consolidated_into <> 'contradicted'`;
+const HAS_PAYLOAD = `(result_json IS NOT NULL OR embedding IS NOT NULL)`;
+/** Clear `result_json` and the embedding of archived and merged experiences. */
+export function compactConsolidatedExperiences(db, options = {}) {
+    const where = `${CONSOLIDATED} AND ${HAS_PAYLOAD}`;
+    return runStep(db, 'compact-experiences', ['captured_experiences'], options.dryRun ?? false, () => {
+        const row = db.prepare(`SELECT COUNT(*) AS n,
+                COALESCE(SUM(COALESCE(length(result_json), 0) + COALESCE(length(embedding), 0)), 0) AS bytes
+           FROM captured_experiences WHERE ${where}`).get();
+        return { affected: row.n, deletes: {}, payloadBytes: row.bytes };
+    }, () => {
+        db.prepare(`UPDATE captured_experiences
+            SET result_json = NULL, embedding = NULL, embedding_dimension = NULL
+          WHERE ${where}`).run();
+    });
+}
+/** Delete routing outcomes older than `olderThanDays`. */
+export function pruneRoutingOutcomes(db, options) {
+    assertWholeNumber('olderThanDays', options.olderThanDays, 1);
+    const where = `julianday(created_at) < julianday('now', ?)`;
+    const window = `-${options.olderThanDays} days`;
+    return runStep(db, 'routing-outcomes', ['routing_outcomes'], options.dryRun ?? false, () => {
+        const n = count(db, `SELECT COUNT(*) AS n FROM routing_outcomes WHERE ${where}`, window);
+        return { affected: n, deletes: { routing_outcomes: n } };
+    }, () => { db.prepare(`DELETE FROM routing_outcomes WHERE ${where}`).run(window); });
+}
+const OFF_VALUES = new Set(['off', 'false', 'no', '0']);
+function parseSetting(name, raw, fallback, min, warnings) {
+    if (raw === undefined || raw.trim() === '')
+        return fallback;
+    const value = raw.trim().toLowerCase();
+    if (OFF_VALUES.has(value))
+        return null;
+    if (/^\d+$/.test(value) && Number(value) >= min)
+        return Number(value);
+    // These settings delete rows, so a value that cannot be read turns the step off.
+    warnings.push(`Invalid ${name}=${JSON.stringify(raw)}; expected a whole number >= ${min} or "off". ` +
+        'That retention step is off until this is fixed; nothing is deleted by it.');
+    return null;
+}
+const ON_VALUES = new Set(['on', 'true', 'yes', '1']);
+/** Unset or "off" disables; "on" enables; anything else disables with a warning. */
+function parseSwitch(name, raw, warnings) {
+    if (raw === undefined || raw.trim() === '')
+        return false;
+    const value = raw.trim().toLowerCase();
+    if (ON_VALUES.has(value))
+        return true;
+    if (!OFF_VALUES.has(value)) {
+        warnings.push(`Invalid ${name}=${JSON.stringify(raw)}; expected "on" or "off". ` +
+            'Dream retention stays off until this is fixed; nothing is deleted.');
+    }
+    return false;
+}
+/**
+ * Read `AQE_DREAM_RETENTION` (off by default) and the per-step settings
+ * `AQE_FAILED_DREAM_CYCLE_RETENTION_DAYS` (default 7) and
+ * `AQE_OUTCOME_CONCEPT_CAP` (default 2000). Each step setting accepts "off".
+ */
+export function resolveDefaultRetention(env = process.env) {
+    const warnings = [];
+    return {
+        enabled: parseSwitch('AQE_DREAM_RETENTION', env.AQE_DREAM_RETENTION, warnings),
+        failedDreamCycleRetentionDays: parseSetting('AQE_FAILED_DREAM_CYCLE_RETENTION_DAYS', env.AQE_FAILED_DREAM_CYCLE_RETENTION_DAYS, DEFAULT_FAILED_DREAM_CYCLE_RETENTION_DAYS, 1, warnings),
+        outcomeConceptCap: parseSetting('AQE_OUTCOME_CONCEPT_CAP', env.AQE_OUTCOME_CONCEPT_CAP, DEFAULT_OUTCOME_CONCEPT_CAP, MIN_OUTCOME_CONCEPT_CAP, warnings),
+        warnings,
+    };
+}
+/**
+ * The dream retention steps that are not switched off individually: run
+ * when retention is enabled, otherwise previewed as a dry run.
+ */
+export function runDreamRetention(db, settings = resolveDefaultRetention()) {
+    const dryRun = !settings.enabled;
+    const steps = [];
+    if (settings.failedDreamCycleRetentionDays !== null) {
+        steps.push(pruneFailedDreamCycles(db, { retentionDays: settings.failedDreamCycleRetentionDays, dryRun }));
+    }
+    if (settings.outcomeConceptCap !== null) {
+        steps.push(capOutcomeConcepts(db, { maxNodes: settings.outcomeConceptCap, dryRun }));
+    }
+    return { enabled: settings.enabled, steps };
+}
+/** Lines for a dream's output: what retention removed, or would remove while off. */
+export function describeDreamRetention(report) {
+    if (report.error)
+        return [`skipped: ${report.error}`];
+    const lines = report.steps.filter(s => s.affected > 0).map(describeRetention);
+    return report.enabled ? lines : lines.map(l => `${l} [off: set AQE_DREAM_RETENTION=on]`);
+}
+/** One line per step, for logs: "outcome-concepts: 3000 removed (concept_nodes 5000 -> 2000, ...)". */
+export function describeRetention(result) {
+    const verb = result.step === 'compact-experiences' ? 'compacted' : 'removed';
+    const counts = result.tables
+        .map(t => `${t.table} ${t.before} -> ${t.after}${result.dryRun ? ' projected' : ''}`)
+        .join(', ');
+    return `${result.step}: ${result.affected} ${result.dryRun ? `would be ${verb}` : verb}` +
+        (counts ? ` (${counts})` : '');
+}
+function assertWholeNumber(name, value, min) {
+    if (!Number.isInteger(value) || value < min) {
+        throw new Error(`${name} must be a whole number >= ${min}, got ${value}`);
+    }
+}
+//# sourceMappingURL=learning-retention.js.map