agentic-qe 3.15.9 → 3.15.11

package/dist/kernel/unified-memory-schemas.js

@@ -7,12 +7,13 @@
 import { HYPERGRAPH_SCHEMA } from '../migrations/20260120_add_hypergraph_tables.js';
 import { PATTERN_NULLS_SCHEMA } from '../migrations/20260611_add_pattern_nulls_table.js';
 import { LEARNING_EVIDENCE_SCHEMA } from '../migrations/20260830_add_learning_evidence_tables.js';
+import { DREAM_INSIGHTS_ARCHIVE_SCHEMA } from '../migrations/20261018_add_dream_insights_archive.js';
 // Re-export for convenience
-export { HYPERGRAPH_SCHEMA, PATTERN_NULLS_SCHEMA, LEARNING_EVIDENCE_SCHEMA };
+export { HYPERGRAPH_SCHEMA, PATTERN_NULLS_SCHEMA, LEARNING_EVIDENCE_SCHEMA, DREAM_INSIGHTS_ARCHIVE_SCHEMA };
 // ============================================================================
 // Schema Version for Migrations
 // ============================================================================
-export const SCHEMA_VERSION = 13; // v12: learning evidence; v13: embedding-space provenance (#633)
+export const SCHEMA_VERSION = 14; // v12: learning evidence; v13: embedding-space provenance (#633); v14: dream_insights_archive (#1116)
 export const SCHEMA_VERSION_TABLE = `
   CREATE TABLE IF NOT EXISTS schema_version (
     id INTEGER PRIMARY KEY CHECK (id = 1),

package/dist/kernel/unified-memory.js

@@ -41,7 +41,7 @@ export { BinaryHeap, UnifiedHnswIndex, createHnswIndex } from './unified-memory-
 // parity tests or benchmarks only.
 import { UnifiedHnswIndex } from './unified-memory-hnsw.js';
 // Import schemas
-import { SCHEMA_VERSION, SCHEMA_VERSION_TABLE, KV_STORE_SCHEMA, VECTORS_SCHEMA, RL_QVALUES_SCHEMA, GOAP_SCHEMA, DREAM_SCHEMA, QE_PATTERNS_SCHEMA, QE_PATTERNS_FTS_SCHEMA, QE_PATTERNS_SCHEMA_WITHOUT_FTS, MINCUT_SCHEMA, SONA_PATTERNS_SCHEMA, FEEDBACK_SCHEMA, HYPERGRAPH_SCHEMA, PATTERN_NULLS_SCHEMA, STATS_TABLES, } from './unified-memory-schemas.js';
+import { SCHEMA_VERSION, SCHEMA_VERSION_TABLE, KV_STORE_SCHEMA, VECTORS_SCHEMA, RL_QVALUES_SCHEMA, GOAP_SCHEMA, DREAM_SCHEMA, QE_PATTERNS_SCHEMA, QE_PATTERNS_FTS_SCHEMA, QE_PATTERNS_SCHEMA_WITHOUT_FTS, MINCUT_SCHEMA, SONA_PATTERNS_SCHEMA, FEEDBACK_SCHEMA, HYPERGRAPH_SCHEMA, PATTERN_NULLS_SCHEMA, DREAM_INSIGHTS_ARCHIVE_SCHEMA, STATS_TABLES, } from './unified-memory-schemas.js';
 import { ensurePatternFtsInSync } from './pattern-fts-self-heal.js';
 import { applyMigration as applyLearningEvidenceMigration } from '../migrations/20260830_add_learning_evidence_tables.js';
 // CRDT imports for distributed state synchronization
@@ -432,10 +432,20 @@ export class UnifiedMemoryManager {
         this.db.exec(SCHEMA_VERSION_TABLE);
         this.handleV2SchemaIncompatibilities();
         const versionRow = this.db.prepare('SELECT version FROM schema_version WHERE id = 1').get();
-        const currentVersion = versionRow?.version ?? 0;
-        if (currentVersion < SCHEMA_VERSION) {
-            console.log(`[UnifiedMemory] Migrating from v${currentVersion} to v${SCHEMA_VERSION}`);
+        const observedVersion = versionRow?.version ?? 0;
+        if (observedVersion < SCHEMA_VERSION) {
+            console.log(`[UnifiedMemory] Migrating from v${observedVersion} to v${SCHEMA_VERSION}`);
+            // #1116: IMMEDIATE takes the write lock before the first statement, so a
+            // concurrent opener waits on busy_timeout instead of failing with
+            // SQLITE_BUSY when a deferred read transaction has to upgrade (CREATE
+            // INDEX after a CREATE TABLE IF NOT EXISTS that only read). The version is
+            // read again under that lock: a process that waited applies only what the
+            // first one left, instead of re-running steps from a stale version.
             const migrate = this.db.transaction(() => {
+                const lockedRow = this.db.prepare('SELECT version FROM schema_version WHERE id = 1').get();
+                const currentVersion = lockedRow?.version ?? 0;
+                if (currentVersion >= SCHEMA_VERSION)
+                    return currentVersion;
                 if (currentVersion < 1)
                     this.db.exec(KV_STORE_SCHEMA);
                 if (currentVersion < 2)
@@ -472,13 +482,22 @@ export class UnifiedMemoryManager {
                     applyLearningEvidenceMigration(this.db);
                 if (currentVersion < 13)
                     this.migrateToV13EmbeddingSpace();
+                // #1116: additive; a table an earlier sweep created lazily is kept with its rows.
+                if (currentVersion < 14)
+                    this.db.exec(DREAM_INSIGHTS_ARCHIVE_SCHEMA);
                 this.db.prepare(`
           INSERT OR REPLACE INTO schema_version (id, version, migrated_at)
           VALUES (1, ?, datetime('now'))
         `).run(SCHEMA_VERSION);
+                return currentVersion;
             });
-            migrate();
-            console.log(`[UnifiedMemory] Migration complete`);
+            const migratedFrom = migrate.immediate();
+            if (migratedFrom >= SCHEMA_VERSION) {
+                console.log(`[UnifiedMemory] Another process completed the migration to v${SCHEMA_VERSION}`);
+            }
+            else {
+                console.log(`[UnifiedMemory] Migration complete`);
+            }
         }
     }
     /**

package/dist/learning/dream/dream-commit.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Dream commit (#1060): compute, then commit.
+ *
+ * A dream cycle is computed entirely in memory (spreading activation and
+ * insight generation never touch the database). Its results are then written
+ * here in ONE synchronous better-sqlite3 transaction: the activation levels,
+ * the insights and the final cycle row. There is no `await` inside the
+ * transaction, so no foreground write on the shared connection can join it,
+ * and no lock is held while asynchronous work runs. A cancelled, failed or
+ * interrupted dream commits nothing beyond its cycle row.
+ *
+ * Before #1060 the dream wrote its results one awaited statement at a time
+ * (and, with opt-in branching, inside a SAVEPOINT held across the whole
+ * asynchronous dream), so a discard or a shutdown mid-dream rolled back
+ * foreground writes that had already been acknowledged.
+ */
+import type { Database as DatabaseType } from 'better-sqlite3';
+import type { DreamCycle } from './types.js';
+import type { DreamInsight } from './insight-generator.js';
+/** Everything a dream cycle writes, computed before anything is written. */
+export interface ComputedDream {
+    cycle: DreamCycle;
+    /**
+     * Final activation level per concept node (clamped to [0, 1] on write),
+     * with the level the dream loaded. A node whose stored level no longer
+     * equals `loadedLevel` was written by someone else during the dream and is
+     * left alone. Omit `loadedLevel` to write unconditionally.
+     */
+    activations: ReadonlyArray<readonly [nodeId: string, level: number, loadedLevel?: number]>;
+    insights: readonly DreamInsight[];
+}
+export interface DreamCommitResult {
+    /** Wall-clock time the write transaction was open, in milliseconds. */
+    commitDurationMs: number;
+    /** Concept nodes updated (a node deleted meanwhile is skipped, not recreated). */
+    activationsWritten: number;
+    /** Activations skipped: the node vanished or a foreground write changed it. */
+    activationsSkipped: number;
+    insightsWritten: number;
+}
+/**
+ * Write a computed dream in one short synchronous transaction.
+ * Throws (and writes nothing) if any statement fails.
+ */
+export declare function commitDream(db: DatabaseType, dream: ComputedDream): DreamCommitResult;
+/**
+ * Margin added to the longest dream duration before a `running` cycle is
+ * treated as abandoned (covers graph loading and insight generation).
+ * DreamEngine.initialize() sweeps cycles older than
+ * max(config.maxDurationMs, 60 s) + this margin, 11 minutes by default.
+ * Library callers running longer dreams must raise maxDurationMs on every
+ * engine sharing the database (see DreamEngine.dream()).
+ */
+export declare const ABANDONED_CYCLE_GRACE_MS: number;
+/** Error recorded on a cycle the abandoned-cycle sweep marks interrupted. */
+export declare const ABANDONED_CYCLE_ERROR = "abandoned: process exited";
+/**
+ * Mark cycles a dead process left `running` as `interrupted` (#1060). Only a
+ * cycle that started more than `olderThanMs` ago is touched, so pass at least
+ * the longest dream duration plus ABANDONED_CYCLE_GRACE_MS: a cycle still
+ * running in another live process is then never marked. Rows are only
+ * updated, never deleted. Returns the number of cycles marked.
+ */
+export declare function markAbandonedCycles(db: DatabaseType, olderThanMs?: number, now?: number): number;
+//# sourceMappingURL=dream-commit.d.ts.map

package/dist/learning/dream/dream-commit.js

@@ -0,0 +1,104 @@
+/**
+ * Dream commit (#1060): compute, then commit.
+ *
+ * A dream cycle is computed entirely in memory (spreading activation and
+ * insight generation never touch the database). Its results are then written
+ * here in ONE synchronous better-sqlite3 transaction: the activation levels,
+ * the insights and the final cycle row. There is no `await` inside the
+ * transaction, so no foreground write on the shared connection can join it,
+ * and no lock is held while asynchronous work runs. A cancelled, failed or
+ * interrupted dream commits nothing beyond its cycle row.
+ *
+ * Before #1060 the dream wrote its results one awaited statement at a time
+ * (and, with opt-in branching, inside a SAVEPOINT held across the whole
+ * asynchronous dream), so a discard or a shutdown mid-dream rolled back
+ * foreground writes that had already been acknowledged.
+ */
+/**
+ * Write a computed dream in one short synchronous transaction.
+ * Throws (and writes nothing) if any statement fails.
+ */
+export function commitDream(db, dream) {
+    const updateActivation = db.prepare(`
+    UPDATE concept_nodes
+    SET activation_level = ?, last_activated = datetime('now')
+    WHERE id = ? AND (? IS NULL OR COALESCE(activation_level, 0) = ?)
+  `);
+    const insertInsight = db.prepare(`
+    INSERT INTO dream_insights
+    (id, cycle_id, insight_type, source_concepts, description, novelty_score,
+     confidence_score, actionable, applied, suggested_action, pattern_id)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, NULL)
+  `);
+    // Upsert: a dream run with commit:false (SpeculativeDreamer) wrote no
+    // 'running' row up front, so its row is created here.
+    const upsertCycle = db.prepare(`
+    INSERT INTO dream_cycles
+    (id, start_time, end_time, duration_ms, concepts_processed, associations_found,
+     insights_generated, status, error)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      end_time = excluded.end_time, duration_ms = excluded.duration_ms,
+      concepts_processed = excluded.concepts_processed,
+      associations_found = excluded.associations_found,
+      insights_generated = excluded.insights_generated,
+      status = excluded.status, error = excluded.error
+  `);
+    const write = db.transaction(() => {
+        // The cycle row first: insights reference it.
+        const { cycle } = dream;
+        upsertCycle.run(cycle.id, cycle.startTime.toISOString(), cycle.endTime?.toISOString() ?? null, cycle.durationMs ?? null, cycle.conceptsProcessed, cycle.associationsFound, cycle.insightsGenerated, cycle.status, cycle.error ?? null);
+        let activationsWritten = 0;
+        for (const [nodeId, level, loadedLevel] of dream.activations) {
+            // Conflict rule: the foreground wins. A node removed since the dream
+            // loaded it is not recreated, and a node whose level changed since then
+            // is not overwritten; in both cases the UPDATE matches no row. The check
+            // compares values: a foreground write that set the node to exactly the
+            // level the dream loaded is indistinguishable and is overwritten.
+            const expected = loadedLevel ?? null;
+            activationsWritten += updateActivation.run(Math.max(0, Math.min(1, level)), nodeId, expected, expected).changes;
+        }
+        for (const insight of dream.insights) {
+            insertInsight.run(insight.id, insight.cycleId, insight.type, JSON.stringify(insight.sourceConcepts), insight.description, insight.noveltyScore, insight.confidenceScore, insight.actionable ? 1 : 0, insight.suggestedAction ?? null);
+        }
+        return {
+            activationsWritten,
+            activationsSkipped: dream.activations.length - activationsWritten,
+            insightsWritten: dream.insights.length,
+        };
+    });
+    const started = performance.now();
+    // IMMEDIATE takes the write lock up front, so the transaction waits on
+    // busy_timeout instead of failing at the read-to-write upgrade.
+    const counts = write.immediate();
+    return { ...counts, commitDurationMs: performance.now() - started };
+}
+/**
+ * Margin added to the longest dream duration before a `running` cycle is
+ * treated as abandoned (covers graph loading and insight generation).
+ * DreamEngine.initialize() sweeps cycles older than
+ * max(config.maxDurationMs, 60 s) + this margin, 11 minutes by default.
+ * Library callers running longer dreams must raise maxDurationMs on every
+ * engine sharing the database (see DreamEngine.dream()).
+ */
+export const ABANDONED_CYCLE_GRACE_MS = 10 * 60 * 1000;
+/** Error recorded on a cycle the abandoned-cycle sweep marks interrupted. */
+export const ABANDONED_CYCLE_ERROR = 'abandoned: process exited';
+/**
+ * Mark cycles a dead process left `running` as `interrupted` (#1060). Only a
+ * cycle that started more than `olderThanMs` ago is touched, so pass at least
+ * the longest dream duration plus ABANDONED_CYCLE_GRACE_MS: a cycle still
+ * running in another live process is then never marked. Rows are only
+ * updated, never deleted. Returns the number of cycles marked.
+ */
+export function markAbandonedCycles(db, olderThanMs = ABANDONED_CYCLE_GRACE_MS, now = Date.now()) {
+    const cutoff = new Date(now - olderThanMs).toISOString();
+    return db.prepare(`
+    UPDATE dream_cycles
+    SET status = 'interrupted',
+        error = ?,
+        end_time = COALESCE(end_time, ?)
+    WHERE status = 'running' AND start_time < ?
+  `).run(ABANDONED_CYCLE_ERROR, new Date(now).toISOString(), cutoff).changes;
+}
+//# sourceMappingURL=dream-commit.js.map

package/dist/learning/dream/dream-engine.d.ts

@@ -16,7 +16,8 @@
  */
 import { type ActivationConfig } from './spreading-activation.js';
 import { type InsightConfig, type DreamInsight } from './insight-generator.js';
-import { RVCOWBranchManager, type Branch, type ValidationResult, type ValidationThresholds } from './rvcow-branch-manager.js';
+import { type Branch, type ValidationResult, type ValidationThresholds } from './rvcow-branch-manager.js';
+import { type ComputedDream } from './dream-commit.js';
 import type { WitnessChain } from '../../audit/witness-chain.js';
 import type { DreamCycle, PatternImportData, FailureImportData, SuccessImportData } from './types.js';
 /**
@@ -32,19 +33,12 @@ export interface DreamConfig {
     /** Insight generation configuration */
     insightConfig: InsightConfig;
     /**
-     * Enable RVCOW branching for reversible dream cycles (ADR-069).
-     * Default: false (#1060).
-     *
-     * RISK — opt in only for an isolated database connection. The branch is a
-     * SQLite SAVEPOINT on the process-wide shared connection, and it stays open
-     * for the whole asynchronous dream (up to `maxDurationMs`). Every other
-     * write made on that connection in the meantime (memory_store, pattern
-     * updates, experience capture) joins the savepoint. A discard (failed
-     * validation, error, cancel) rolls those foreground writes back even though
-     * they were already acknowledged. Branch validation only inspects
-     * `qe_patterns`, which `dream()` does not write, so branching adds little
-     * protection to a dream cycle. A compute-then-commit rework is tracked in
-     * #1060.
+     * @deprecated #1060: ignored. A dream is computed in memory and its results
+     * are committed in one short synchronous transaction (see dream-commit.ts),
+     * which gives the atomicity branching was meant to provide without holding
+     * a SAVEPOINT on the shared connection across the asynchronous dream (that
+     * rolled back acknowledged foreground writes). Setting it to true logs a
+     * warning and has no other effect. Kept so existing configs still load.
      */
     enableBranching: boolean;
     /** Validation thresholds for dream branch quality checks */
@@ -70,6 +64,30 @@ export interface DreamCycleResult {
     };
     /** Number of patterns created from insights */
     patternsCreated: number;
+    /** #1060: how long the single commit transaction held the write lock, in ms. */
+    commitDurationMs?: number;
+    /**
+     * #1060: the uncommitted results of a `dream(…, { commit: false })` run.
+     * Pass the result to DreamEngine.commitDreamResult() to write them.
+     */
+    pending?: ComputedDream;
+}
+/** #1060: per-call options for DreamEngine.dream(). */
+export interface DreamRunOptions {
+    /** Minimum concepts required for this run (default: config.minConceptsRequired). */
+    minConceptsRequired?: number;
+    /** Activation parameter overrides for this run (SpeculativeDreamer strategies). */
+    activationConfig?: Partial<ActivationConfig>;
+    /**
+     * Write the results (default true). With false the dream writes nothing at
+     * all, not even its cycle row, and returns them in `pending`.
+     */
+    commit?: boolean;
+}
+/** #1060: a second dream on the same engine while one is running. */
+export declare class DreamAlreadyRunningError extends Error {
+    readonly cycleId: string;
+    constructor(cycleId: string);
 }
 /**
  * Result of applying an insight
@@ -117,19 +135,25 @@ export declare class DreamEngine {
     private persistence;
     private graph;
     private db;
-    private branchManager;
     /** #1060: marks the in-flight dream cycle interrupted (set while dreaming). */
     private interruptActiveDream;
+    /** #1060: the in-flight dream, so close() can wait for it to stop. */
+    private activeDream;
+    /** #1060: stops the in-flight dream's activation loop. */
+    private activeAbort;
     private currentCycle;
     private initialized;
     private cancelled;
+    /** #1060: how many times cancelDream() was called (see cancelCount). */
+    private cancelRequests;
+    /** #1060: set by close() until the engine is initialized again. */
+    private closing;
     private branchEventListeners;
     /** Optional witness chain for audit trail of dream decisions (ADR-070) */
     private _witnessChain;
     set witnessChain(wc: WitnessChain | null);
-    /** Optional RVF adapter for COW branching (ADR-069). Set externally to share with PatternStore. */
-    private _rvfAdapter;
-    set rvfAdapter(adapter: any);
+    /** @deprecated #1060: branching is ignored, so the RVF adapter is unused. */
+    set rvfAdapter(_adapter: any);
     constructor(config?: Partial<DreamConfig>);
     /**
      * Initialize using unified persistence
@@ -150,17 +174,41 @@ export declare class DreamEngine {
      * 4. Generates insights from activation patterns
      * 5. Persists the cycle and insights
      *
+     * The cycle row is written as 'running' when the dream starts. Another
+     * process's DreamEngine.initialize() marks a cycle still 'running' after
+     * max(its config.maxDurationMs, 60 s) + ABANDONED_CYCLE_GRACE_MS (10 min),
+     * i.e. 11 minutes by default, as interrupted ('abandoned: process exited').
+     * The MCP tool and the CLI cap dreams at 60 s. A library caller that runs
+     * dreams longer than about 10 minutes (graph loading included) should give
+     * every engine sharing the database a maxDurationMs at least that long, or
+     * a live cycle can be marked interrupted. Its commit still records the
+     * final status, and nothing is deleted.
+     *
      * @param durationMs - Maximum duration in milliseconds (default: config.maxDurationMs)
      * @returns Dream cycle result with insights and statistics
      */
-    dream(durationMs?: number): Promise<DreamCycleResult>;
+    dream(durationMs?: number, options?: DreamRunOptions): Promise<DreamCycleResult>;
+    /**
+     * #1060: compute, then commit. Everything up to the commit reads the
+     * database or works in memory; the results are written by commitDream() in
+     * one synchronous transaction, so no lock is held across an await and no
+     * foreground write can join the dream's writes or be rolled back with them.
+     */
+    private runDream;
     /**
-     * #1060: build a synchronous handler that records a still-running cycle as
-     * 'interrupted'. It runs from close() and from a process store closer,
-     * which runs before the shared database closes (closers run newest first).
-     * When opt-in branching left a savepoint open, the savepoint is released
-     * first so foreground writes made on the shared connection during the
-     * dream are committed rather than lost with the connection.
+     * #1060: write the results of a `dream(…, { commit: false })` run, in the
+     * same single transaction a normal dream uses. Returns the commit duration
+     * in ms. Throws (and writes nothing) if the engine is closed or the write
+     * fails.
+     */
+    commitDreamResult(result: DreamCycleResult): number;
+    /** Commit a computed dream, then record it in the witness chain (#1060). */
+    private commitComputed;
+    /**
+     * #1060: build a synchronous handler that stops a still-running dream and
+     * records its cycle as 'interrupted'. It runs from close() and from a
+     * process store closer, which runs before the shared database closes
+     * (closers run newest first). Nothing of the dream is committed afterwards.
      */
     private createCycleInterrupter;
     /**
@@ -214,6 +262,11 @@ export declare class DreamEngine {
      * @returns Array of dream cycles
      */
     getDreamHistory(limit?: number): Promise<DreamCycle[]>;
+    /**
+     * #1060: number of cancelDream() calls so far. A caller that runs several
+     * dreams in a row (SpeculativeDreamer) compares it to stop after a cancel.
+     */
+    get cancelCount(): number;
     /**
      * Cancel the running dream cycle.
      */
@@ -227,19 +280,16 @@ export declare class DreamEngine {
      */
     getCurrentCycle(): DreamCycle | null;
     private saveCycle;
-    private updateCycle;
     private updateCycleSync;
-    private saveInsight;
     private getInsightRowById;
     /**
-     * Get the RVCOW branch manager, if branching is enabled.
+     * @deprecated #1060: branching is ignored; always null.
      */
-    getBranchManager(): RVCOWBranchManager | null;
+    getBranchManager(): null;
     /**
-     * Register a listener for branch lifecycle events.
+     * @deprecated #1060: branching is ignored, so no branch event is emitted.
      */
     onBranchEvent(listener: (event: string, branch: Branch, detail?: ValidationResult) => void): void;
-    private emitBranchEvent;
     /**
      * Release resources (does NOT close the shared database)
      */