agent-working-memory 0.7.17 → 0.8.5

package/src/engine/connections.ts

@@ -1,103 +1,162 @@
-// Copyright 2026 Robert Winter / Complete Ideas
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Connection Engine — discovers links between memories.
- *
- * Runs asynchronously. When a new engram is written, the connection
- * engine checks it against existing memories and forms association
- * edges where resonance exceeds a threshold.
- *
- * Connection memories are first-class engrams — they can themselves
- * activate and form higher-order connections, producing emergent
- * associative structure over time.
- */
-
-import type { EngramStore } from '../storage/sqlite.js';
-import type { ActivationEngine } from './activation.js';
-import type { Engram } from '../types/index.js';
-
-export class ConnectionEngine {
-  private store: EngramStore;
-  private engine: ActivationEngine;
-  private threshold: number;
-  private queue: string[] = [];
-  private processing = false;
-
-  constructor(
-    store: EngramStore,
-    engine: ActivationEngine,
-    threshold: number = 0.7
-  ) {
-    this.store = store;
-    this.engine = engine;
-    this.threshold = threshold;
-  }
-
-  /**
-   * Queue a newly written engram for connection discovery.
-   */
-  enqueue(engramId: string): void {
-    this.queue.push(engramId);
-    if (!this.processing) {
-      this.processQueue();
-    }
-  }
-
-  private async processQueue(): Promise<void> {
-    this.processing = true;
-
-    while (this.queue.length > 0) {
-      const engramId = this.queue.shift()!;
-      const engram = this.store.getEngram(engramId);
-      if (!engram || engram.stage !== 'active') continue;
-
-      try {
-        await this.findConnections(engram);
-      } catch {
-        // Connection discovery is best-effort — don't crash the server
-      }
-    }
-
-    this.processing = false;
-  }
-
-  /**
-   * Find and create connections for a given engram.
-   */
-  private async findConnections(engram: Engram): Promise<void> {
-    const results = await this.engine.activate({
-      agentId: engram.agentId,
-      context: `${engram.concept} ${engram.content}`,
-      limit: 5,
-      minScore: this.threshold,
-      internal: true,
-    });
-
-    // Filter out self and already-connected engrams
-    const existing = this.store.getAssociationsFor(engram.id);
-    const existingIds = new Set(existing.map(a =>
-      a.fromEngramId === engram.id ? a.toEngramId : a.fromEngramId
-    ));
-
-    for (const result of results) {
-      if (result.engram.id === engram.id) continue;
-      if (existingIds.has(result.engram.id)) continue;
-
-      // Create a connection association
-      this.store.upsertAssociation(
-        engram.id,
-        result.engram.id,
-        result.score,
-        'connection'
-      );
-
-      // Bidirectional
-      this.store.upsertAssociation(
-        result.engram.id,
-        engram.id,
-        result.score,
-        'connection'
-      );
-    }
-  }
-}
+// Copyright 2026 Robert Winter / Complete Ideas
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Connection Engine — discovers links between memories.
+ *
+ * **Lifecycle (v0.8.2):** `enqueue()` just appends to an in-memory queue and
+ * returns. The queue is drained by `processQueue()`, which is called from
+ * the consolidation cycle. Per-write inline drain was removed because each
+ * `findConnections` call runs a full activation cycle (embed + BM25 + vector
+ * + rerank) — ~200-500 ms of event-loop blocking per write, queued ahead
+ * of subsequent requests under load.
+ *
+ * Cold-start exception: when the agent has fewer than
+ * `AWM_CONNECTION_COLD_START_THRESHOLD` (default 10) active engrams, callers
+ * can opt into inline drain via `enqueueAndMaybeFlush()` so the first few
+ * writes still produce a useful association graph before the next
+ * consolidation cycle fires. Once the pool grows past the threshold, all
+ * discovery defers to consolidation regardless.
+ *
+ * Footprint: when AWM is idle and no consolidation is running, the queue
+ * is a plain `string[]` — no timers, no background work, ~24 bytes per
+ * queued ID. AWM remains cheap to NOT use.
+ */
+
+import type { IEngramStore as EngramStore } from '../storage/store.js';
+import type { ActivationEngine } from './activation.js';
+import type { Engram } from '../types/index.js';
+
+const COLD_START_THRESHOLD = Number(process.env.AWM_CONNECTION_COLD_START_THRESHOLD ?? 10);
+
+export class ConnectionEngine {
+  private store: EngramStore;
+  private engine: ActivationEngine;
+  private threshold: number;
+  private queue: string[] = [];
+  private processing = false;
+
+  constructor(
+    store: EngramStore,
+    engine: ActivationEngine,
+    threshold: number = 0.7
+  ) {
+    this.store = store;
+    this.engine = engine;
+    this.threshold = threshold;
+  }
+
+  /**
+   * Queue a newly written engram for connection discovery.
+   *
+   * Synchronous and non-triggering. The queue is drained later by:
+   *   - `processQueue()` called from the consolidation cycle, or
+   *   - `enqueueAndMaybeFlush()` for cold-start inline drain.
+   */
+  enqueue(engramId: string): void {
+    this.queue.push(engramId);
+  }
+
+  /**
+   * Queue + opportunistic inline drain for cold-start agents.
+   *
+   * If the agent has fewer than `AWM_CONNECTION_COLD_START_THRESHOLD`
+   * active engrams (default 10), drain the queue inline so the first few
+   * writes produce a useful association graph before consolidation runs.
+   * Once the pool grows past the threshold, this falls back to deferred
+   * (consolidation-driven) drain.
+   *
+   * Returns immediately — the inline drain runs as a fire-and-forget
+   * background task so the calling write doesn't block on it.
+   */
+  enqueueAndMaybeFlush(engramId: string, agentId: string): void {
+    this.queue.push(engramId);
+    if (this.processing) return;
+    void this.maybeDrainColdStart(agentId);
+  }
+
+  private async maybeDrainColdStart(agentId: string): Promise<void> {
+    try {
+      const count = await this.store.getActiveCount(agentId);
+      if (count < COLD_START_THRESHOLD) {
+        await this.processQueue();
+      }
+    } catch {
+      // Cold-start drain is best-effort. The next consolidation cycle
+      // will drain whatever stayed queued.
+    }
+  }
+
+  /**
+   * Drain the queue: run connection discovery for every queued engram.
+   *
+   * Called from the consolidation cycle (`ConsolidationEngine.consolidate`)
+   * at the start of each run, and from `enqueueAndMaybeFlush()` for
+   * cold-start agents. Reentrant-safe via the `processing` flag.
+   *
+   * Exposed publicly so callers (consolidation, tests) can explicitly drain.
+   */
+  async processQueue(): Promise<void> {
+    if (this.processing) return; // Reentrancy guard
+    this.processing = true;
+    try {
+      while (this.queue.length > 0) {
+        const engramId = this.queue.shift()!;
+        const engram = await this.store.getEngram(engramId);
+        if (!engram || engram.stage !== 'active') continue;
+
+        try {
+          await this.findConnections(engram);
+        } catch {
+          // Connection discovery is best-effort — don't crash the server
+        }
+      }
+    } finally {
+      this.processing = false;
+    }
+  }
+
+  /** Number of engrams currently queued for connection discovery. */
+  queueSize(): number {
+    return this.queue.length;
+  }
+
+  /**
+   * Find and create connections for a given engram.
+   */
+  private async findConnections(engram: Engram): Promise<void> {
+    const results = await this.engine.activate({
+      agentId: engram.agentId,
+      context: `${engram.concept} ${engram.content}`,
+      limit: 5,
+      minScore: this.threshold,
+      internal: true,
+    });
+
+    // Filter out self and already-connected engrams
+    const existing = await this.store.getAssociationsFor(engram.id);
+    const existingIds = new Set(existing.map(a =>
+      a.fromEngramId === engram.id ? a.toEngramId : a.fromEngramId
+    ));
+
+    for (const result of results) {
+      if (result.engram.id === engram.id) continue;
+      if (existingIds.has(result.engram.id)) continue;
+
+      // Create a connection association
+      await this.store.upsertAssociation(
+        engram.id,
+        result.engram.id,
+        result.score,
+        'connection'
+      );
+
+      // Bidirectional
+      await this.store.upsertAssociation(
+        result.engram.id,
+        engram.id,
+        result.score,
+        'connection'
+      );
+    }
+  }
+}

package/src/engine/consolidation-scheduler.ts

@@ -1,125 +1,242 @@
-// Copyright 2026 Robert Winter / Complete Ideas
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Consolidation Scheduler — automatically triggers sleep cycles.
- *
- * Four triggers:
- *   1. Idle — agent inactive >10min → full consolidation
- *   2. Volume — 50+ writes since last consolidation → full consolidation
- *   3. Time — 30min since last consolidation → full consolidation
- *   4. Adaptive — retrieval precision <0.4 → full consolidation
- *
- * Also provides mini-consolidation for restore (fire-and-forget, lightweight).
- * Checks every 30 seconds across all active agents.
- */
-
-import type { EngramStore } from '../storage/sqlite.js';
-import type { ConsolidationEngine } from './consolidation.js';
-
-const TICK_INTERVAL_MS = 30_000;         // Check every 30s
-const IDLE_THRESHOLD_MS = 10 * 60_000;   // 10 minutes
-const VOLUME_THRESHOLD = 50;             // 50 writes
-const TIME_THRESHOLD_MS = 30 * 60_000;   // 30 minutes
-const PRECISION_THRESHOLD = 0.4;         // Below this triggers consolidation
-
-export class ConsolidationScheduler {
-  private timer: ReturnType<typeof setInterval> | null = null;
-  private running = false;
-
-  constructor(
-    private store: EngramStore,
-    private consolidationEngine: ConsolidationEngine,
-  ) {}
-
-  start(): void {
-    if (this.timer) return;
-    this.timer = setInterval(() => this.tick(), TICK_INTERVAL_MS);
-    console.log('ConsolidationScheduler started (30s tick)');
-  }
-
-  stop(): void {
-    if (this.timer) {
-      clearInterval(this.timer);
-      this.timer = null;
-    }
-    console.log('ConsolidationScheduler stopped');
-  }
-
-  /**
-   * Mini-consolidation — lightweight, called from restore path.
-   * Only runs replay + strengthen (phases 1-2), skips heavy phases.
-   */
-  async runMiniConsolidation(agentId: string): Promise<void> {
-    if (this.running) return;
-    this.running = true;
-    try {
-      console.log(`[scheduler] mini-consolidation for ${agentId}`);
-      await this.consolidationEngine.consolidate(agentId);
-      this.store.markConsolidation(agentId, true);
-    } catch (err) {
-      console.error(`[scheduler] mini-consolidation failed for ${agentId}:`, err);
-    } finally {
-      this.running = false;
-    }
-  }
-
-  private tick(): void {
-    if (this.running) return;
-
-    const agents = this.store.getActiveAgents();
-    const now = Date.now();
-
-    for (const agent of agents) {
-      const idleMs = now - agent.lastActivityAt.getTime();
-      const sinceConsolidation = agent.lastConsolidationAt
-        ? now - agent.lastConsolidationAt.getTime()
-        : Infinity;
-
-      let trigger: string | null = null;
-
-      // 1. Idle trigger — agent stopped writing/recalling >10min ago
-      if (idleMs > IDLE_THRESHOLD_MS && sinceConsolidation > IDLE_THRESHOLD_MS) {
-        trigger = `idle (${Math.round(idleMs / 60_000)}min)`;
-      }
-
-      // 2. Volume trigger — many writes accumulated
-      if (!trigger && agent.writeCount >= VOLUME_THRESHOLD) {
-        trigger = `volume (${agent.writeCount} writes)`;
-      }
-
-      // 3. Time trigger — been too long since last consolidation
-      if (!trigger && sinceConsolidation > TIME_THRESHOLD_MS) {
-        trigger = `time (${Math.round(sinceConsolidation / 60_000)}min)`;
-      }
-
-      // 4. Adaptive trigger — precision is low
-      if (!trigger) {
-        try {
-          const precision = this.store.getRetrievalPrecision(agent.agentId, 1);
-          if (precision > 0 && precision < PRECISION_THRESHOLD) {
-            trigger = `adaptive (precision ${(precision * 100).toFixed(0)}%)`;
-          }
-        } catch { /* precision check is non-fatal */ }
-      }
-
-      if (trigger) {
-        this.runFullConsolidation(agent.agentId, trigger);
-        return; // One consolidation per tick to avoid overload
-      }
-    }
-  }
-
-  private async runFullConsolidation(agentId: string, reason: string): Promise<void> {
-    this.running = true;
-    try {
-      console.log(`[scheduler] full consolidation for ${agentId} — trigger: ${reason}`);
-      const result = await this.consolidationEngine.consolidate(agentId);
-      this.store.markConsolidation(agentId, false);
-      console.log(`[scheduler] consolidation done: ${result.edgesStrengthened} strengthened, ${result.memoriesForgotten} forgotten`);
-    } catch (err) {
-      console.error(`[scheduler] consolidation failed for ${agentId}:`, err);
-    } finally {
-      this.running = false;
-    }
-  }
-}
+// Copyright 2026 Robert Winter / Complete Ideas
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Consolidation Scheduler - sleep-only consolidation (AWM 0.8.x).
+ *
+ * Two triggers, both modeled on biological sleep (offline consolidation, not in-band):
+ *
+ *   1. Cron - fires at a configured time (default: 0 3 * * * = 3 AM local time).
+ *      Configurable via AWM_CONSOLIDATION_CRON env var.
+ *
+ *   2. Quiescence - fires when ALL active agents have been idle >30 min.
+ *      "Truly asleep" - no agent is currently writing or recalling.
+ *
+ * Kill switch: AWM_DISABLE_SCHEDULER=1 skips both triggers. Manual
+ * consolidation via POST /system/consolidate still works.
+ *
+ * Removed in 2.0: in-band idle/volume/time/precision triggers that fired
+ * during active hours and blocked HTTP.
+ *
+ * Tick granularity: 1 minute (sufficient for cron-at-the-minute precision
+ * and quiescence checks at human timescales).
+ */
+
+import type { IEngramStore as EngramStore } from '../storage/store.js';
+import type { ConsolidationEngine } from './consolidation.js';
+
+const TICK_INTERVAL_MS = 60_000;             // Check every 60s
+const QUIESCENCE_THRESHOLD_MS = 30 * 60_000; // 30 minutes
+const DEFAULT_CRON = '0 3 * * *';            // 3 AM local time daily
+
+// --- Cron matcher (hand-rolled, minute-granularity) ---
+
+/**
+ * Parse a single cron field into a Set of valid integer values.
+ * Supports: wildcard, literal value, range A-B, list A,B,C, step A-B/N.
+ */
+function parseCronField(field: string, min: number, max: number): Set<number> {
+  const result = new Set<number>();
+  for (const part of field.split(',')) {
+    // Handle step: <range>/N
+    const stepMatch = part.match(/^(.+?)\/(\d+)$/);
+    if (stepMatch) {
+      const range = stepMatch[1];
+      const step = parseInt(stepMatch[2], 10);
+      if (step <= 0) continue;
+      const [lo, hi] = range === '*'
+        ? [min, max]
+        : range.includes('-')
+          ? range.split('-').map(n => parseInt(n, 10)) as [number, number]
+          : [parseInt(range, 10), max];
+      for (let n = lo; n <= hi; n += step) {
+        if (n >= min && n <= max) result.add(n);
+      }
+      continue;
+    }
+    // Range: A-B
+    if (part.includes('-')) {
+      const [lo, hi] = part.split('-').map(n => parseInt(n, 10));
+      for (let n = lo; n <= hi; n++) {
+        if (n >= min && n <= max) result.add(n);
+      }
+      continue;
+    }
+    // Wildcard
+    if (part === '*') {
+      for (let n = min; n <= max; n++) result.add(n);
+      continue;
+    }
+    // Single value
+    const n = parseInt(part, 10);
+    if (!Number.isNaN(n) && n >= min && n <= max) result.add(n);
+  }
+  return result;
+}
+
+/**
+ * Return true if `now` matches the cron expression at minute granularity.
+ * Format: "minute hour dayOfMonth month dayOfWeek" (5 fields, space-separated).
+ * Day-of-week: 0-6 (0 = Sunday). 7 is also accepted as Sunday alias.
+ */
+export function cronMatches(now: Date, expr: string): boolean {
+  const fields = expr.trim().split(/\s+/);
+  if (fields.length !== 5) return false;
+
+  const minutes = parseCronField(fields[0], 0, 59);
+  const hours = parseCronField(fields[1], 0, 23);
+  const daysOfMonth = parseCronField(fields[2], 1, 31);
+  const months = parseCronField(fields[3], 1, 12);
+  const daysOfWeekRaw = parseCronField(fields[4], 0, 7);
+  // Normalize: 7 == 0 (Sunday)
+  const daysOfWeek = new Set<number>();
+  for (const d of daysOfWeekRaw) daysOfWeek.add(d === 7 ? 0 : d);
+
+  return (
+    minutes.has(now.getMinutes()) &&
+    hours.has(now.getHours()) &&
+    daysOfMonth.has(now.getDate()) &&
+    months.has(now.getMonth() + 1) &&
+    daysOfWeek.has(now.getDay())
+  );
+}
+
+// --- Scheduler ---
+
+export class ConsolidationScheduler {
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private running = false;
+  private readonly cronExpr: string;
+  private readonly disabled: boolean;
+  private lastCronTriggeredMinute: string | null = null;
+
+  constructor(
+    private store: EngramStore,
+    private consolidationEngine: ConsolidationEngine,
+  ) {
+    this.cronExpr = process.env.AWM_CONSOLIDATION_CRON ?? DEFAULT_CRON;
+    this.disabled = process.env.AWM_DISABLE_SCHEDULER === '1' || process.env.AWM_DISABLE_SCHEDULER === 'true';
+  }
+
+  start(): void {
+    if (this.timer) return;
+    if (this.disabled) {
+      console.log('ConsolidationScheduler disabled (AWM_DISABLE_SCHEDULER=1)');
+      return;
+    }
+    this.timer = setInterval(() => this.tick(), TICK_INTERVAL_MS);
+    console.log(`ConsolidationScheduler started - cron='${this.cronExpr}', quiescence-gate=${QUIESCENCE_THRESHOLD_MS / 60_000}min`);
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    console.log('ConsolidationScheduler stopped');
+  }
+
+  /** True if the scheduler is currently running a consolidation cycle. */
+  isRunning(): boolean {
+    return this.running;
+  }
+
+  /** True if the scheduler's automatic triggers are disabled (kill switch). */
+  isDisabled(): boolean {
+    return this.disabled;
+  }
+
+  /**
+   * Mini-consolidation - lightweight, called from restore path.
+   * Only runs replay + strengthen (phases 1-2), skips heavy phases.
+   */
+  async runMiniConsolidation(agentId: string): Promise<void> {
+    if (this.running) return;
+    this.running = true;
+    try {
+      console.log(`[scheduler] mini-consolidation for ${agentId}`);
+      await this.consolidationEngine.consolidate(agentId);
+      await this.store.markConsolidation(agentId, true);
+    } catch (err) {
+      console.error(`[scheduler] mini-consolidation failed for ${agentId}:`, err);
+    } finally {
+      this.running = false;
+    }
+  }
+
+  /**
+   * Tick handler - checks both triggers. Cron first (planned), then quiescence (opportunistic).
+   * Fires consolidation for at most one agent per tick to avoid overload.
+   */
+  private async tick(): Promise<void> {
+    if (this.running) return;
+    const now = new Date();
+
+    // Trigger 1: cron
+    if (cronMatches(now, this.cronExpr)) {
+      const minuteKey = `${now.getFullYear()}-${now.getMonth()}-${now.getDate()}T${now.getHours()}:${now.getMinutes()}`;
+      if (this.lastCronTriggeredMinute !== minuteKey) {
+        this.lastCronTriggeredMinute = minuteKey;
+        const agent = await this.pickAgent();
+        if (agent) {
+          this.runFullConsolidation(agent.agentId, `cron (${this.cronExpr})`);
+          return;
+        }
+      }
+    }
+
+    // Trigger 2: quiescence
+    if (await this.isQuiescent(now)) {
+      const agent = await this.pickAgent();
+      if (agent) {
+        this.runFullConsolidation(agent.agentId, `quiescence (>${QUIESCENCE_THRESHOLD_MS / 60_000}min idle, all agents)`);
+        return;
+      }
+    }
+  }
+
+  /**
+   * Pick the agent that benefits most from consolidation:
+   * highest writeCount since last consolidation, tie-break by oldest consolidation.
+   */
+  private async pickAgent(): Promise<{ agentId: string } | null> {
+    const agents = await this.store.getActiveAgents();
+    if (agents.length === 0) return null;
+    const sorted = [...agents].sort((a, b) => {
+      if (b.writeCount !== a.writeCount) return b.writeCount - a.writeCount;
+      const aT = a.lastConsolidationAt?.getTime() ?? 0;
+      const bT = b.lastConsolidationAt?.getTime() ?? 0;
+      return aT - bT;
+    });
+    return sorted[0];
+  }
+
+  /**
+   * Quiescence check: ALL active agents must have lastActivityAt > threshold ago.
+   * `lastActivityAt` is updated on both writes and recalls so this captures both.
+   * If no active agents, the system is trivially quiescent.
+   */
+  private async isQuiescent(now: Date): Promise<boolean> {
+    const agents = await this.store.getActiveAgents();
+    if (agents.length === 0) return true;
+    const nowMs = now.getTime();
+    for (const agent of agents) {
+      const idleMs = nowMs - agent.lastActivityAt.getTime();
+      if (idleMs < QUIESCENCE_THRESHOLD_MS) return false;
+    }
+    return true;
+  }
+
+  private async runFullConsolidation(agentId: string, reason: string): Promise<void> {
+    this.running = true;
+    try {
+      console.log(`[scheduler] full consolidation for ${agentId} - trigger: ${reason}`);
+      const result = await this.consolidationEngine.consolidate(agentId);
+      await this.store.markConsolidation(agentId, false);
+      console.log(`[scheduler] consolidation done: ${result.edgesStrengthened} strengthened, ${result.memoriesForgotten} forgotten`);
+    } catch (err) {
+      console.error(`[scheduler] consolidation failed for ${agentId}:`, err);
+    } finally {
+      this.running = false;
+    }
+  }
+}