agent-working-memory 0.7.17 → 0.8.5

package/src/coordination/plugin.ts

@@ -1,44 +1,44 @@
-// Copyright 2026 Robert Winter / Complete Ideas
-// SPDX-License-Identifier: Apache-2.0
-/**
- * AWM Plugin contract.
- *
- * Plugins extend coordination with custom behavior by subscribing to
- * events, adding routes, or querying the DB. They are loaded at startup
- * via AWM_PLUGINS env var (comma-separated module paths).
- *
- * Example plugin:
- *   import type { AWMPlugin } from 'agent-working-memory';
- *   export default {
- *     name: 'my-plugin',
- *     register(ctx) {
- *       ctx.events.on('assignment.completed', (evt) => {
- *         console.log(`Task done: ${evt.assignmentId}`);
- *       });
- *     },
- *   } satisfies AWMPlugin;
- */
-
-import type { FastifyInstance } from 'fastify';
-import type Database from 'better-sqlite3';
-import type { CoordinationEventBus } from './events.js';
-
-/** Context passed to plugin register(). */
-export interface AWMPluginContext {
-  /** Typed event bus — subscribe to coordination events. */
-  events: CoordinationEventBus;
-  /** Raw better-sqlite3 database handle (coordination tables). */
-  db: Database.Database;
-  /** Fastify instance — add custom routes if needed. */
-  fastify: FastifyInstance;
-}
-
-/** Plugin contract. Every AWM plugin must export this shape. */
-export interface AWMPlugin {
-  /** Unique plugin name (for logging and dedup). */
-  name: string;
-  /** Called once at startup with the plugin context. */
-  register(ctx: AWMPluginContext): void | Promise<void>;
-  /** Optional cleanup on shutdown. */
-  teardown?(): void | Promise<void>;
-}
+// Copyright 2026 Robert Winter / Complete Ideas
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * AWM Plugin contract.
+ *
+ * Plugins extend coordination with custom behavior by subscribing to
+ * events, adding routes, or querying the DB. They are loaded at startup
+ * via AWM_PLUGINS env var (comma-separated module paths).
+ *
+ * Example plugin:
+ *   import type { AWMPlugin } from 'agent-working-memory';
+ *   export default {
+ *     name: 'my-plugin',
+ *     register(ctx) {
+ *       ctx.events.on('assignment.completed', (evt) => {
+ *         console.log(`Task done: ${evt.assignmentId}`);
+ *       });
+ *     },
+ *   } satisfies AWMPlugin;
+ */
+
+import type { FastifyInstance } from 'fastify';
+import type Database from 'better-sqlite3';
+import type { CoordinationEventBus } from './events.js';
+
+/** Context passed to plugin register(). */
+export interface AWMPluginContext {
+  /** Typed event bus — subscribe to coordination events. */
+  events: CoordinationEventBus;
+  /** Raw better-sqlite3 database handle (coordination tables). */
+  db: Database.Database;
+  /** Fastify instance — add custom routes if needed. */
+  fastify: FastifyInstance;
+}
+
+/** Plugin contract. Every AWM plugin must export this shape. */
+export interface AWMPlugin {
+  /** Unique plugin name (for logging and dedup). */
+  name: string;
+  /** Called once at startup with the plugin context. */
+  register(ctx: AWMPluginContext): void | Promise<void>;
+  /** Optional cleanup on shutdown. */
+  teardown?(): void | Promise<void>;
+}

package/src/coordination/routes.ts

@@ -21,7 +21,9 @@ import {
   agentIdParamSchema, timelineQuerySchema,
   channelRegisterSchema, channelDeregisterSchema, channelPushSchema,
 } from './schemas.js';
-import { detectStale, cleanupStale } from './stale.js';
+import { detectStale, cleanupStale, retryOrFailAssignment } from './stale.js';
+import { classifyFailure, FailureMode } from './failure-modes.js';
+import { recordSuccess, recordFailure as circuitRecordFailure, isAvailable } from './circuit-breaker.js';
 
 /** Pretty timestamp for coordination logs. */
 function ts(): string {
@@ -444,6 +446,11 @@ export function registerCoordinationRoutes(app: FastifyInstance, db: Database.Da
             `SELECT * FROM coord_assignments WHERE status = 'pending' AND agent_id IS NULL ${blockedFilter} ORDER BY priority DESC, created_at ASC LIMIT 1`
           ).get() as { id: string } | undefined);
 
+      // Circuit breaker: refuse assignment if worker is in open state
+      if (pending && !isAvailable(db, agentId)) {
+        return reply.code(423).send({ status: 'idle', assignment: null, circuit_open: true, reason: 'circuit_open' });
+      }
+
       if (pending) {
         const claimed = db.prepare(
           `UPDATE coord_assignments SET agent_id = ?, status = 'assigned', started_at = datetime('now') WHERE id = ? AND status = 'pending'`
@@ -724,6 +731,11 @@ export function registerCoordinationRoutes(app: FastifyInstance, db: Database.Da
           `SELECT * FROM coord_assignments WHERE status = 'pending' AND agent_id IS NULL ${blockedFilter} ORDER BY priority DESC, created_at ASC LIMIT 1`
         ).get() as { id: string } | undefined);
 
+    // Circuit breaker: refuse assignment if worker is in open state
+    if (pending && !isAvailable(db, agentId)) {
+      return reply.code(423).send({ assignment: null, circuit_open: true, reason: 'circuit_open' });
+    }
+
     if (pending) {
       const claimed = db.prepare(
         `UPDATE coord_assignments SET agent_id = ?, status = 'assigned', started_at = datetime('now') WHERE id = ? AND status = 'pending'`
@@ -847,6 +859,12 @@ export function registerCoordinationRoutes(app: FastifyInstance, db: Database.Da
       coordLog(`${assignInfo?.agent_name ?? 'unknown'} ${status}: ${assignInfo?.task?.slice(0, 80) ?? id}`);
     }
 
+    // Circuit breaker: track success/failure per worker
+    if (assignInfo?.agent_id) {
+      if (status === 'completed') recordSuccess(db, assignInfo.agent_id);
+      else if (status === 'failed') circuitRecordFailure(db, assignInfo.agent_id);
+    }
+
     // Emit events
     eventBus?.emit('assignment.updated', { assignmentId: id, agentId: assignInfo?.agent_id ?? null, status, result });
     if (status === 'completed') {
@@ -1011,6 +1029,38 @@ export function registerCoordinationRoutes(app: FastifyInstance, db: Database.Da
     return reply.send({ ok: true });
   });
 
+  /** POST /assignment/:id/fail — Worker voluntarily fails an assignment with retry logic.
+   *  Body: { result: string, mode?: FailureMode }
+   *  Returns: { outcome: 'retried' | 'failed', attempt_count, last_failure_mode }
+   */
+  app.post('/assignment/:id/fail', async (req, reply) => {
+    const { id } = assignmentIdParamSchema.parse(req.params);
+    const body = req.body as { result?: string; mode?: string } | undefined;
+    const result = body?.result ?? 'worker-initiated failure';
+    const mode = body?.mode as FailureMode | undefined;
+
+    const row = db.prepare(
+      `SELECT id, agent_id, status FROM coord_assignments WHERE id = ?`
+    ).get(id) as { id: string; agent_id: string | null; status: string } | undefined;
+
+    if (!row) return reply.code(404).send({ error: 'assignment not found' });
+    if (['completed', 'failed'].includes(row.status)) {
+      return reply.code(400).send({ error: `cannot fail a ${row.status} assignment` });
+    }
+
+    const agentId = row.agent_id ?? 'unknown';
+    const outcome = retryOrFailAssignment(db, id, agentId, result, mode);
+
+    // Circuit breaker: record failure on voluntary fail
+    if (agentId !== 'unknown') circuitRecordFailure(db, agentId);
+
+    const updated = db.prepare(
+      `SELECT attempt_count, last_failure_mode FROM coord_assignments WHERE id = ?`
+    ).get(id) as { attempt_count: number; last_failure_mode: string | null } | undefined;
+
+    return reply.send({ ok: true, outcome, attempt_count: updated?.attempt_count ?? 0, last_failure_mode: updated?.last_failure_mode ?? null });
+  });
+
   app.put('/assignment/:id', async (req, reply) => {
     const { id } = assignmentIdParamSchema.parse(req.params);
     const parsed = assignmentUpdateSchema.safeParse(req.body);
@@ -1961,6 +2011,11 @@ export function registerCoordinationRoutes(app: FastifyInstance, db: Database.Da
     const agent = db.prepare(`SELECT name, workspace FROM coord_agents WHERE id = ?`).get(agentId) as { name: string; workspace: string | null } | undefined;
     if (!agent) return reply.status(404).send({ error: 'Agent not found' });
 
+    // Circuit breaker: refuse push to open-circuit workers
+    if (!isAvailable(db, agentId)) {
+      return reply.status(423).send({ error: 'circuit_open', reason: 'Worker circuit is open — too many consecutive failures. Try again after 30s or after a successful assignment.' });
+    }
+
     // Try live channel delivery first
     const session = db.prepare(
       `SELECT agent_id, channel_id FROM coord_channel_sessions WHERE agent_id = ? AND status = 'connected'`

package/src/coordination/schema.ts

@@ -130,6 +130,15 @@ CREATE TABLE IF NOT EXISTS coord_events (
   detail       TEXT,
   created_at   TEXT NOT NULL DEFAULT (datetime('now'))
 );
+
+-- Coordination: per-worker circuit breaker state (AWM 0.8.1)
+CREATE TABLE IF NOT EXISTS coord_circuit_state (
+  agent_id             TEXT PRIMARY KEY REFERENCES coord_agents(id),
+  state                TEXT NOT NULL DEFAULT 'closed',
+  consecutive_failures INTEGER NOT NULL DEFAULT 0,
+  opened_at            TEXT NULL,
+  last_transition_at   TEXT NOT NULL DEFAULT (datetime('now'))
+);
 `;
 
 /**
@@ -170,4 +179,8 @@ export function initCoordinationTables(db: Database.Database): void {
 
   // Migration: session token for hijack prevention (added 2026-03-26)
   try { db.exec(`ALTER TABLE coord_agents ADD COLUMN session_token TEXT`); } catch { /* exists */ }
+
+  // Migrations: control-layer columns for retry tracking (AWM 0.8.1)
+  try { db.exec(`ALTER TABLE coord_assignments ADD COLUMN attempt_count INTEGER NOT NULL DEFAULT 0`); } catch { /* exists */ }
+  try { db.exec(`ALTER TABLE coord_assignments ADD COLUMN last_failure_mode TEXT`); } catch { /* exists */ }
 }

package/src/coordination/stale.ts

@@ -5,6 +5,8 @@
  */
 
 import type Database from 'better-sqlite3';
+import { classifyFailure, FailureMode, MUTATION_HINTS } from './failure-modes.js';
+import { recordFailure as circuitRecordFailure } from './circuit-breaker.js';
 
 interface StaleAgent {
   id: string;
@@ -26,22 +28,89 @@ export function detectStale(db: Database.Database, thresholdSeconds: number): St
   ).all(thresholdSeconds) as StaleAgent[];
 }
 
-/** Clean up stale agents: fail assignments, release locks, mark dead. */
+/** Maximum retry attempts before an assignment is permanently failed. */
+const MAX_RETRY_ATTEMPTS = 3;
+
+/**
+ * Retry or permanently fail a single assignment.
+ * Called from cleanupStale (stale path) and POST /assignment/:id/fail (voluntary path).
+ *
+ * @returns 'retried' | 'failed'
+ */
+export function retryOrFailAssignment(
+  db: Database.Database,
+  assignmentId: string,
+  agentId: string,
+  failureResult: string,
+  mode?: FailureMode,
+): 'retried' | 'failed' {
+  const row = db.prepare(
+    `SELECT attempt_count, description FROM coord_assignments WHERE id = ?`
+  ).get(assignmentId) as { attempt_count: number; description: string | null } | undefined;
+
+  const attemptCount = row?.attempt_count ?? 0;
+  const failureMode = mode ?? classifyFailure(failureResult);
+
+  if (attemptCount < MAX_RETRY_ATTEMPTS) {
+    const hint = MUTATION_HINTS[failureMode];
+    const nextAttempt = attemptCount + 1;
+    const hintBlock = `\n\n--- RETRY HINT (attempt ${nextAttempt}) ---\n${hint}`;
+    const newDescription = (row?.description ?? '') + hintBlock;
+
+    db.prepare(
+      `UPDATE coord_assignments
+       SET status = 'pending', agent_id = NULL, started_at = NULL, result = NULL,
+           attempt_count = ?, last_failure_mode = ?, description = ?
+       WHERE id = ?`
+    ).run(nextAttempt, failureMode, newDescription, assignmentId);
+
+    db.prepare(
+      `INSERT INTO coord_events (agent_id, event_type, detail) VALUES (?, 'assignment_retry', ?)`
+    ).run(agentId, `attempt ${nextAttempt}: ${failureMode} — ${failureResult.slice(0, 200)}`);
+
+    return 'retried';
+  } else {
+    db.prepare(
+      `UPDATE coord_assignments
+       SET status = 'failed', result = ?, last_failure_mode = ?, completed_at = datetime('now')
+       WHERE id = ?`
+    ).run(failureResult, failureMode, assignmentId);
+
+    db.prepare(
+      `INSERT INTO coord_events (agent_id, event_type, detail) VALUES (?, 'assignment_failed', ?)`
+    ).run(agentId, `permanently failed after ${attemptCount} attempt(s): ${failureResult.slice(0, 200)}`);
+
+    return 'failed';
+  }
+}
+
+/** Clean up stale agents: retry or fail assignments, release locks, mark dead. */
 export function cleanupStale(db: Database.Database, thresholdSeconds: number): { stale: StaleAgent[]; cleaned: number } {
   const stale = detectStale(db, thresholdSeconds);
   let cleaned = 0;
 
   for (const agent of stale) {
-    // Fail active assignments
-    const orphaned = db.prepare(
-      `UPDATE coord_assignments SET status = 'failed', result = 'agent disconnected (stale)', completed_at = datetime('now')
-       WHERE agent_id = ? AND status IN ('assigned', 'in_progress')`
-    ).run(agent.id);
+    // Handle each orphaned assignment with retry logic
+    const orphanedRows = db.prepare(
+      `SELECT id FROM coord_assignments WHERE agent_id = ? AND status IN ('assigned', 'in_progress')`
+    ).all(agent.id) as Array<{ id: string }>;
+
+    let retried = 0;
+    let failed = 0;
+
+    for (const row of orphanedRows) {
+      const outcome = retryOrFailAssignment(db, row.id, agent.id, 'agent disconnected (stale)');
+      if (outcome === 'retried') retried++;
+      else failed++;
+    }
 
-    if (orphaned.changes > 0) {
+    if (orphanedRows.length > 0) {
       db.prepare(
         `INSERT INTO coord_events (agent_id, event_type, detail) VALUES (?, 'assignment_failed', ?)`
-      ).run(agent.id, `auto-failed ${orphaned.changes} orphaned assignment(s) — agent stale`);
+      ).run(agent.id, `stale cleanup: retried ${retried}, permanently failed ${failed}`);
+
+      // Record failure in circuit breaker for each orphaned assignment
+      circuitRecordFailure(db, agent.id);
     }
 
     // Release locks
@@ -50,12 +119,12 @@ export function cleanupStale(db: Database.Database, thresholdSeconds: number): {
     // Mark dead
     db.prepare(`UPDATE coord_agents SET status = 'dead', current_task = NULL WHERE id = ?`).run(agent.id);
 
-    cleaned += orphaned.changes + locks.changes;
+    cleaned += orphanedRows.length + locks.changes;
 
-    if (orphaned.changes > 0 || locks.changes > 0) {
+    if (orphanedRows.length > 0 || locks.changes > 0) {
       db.prepare(
         `INSERT INTO coord_events (agent_id, event_type, detail) VALUES (?, 'stale_cleanup', ?)`
-      ).run(agent.id, `failed ${orphaned.changes} assignment(s), released ${locks.changes} lock(s)`);
+      ).run(agent.id, `failed ${orphanedRows.length} assignment(s), released ${locks.changes} lock(s)`);
     }
   }