@springbrand/agent-runtime 0.2.0-alpha.22 → 0.2.0-alpha.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@springbrand/agent-runtime",
-  "version": "0.2.0-alpha.22",
+  "version": "0.2.0-alpha.24",
   "type": "module",
   "files": [
     "src",

package/src/db/runtime-event-outbox.repo.ts

@@ -5,6 +5,9 @@ export interface StoredRuntimeEvent {
   body: string;
   createdAt: number;
   deliveredAt: number | null;
+  attemptCount: number;
+  quarantinedAt: number | null;
+  lastError: string | null;
 }
 
 export interface NewRuntimeEvent {
@@ -18,6 +21,9 @@ type RuntimeEventRow = {
   body: string;
   created_at: number;
   delivered_at: number | null;
+  attempt_count: number;
+  quarantined_at: number | null;
+  last_error: string | null;
 };
 
 function mapRow(row: RuntimeEventRow): StoredRuntimeEvent {
@@ -26,6 +32,9 @@ function mapRow(row: RuntimeEventRow): StoredRuntimeEvent {
     body: row.body,
     createdAt: row.created_at,
     deliveredAt: row.delivered_at,
+    attemptCount: row.attempt_count,
+    quarantinedAt: row.quarantined_at,
+    lastError: row.last_error,
   };
 }
 
@@ -42,9 +51,11 @@ export class RuntimeEventOutboxRepository {
     }
     this.sql`
       INSERT INTO pi_turn_event_outbox (
-        event_id, body, created_at, delivered_at
+        event_id, body, created_at, delivered_at,
+        attempt_count, quarantined_at, last_error
       ) VALUES (
-        ${event.eventId}, ${event.body}, ${event.createdAt}, NULL
+        ${event.eventId}, ${event.body}, ${event.createdAt}, NULL,
+        0, NULL, NULL
       )
     `;
     return true;
@@ -52,9 +63,10 @@ export class RuntimeEventOutboxRepository {
 
   listPending(): StoredRuntimeEvent[] {
     return this.sql<RuntimeEventRow>`
-      SELECT event_id, body, created_at, delivered_at
+      SELECT event_id, body, created_at, delivered_at,
+             attempt_count, quarantined_at, last_error
       FROM pi_turn_event_outbox
-      WHERE delivered_at IS NULL
+      WHERE delivered_at IS NULL AND quarantined_at IS NULL
       ORDER BY created_at, rowid
     `.map(mapRow);
   }
@@ -64,7 +76,7 @@ export class RuntimeEventOutboxRepository {
       this.sql<{ pending: number }>`
         SELECT 1 AS pending
         FROM pi_turn_event_outbox
-        WHERE delivered_at IS NULL
+        WHERE delivered_at IS NULL AND quarantined_at IS NULL
         LIMIT 1
       `[0],
     );
@@ -106,14 +118,33 @@ export class RuntimeEventOutboxRepository {
   markDelivered(eventId: string, deliveredAt: number): void {
     this.sql`
       UPDATE pi_turn_event_outbox
-      SET delivered_at = ${deliveredAt}
+      SET delivered_at = ${deliveredAt}, last_error = NULL
       WHERE event_id = ${eventId} AND delivered_at IS NULL
     `;
   }
 
-  private find(eventId: string): StoredRuntimeEvent | null {
+  markFailed(eventId: string, error: string): number {
+    this.sql`
+      UPDATE pi_turn_event_outbox
+      SET attempt_count = attempt_count + 1, last_error = ${error}
+      WHERE event_id = ${eventId}
+        AND delivered_at IS NULL AND quarantined_at IS NULL
+    `;
+    return this.find(eventId)?.attemptCount ?? 0;
+  }
+
+  quarantine(eventId: string, quarantinedAt: number, error: string): void {
+    this.sql`
+      UPDATE pi_turn_event_outbox
+      SET quarantined_at = ${quarantinedAt}, last_error = ${error}
+      WHERE event_id = ${eventId} AND delivered_at IS NULL
+    `;
+  }
+
+  find(eventId: string): StoredRuntimeEvent | null {
     const row = this.sql<RuntimeEventRow>`
-      SELECT event_id, body, created_at, delivered_at
+      SELECT event_id, body, created_at, delivered_at,
+             attempt_count, quarantined_at, last_error
       FROM pi_turn_event_outbox
       WHERE event_id = ${eventId}
     `[0];

package/src/db/schema.ts

@@ -102,8 +102,25 @@ export function initializeSchema(sql: SqlTaggedTemplate): void {
     event_id TEXT PRIMARY KEY,
     body TEXT NOT NULL,
     created_at INTEGER NOT NULL,
-    delivered_at INTEGER
+    delivered_at INTEGER,
+    attempt_count INTEGER NOT NULL DEFAULT 0,
+    quarantined_at INTEGER,
+    last_error TEXT
   )`;
+  const runtimeEventColumns = new Set(
+    sql<{ name: string }>`PRAGMA table_info(pi_turn_event_outbox)`
+      .map((column) => column.name),
+  );
+  if (!runtimeEventColumns.has("attempt_count")) {
+    sql`ALTER TABLE pi_turn_event_outbox
+        ADD COLUMN attempt_count INTEGER NOT NULL DEFAULT 0`;
+  }
+  if (!runtimeEventColumns.has("quarantined_at")) {
+    sql`ALTER TABLE pi_turn_event_outbox ADD COLUMN quarantined_at INTEGER`;
+  }
+  if (!runtimeEventColumns.has("last_error")) {
+    sql`ALTER TABLE pi_turn_event_outbox ADD COLUMN last_error TEXT`;
+  }
   sql`CREATE TABLE IF NOT EXISTS pi_telemetry_outbox (
     event_id TEXT PRIMARY KEY,
     body TEXT NOT NULL,

package/src/runtime.ts

@@ -129,6 +129,7 @@ import {
 
 const SCHEDULED_STABLE_TIMEOUT_MS = 30_000;
 const TURN_EVENT_RETRY_SECONDS = 10;
+const RUNTIME_EVENT_MAX_ATTEMPTS = 5;
 
 // 计划续跑丢失多久后由看门狗接手。必须显著长于一个正常执行片的时长，
 // 否则会在健康的长执行片上误判并派发出重复的一片。
@@ -1103,25 +1104,41 @@ export abstract class AgentRuntimeKernel<
           if (!turnEvents.onSubagentUsage) continue;
           const confirmation = await turnEvents.onSubagentUsage(payload.event);
           if (confirmation?.confirmed !== true) {
-            failed = true;
-            break;
+            throw new Error("Runtime event was not confirmed");
           }
         } else {
           if (!turnEvents.onLifecycleFact) continue;
           const confirmation = await turnEvents.onLifecycleFact(payload.event);
           if (confirmation?.confirmed !== true) {
-            failed = true;
-            break;
+            throw new Error("Runtime event was not confirmed");
           }
         }
         this.db.transaction(() =>
           this.db.runtimeEvents.markDelivered(row.eventId, Date.now())
         );
       } catch (error) {
+        const failure = errorText(error);
+        const attemptCount = this.db.transaction(() => {
+          const nextAttemptCount = this.db.runtimeEvents.markFailed(
+            row.eventId,
+            failure,
+          );
+          if (nextAttemptCount >= RUNTIME_EVENT_MAX_ATTEMPTS) {
+            this.db.runtimeEvents.quarantine(row.eventId, Date.now(), failure);
+          }
+          return nextAttemptCount;
+        });
+        if (attemptCount >= RUNTIME_EVENT_MAX_ATTEMPTS) {
+          console.warn(
+            "[runtime-turn-event:quarantined]",
+            json({ eventId: row.eventId, attemptCount, error: failure }),
+          );
+          continue;
+        }
         failed = true;
         console.warn(
           "[runtime-turn-event:degraded]",
-          json({ eventId: row.eventId, error: errorText(error) }),
+          json({ eventId: row.eventId, attemptCount, error: failure }),
         );
         break;
       }