agent-relay-server 0.10.7 → 0.10.8

package/src/lifecycle-manager.ts

@@ -0,0 +1,369 @@
+import { createCommand } from "./commands-db";
+import { getAgent, getDb, getOrchestrator, resolveQueuedPolicyMessages } from "./db";
+import {
+  getManagedAgentState,
+  listSpawnPolicies,
+  updateManagedAgentState,
+  upsertManagedAgentState,
+} from "./config-store";
+import { emitRelayEvent } from "./events";
+import type { Command, ManagedAgentState, SpawnPolicy } from "./types";
+
+const DEFAULT_TICK_MS = 10_000;
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+interface LifecycleManagerOptions {
+  tickMs?: number;
+  now?: () => number;
+}
+
+export class LifecycleManager {
+  private timer?: Timer;
+  private readonly tickMs: number;
+  private readonly now: () => number;
+  private readonly updateRestartPending = new Set<string>();
+
+  constructor(options: LifecycleManagerOptions = {}) {
+    this.tickMs = options.tickMs ?? DEFAULT_TICK_MS;
+    this.now = options.now ?? (() => Date.now());
+  }
+
+  start(): void {
+    if (this.timer) return;
+    void this.tick();
+    this.timer = setInterval(() => void this.tick(), this.tickMs);
+  }
+
+  stop(): void {
+    if (!this.timer) return;
+    clearInterval(this.timer);
+    this.timer = undefined;
+  }
+
+  loadPolicies(): SpawnPolicy[] {
+    return listSpawnPolicies().map((entry) => entry.value);
+  }
+
+  async tick(): Promise<void> {
+    for (const policy of this.loadPolicies()) {
+      this.reconcilePolicy(policy);
+    }
+  }
+
+  onMessageForPolicy(policyName: string): void {
+    const policy = this.loadPolicies().find((item) => item.name === policyName);
+    if (!policy || policy.mode !== "on-demand") return;
+    const state = getManagedAgentState(policy.name);
+    if (state && ["starting", "running", "stopping"].includes(state.status)) return;
+    this.spawnAgent(policy, "message-trigger");
+  }
+
+  onAgentRegistered(agentId: string, meta: { policyName?: string; spawnRequestId?: string }): void {
+    if (!meta.policyName || !meta.spawnRequestId) return;
+    const state = getManagedAgentState(meta.policyName);
+    if (!state || state.spawnRequestId !== meta.spawnRequestId) return;
+    const next = updateManagedAgentState(meta.policyName, {
+      status: "running",
+      agentId,
+      healthySince: this.now(),
+      backoffUntil: undefined,
+      lastError: undefined,
+    });
+    if (next) this.emitState(next);
+    const available = resolveQueuedPolicyMessages(meta.policyName, agentId);
+    if (available.length) {
+      emitRelayEvent({
+        type: "message.available",
+        source: "server",
+        subject: `policy:${meta.policyName}`,
+        data: { policyName: meta.policyName, agentId, messageIds: available.map((message) => message.id), count: available.length },
+      });
+    }
+  }
+
+  onAgentDisappeared(agentId: string): void {
+    const policy = this.loadPolicies().find((item) => getManagedAgentState(item.name)?.agentId === agentId);
+    if (!policy) return;
+    const state = getManagedAgentState(policy.name);
+    if (!state) return;
+    if (state.status === "stopping") {
+      const next = updateManagedAgentState(policy.name, {
+        status: "stopped",
+        agentId: undefined,
+        tmuxSession: undefined,
+        lastStopAt: this.now(),
+      });
+      if (next) this.emitState(next);
+      return;
+    }
+    this.markBackoff(policy, state, "agent disappeared");
+  }
+
+  onConfigChanged(namespace: string, key: string): void {
+    if (namespace !== "spawn-policy") return;
+    const policy = this.loadPolicies().find((item) => item.name === key);
+    if (policy) this.reconcilePolicy(policy);
+  }
+
+  onRelayUpdated(_newVersion: string): void {
+    for (const policy of this.loadPolicies()) {
+      if (policy.restartOnUpdate) this.updateRestartPending.add(policy.name);
+    }
+  }
+
+  spawnAgent(policy: SpawnPolicy, reason = "reconcile"): Command | null {
+    const orch = getOrchestrator(policy.orchestratorId);
+    if (!orch || orch.status !== "online") {
+      const state = upsertManagedAgentState({
+        policyName: policy.name,
+        status: "backoff",
+        orchestratorId: policy.orchestratorId,
+        provider: policy.provider,
+        backoffUntil: this.now() + this.backoffDelay(policy, getManagedAgentState(policy.name)) * 1000,
+        lastError: "orchestrator offline",
+      });
+      this.emitState(state);
+      return null;
+    }
+    const spawnRequestId = `sp_${crypto.randomUUID()}`;
+    const state = upsertManagedAgentState({
+      policyName: policy.name,
+      status: "starting",
+      orchestratorId: policy.orchestratorId,
+      provider: policy.provider,
+      spawnRequestId,
+      lastSpawnAt: this.now(),
+      restartCount: getManagedAgentState(policy.name)?.restartCount ?? 0,
+      consecutiveFailures: getManagedAgentState(policy.name)?.consecutiveFailures ?? 0,
+    });
+    this.emitState(state);
+    const command = createCommand({
+      type: "agent.spawn",
+      source: "system",
+      target: orch.agentId,
+      correlationId: spawnRequestId,
+      params: {
+        action: "spawn",
+        provider: policy.provider,
+        cwd: policy.cwd,
+        label: policy.label,
+        tags: policy.tags,
+        capabilities: policy.capabilities,
+        approvalMode: policy.permissionMode,
+        permissionMode: policy.permissionMode,
+        providerArgs: policy.providerArgs,
+        prompt: policy.prompt,
+        headless: true,
+        policyName: policy.name,
+        spawnRequestId,
+        reason,
+        requestedBy: "lifecycle-manager",
+        requestedAt: this.now(),
+      },
+    });
+    this.emitCommand(command);
+    return command;
+  }
+
+  stopAgent(policy: SpawnPolicy, graceful = true, reason = "reconcile-stop"): Command | null {
+    const orch = getOrchestrator(policy.orchestratorId);
+    if (!orch) return null;
+    const state = getManagedAgentState(policy.name);
+    const next = upsertManagedAgentState({
+      policyName: policy.name,
+      status: "stopping",
+      agentId: state?.agentId,
+      orchestratorId: policy.orchestratorId,
+      provider: policy.provider,
+      tmuxSession: state?.tmuxSession,
+      spawnRequestId: state?.spawnRequestId,
+      lastSpawnAt: state?.lastSpawnAt,
+      lastStopAt: this.now(),
+      restartCount: state?.restartCount ?? 0,
+      consecutiveFailures: state?.consecutiveFailures ?? 0,
+    });
+    this.emitState(next);
+    const command = createCommand({
+      type: "agent.shutdown",
+      source: "system",
+      target: orch.agentId,
+      correlationId: state?.spawnRequestId,
+      params: {
+        action: "shutdown",
+        policyName: policy.name,
+        spawnRequestId: state?.spawnRequestId,
+        agentId: state?.agentId,
+        tmuxSession: state?.tmuxSession,
+        graceful,
+        timeoutMs: 10_000,
+        reason,
+        requestedBy: "lifecycle-manager",
+        requestedAt: this.now(),
+      },
+    });
+    this.emitCommand(command);
+    return command;
+  }
+
+  restartAgent(policy: SpawnPolicy, reason = "reconcile-restart"): Command | null {
+    const state = getManagedAgentState(policy.name);
+    const restarted = upsertManagedAgentState({
+      policyName: policy.name,
+      status: "stopping",
+      agentId: state?.agentId,
+      orchestratorId: policy.orchestratorId,
+      provider: policy.provider,
+      tmuxSession: state?.tmuxSession,
+      spawnRequestId: state?.spawnRequestId,
+      lastSpawnAt: state?.lastSpawnAt,
+      lastStopAt: this.now(),
+      restartCount: (state?.restartCount ?? 0) + 1,
+      consecutiveFailures: state?.consecutiveFailures ?? 0,
+    });
+    this.emitState(restarted);
+    return this.stopAgent(policy, true, reason);
+  }
+
+  private reconcilePolicy(policy: SpawnPolicy): void {
+    const state = getManagedAgentState(policy.name);
+    if (!state) {
+      if (policy.mode === "always-on") this.spawnAgent(policy, "always-on");
+      else if (this.hasQueuedMessages(policy.name)) this.spawnAgent(policy, "message-trigger");
+      else this.ensureStoppedState(policy);
+      return;
+    }
+
+    if (state.status === "running") {
+      const agent = state.agentId ? getAgent(state.agentId) : null;
+      if (!agent || agent.status === "offline") {
+        this.markBackoff(policy, state, "agent offline");
+        return;
+      }
+      this.resetBackoffAfterHealthyRun(policy, state);
+      if (policy.mode === "on-demand" && this.isIdle(policy, state)) {
+        this.stopAgent(policy, true, "idle-timeout");
+        return;
+      }
+      if (policy.scheduledDailyRestart && this.dailyRestartDue(policy, state) && this.isIdle(policy, state, 3600)) {
+        this.restartAgent(policy, "daily-restart");
+        return;
+      }
+      if (this.updateRestartPending.has(policy.name) && this.isIdle(policy, state, 60)) {
+        this.updateRestartPending.delete(policy.name);
+        this.restartAgent(policy, "restart-on-update");
+      }
+      return;
+    }
+
+    if (state.status === "backoff") {
+      if ((state.backoffUntil ?? 0) <= this.now() && (policy.mode === "always-on" || this.hasQueuedMessages(policy.name))) {
+        this.spawnAgent(policy, "backoff-expired");
+      }
+      return;
+    }
+
+    if (state.status === "stopped") {
+      if (policy.mode === "always-on" || this.hasQueuedMessages(policy.name)) this.spawnAgent(policy, policy.mode === "always-on" ? "always-on" : "message-trigger");
+    }
+  }
+
+  private ensureStoppedState(policy: SpawnPolicy): void {
+    const state = upsertManagedAgentState({
+      policyName: policy.name,
+      status: "stopped",
+      orchestratorId: policy.orchestratorId,
+      provider: policy.provider,
+    });
+    this.emitState(state);
+  }
+
+  private markBackoff(policy: SpawnPolicy, state: ManagedAgentState, error: string): void {
+    if (state.status === "stopping") return;
+    const failures = state.consecutiveFailures + 1;
+    const delay = this.backoffDelay(policy, state);
+    const next = upsertManagedAgentState({
+      policyName: policy.name,
+      status: "backoff",
+      agentId: undefined,
+      orchestratorId: policy.orchestratorId,
+      provider: policy.provider,
+      spawnRequestId: state.spawnRequestId,
+      lastSpawnAt: state.lastSpawnAt,
+      restartCount: state.restartCount,
+      consecutiveFailures: failures,
+      backoffUntil: this.now() + delay * 1000,
+      lastError: error,
+    });
+    this.emitState(next);
+  }
+
+  private backoffDelay(policy: SpawnPolicy, state: ManagedAgentState | null): number {
+    const schedule = policy.backoff.schedule.length ? policy.backoff.schedule : [30];
+    const index = Math.min(state?.consecutiveFailures ?? 0, schedule.length - 1);
+    return schedule[index]!;
+  }
+
+  private resetBackoffAfterHealthyRun(policy: SpawnPolicy, state: ManagedAgentState): void {
+    if (state.consecutiveFailures === 0) return;
+    const healthySince = state.healthySince ?? state.lastSpawnAt;
+    if (!healthySince || this.now() - healthySince < policy.backoff.resetAfterSeconds * 1000) return;
+    const next = updateManagedAgentState(policy.name, {
+      consecutiveFailures: 0,
+      backoffUntil: undefined,
+      lastError: undefined,
+    });
+    if (next) this.emitState(next);
+  }
+
+  private dailyRestartDue(_policy: SpawnPolicy, state: ManagedAgentState): boolean {
+    return Boolean(state.lastSpawnAt && this.now() - state.lastSpawnAt >= DAY_MS);
+  }
+
+  private isIdle(policy: SpawnPolicy, state: ManagedAgentState, overrideSeconds?: number): boolean {
+    if (!state.agentId) return false;
+    const agent = getAgent(state.agentId);
+    if (!agent || agent.status === "busy") return false;
+    const idleSeconds = overrideSeconds ?? policy.onDemand?.keepaliveSeconds ?? 0;
+    if (idleSeconds === 0) return false;
+    return this.now() - this.lastActivityAt(state.agentId) >= idleSeconds * 1000;
+  }
+
+  private lastActivityAt(agentId: string): number {
+    const row = getDb().prepare(`
+      SELECT max(created_at) AS at
+      FROM messages
+      WHERE from_agent = ? OR to_target = ? OR resolved_to_agent = ?
+    `).get(agentId, agentId, agentId) as { at?: number | null } | undefined;
+    return row?.at ?? 0;
+  }
+
+  private hasQueuedMessages(policyName: string): boolean {
+    const row = getDb().prepare("SELECT 1 FROM messages WHERE to_target = ? AND delivery_status = 'queued' LIMIT 1").get(`policy:${policyName}`);
+    return Boolean(row);
+  }
+
+  private emitState(state: ManagedAgentState): void {
+    emitRelayEvent({
+      type: "policy.state.changed",
+      source: "lifecycle-manager",
+      subject: state.policyName,
+      data: state as unknown as Record<string, unknown>,
+    });
+  }
+
+  private emitCommand(command: Command): void {
+    emitRelayEvent({
+      type: "command.requested",
+      source: command.source,
+      subject: command.id,
+      data: { command },
+    });
+  }
+}
+
+let singleton: LifecycleManager | null = null;
+
+export function getLifecycleManager(): LifecycleManager {
+  if (!singleton) singleton = new LifecycleManager();
+  return singleton;
+}