maestro-agent 0.0.1 → 0.0.2

package/src/ops/runtime-scheduler.ts

@@ -0,0 +1,447 @@
+import type { Database } from "bun:sqlite";
+import { generateId, now } from "../core/db";
+import { getWorkflowForTask } from "../workflows";
+
+export interface RuntimeSchedulerOptions {
+  oversubscription_factor?: number; // 默认 1.0（不超分）
+  queue_timeout_ms?: number;        // 排队超时，默认 30000
+}
+
+export const RUNTIME_TASK_POLL_INTERVAL_MS = 3_000;
+export const AGENT_HEARTBEAT_INTERVAL_MS = 15_000;
+export const DEFAULT_TASK_TIMEOUT_MS = 5 * 60 * 1000;
+export const DEFAULT_HEARTBEAT_TIMEOUT_MS = 30_000;
+
+type SchedulerLogger = Pick<Console, "info" | "warn" | "error">;
+
+export interface RuntimeMaintenanceOptions {
+  task_timeout_ms?: number;
+  heartbeat_timeout_ms?: number;
+  clock?: () => number;
+  logger?: SchedulerLogger;
+  activeSessionIds?: Iterable<string>;
+  publish?: (type: string, payload: unknown) => void;
+}
+
+export interface RuntimeMaintenanceResult {
+  scanned: number;
+  recovered_tasks: string[];
+  interrupted_sessions: string[];
+  offline_agents: string[];
+  heartbeat_agents: string[];
+  runtime_ids: string[];
+}
+
+export interface QueueEntry {
+  resolve: () => void;
+  reject: (error: Error) => void;
+  timer: Timer;
+}
+
+export type SessionQueue = Map<string, QueueEntry[]>;
+
+export function createSessionQueue(): SessionQueue {
+  return new Map();
+}
+
+export function effectiveCapacity(capacity: number, factor = 1.0): number {
+  if (capacity < 0) return Infinity;
+  return Math.ceil(capacity * factor);
+}
+
+export function runningSessionCount(db: Database, runtimeId: string): number {
+  const row = db.query(`
+    SELECT COUNT(*) AS count
+    FROM session
+    JOIN agent ON agent.id = session.agent_id
+    WHERE agent.runtime_id = ?
+      AND session.status = 'running'
+  `).get(runtimeId) as { count: number };
+  return row.count;
+}
+
+export function isAtCapacity(db: Database, runtimeId: string, capacity: number, factor = 1.0): boolean {
+  const effective = effectiveCapacity(capacity, factor);
+  if (!isFinite(effective)) return false;
+  return runningSessionCount(db, runtimeId) >= effective;
+}
+
+export function waitForSlot(
+  queue: SessionQueue,
+  runtimeId: string,
+  timeoutMs = 30000,
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    let entry: QueueEntry;
+    const timer = setTimeout(() => {
+      removeFromQueue(queue, runtimeId, entry);
+      reject(new Error("Queue timeout: runtime capacity not freed in time"));
+    }, timeoutMs);
+
+    entry = {
+      resolve: () => {
+        clearTimeout(timer);
+        resolve();
+      },
+      reject,
+      timer,
+    };
+
+    if (!queue.has(runtimeId)) queue.set(runtimeId, []);
+    queue.get(runtimeId)!.push(entry);
+  });
+}
+
+export function notifyNextInQueue(queue: SessionQueue, runtimeId: string) {
+  const entries = queue.get(runtimeId);
+  if (!entries || entries.length === 0) return;
+  const next = entries.shift()!;
+  clearTimeout(next.timer);
+  next.resolve();
+  if (entries.length === 0) queue.delete(runtimeId);
+}
+
+function removeFromQueue(queue: SessionQueue, runtimeId: string, entry: QueueEntry) {
+  const entries = queue.get(runtimeId);
+  if (!entries) return;
+  const index = entries.indexOf(entry);
+  if (index !== -1) entries.splice(index, 1);
+  if (entries.length === 0) queue.delete(runtimeId);
+}
+
+export function sendHeartbeat(db: Database, agentId: string, options: RuntimeMaintenanceOptions = {}) {
+  const ts = getTimestamp(options);
+  db.run(
+    `UPDATE agent
+     SET last_active_at = ?,
+         status = CASE WHEN status = 'offline' THEN 'idle' ELSE status END
+     WHERE id = ?`,
+    [ts, agentId],
+  );
+  return { agent_id: agentId, heartbeat_at: ts };
+}
+
+export function sendHeartbeatsForRunningSessions(
+  db: Database,
+  activeSessionIds: Iterable<string>,
+  options: RuntimeMaintenanceOptions = {},
+): RuntimeMaintenanceResult {
+  const sessionIds = Array.from(new Set(activeSessionIds));
+  const result = emptyResult();
+  if (sessionIds.length === 0) return result;
+
+  const placeholders = sessionIds.map(() => "?").join(", ");
+  const rows = db.query(`
+    SELECT DISTINCT session.agent_id, agent.runtime_id
+    FROM session
+    JOIN agent ON agent.id = session.agent_id
+    WHERE session.status = 'running'
+      AND session.id IN (${placeholders})
+  `).all(...sessionIds) as any[];
+
+  for (const row of rows) {
+    sendHeartbeat(db, row.agent_id, options);
+    result.heartbeat_agents.push(row.agent_id);
+    if (row.runtime_id) addUnique(result.runtime_ids, row.runtime_id);
+  }
+  return result;
+}
+
+export function pollClaimedTasks(db: Database, options: RuntimeMaintenanceOptions = {}): RuntimeMaintenanceResult {
+  const ts = getTimestamp(options);
+  const taskCutoff = ts - getTaskTimeout(options);
+  const heartbeatCutoff = ts - getHeartbeatTimeout(options);
+  const activeSessionSet = options.activeSessionIds
+    ? new Set(Array.from(options.activeSessionIds))
+    : null;
+  const result = emptyResult();
+
+  const rows = db.query(`
+    SELECT
+      task.id,
+      task.status,
+      task.assignee_agent_id,
+      task.updated_at,
+      agent.status AS agent_status,
+      agent.last_active_at,
+      agent.runtime_id,
+      (
+        SELECT session.id
+        FROM session
+        WHERE session.task_id = task.id
+          AND session.status = 'running'
+        ORDER BY session.started_at DESC
+        LIMIT 1
+      ) AS session_id
+    FROM task
+    LEFT JOIN agent ON agent.id = task.assignee_agent_id
+    WHERE task.status IN ('claimed', 'in_progress')
+  `).all() as any[];
+
+  result.scanned = rows.length;
+  for (const row of rows) {
+    const hasRunningSession = Boolean(row.session_id);
+    const missingTrackedProcess = Boolean(activeSessionSet && row.session_id && !activeSessionSet.has(row.session_id));
+    const agentTimedOut = !row.assignee_agent_id
+      || row.agent_status === "offline"
+      || Number(row.last_active_at || 0) <= heartbeatCutoff;
+    const taskTimedOut = Number(row.updated_at || 0) <= taskCutoff;
+
+    let reason: string | null = null;
+    if (!hasRunningSession && taskTimedOut) reason = "orphaned_task";
+    else if (missingTrackedProcess && taskTimedOut) reason = "missing_tracked_process";
+    else if (taskTimedOut && agentTimedOut) reason = "task_and_heartbeat_timeout";
+    if (!reason) continue;
+
+    recoverTask(db, row, reason, ts, result, options);
+  }
+
+  if (result.recovered_tasks.length > 0) {
+    options.logger?.warn?.(
+      `[runtime-scheduler] poll recovered ${result.recovered_tasks.length}/${result.scanned} claimed task(s)`,
+    );
+  }
+  return result;
+}
+
+export function markHeartbeatTimedOutAgents(
+  db: Database,
+  options: RuntimeMaintenanceOptions = {},
+): RuntimeMaintenanceResult {
+  const ts = getTimestamp(options);
+  const cutoff = ts - getHeartbeatTimeout(options);
+  const result = emptyResult();
+  // Only timeout agents that are actively working (busy/running).
+  // Idle agents don't have running sessions to send heartbeats,
+  // so they should NOT be marked offline due to heartbeat timeout.
+  const agents = db.query(`
+    SELECT id, runtime_id, status, last_active_at
+    FROM agent
+    WHERE status NOT IN ('offline', 'idle')
+      AND last_active_at <= ?
+  `).all(cutoff) as any[];
+
+  result.scanned = agents.length;
+  for (const agent of agents) {
+    db.run("UPDATE agent SET status = 'offline', last_active_at = ? WHERE id = ?", [ts, agent.id]);
+    result.offline_agents.push(agent.id);
+    if (agent.runtime_id) addUnique(result.runtime_ids, agent.runtime_id);
+
+    const sessions = db.query(
+      "SELECT id FROM session WHERE agent_id = ? AND status = 'running'",
+    ).all(agent.id) as any[];
+    db.run("UPDATE session SET status = 'interrupted', ended_at = ? WHERE agent_id = ? AND status = 'running'", [
+      ts,
+      agent.id,
+    ]);
+    for (const session of sessions) result.interrupted_sessions.push(session.id);
+
+    const tasks = db.query(`
+      SELECT id, status, assignee_agent_id, updated_at, ? AS runtime_id
+      FROM task
+      WHERE assignee_agent_id = ?
+        AND status IN ('claimed', 'in_progress')
+    `).all(agent.runtime_id || null, agent.id) as any[];
+    for (const task of tasks) {
+      recoverTask(db, task, "heartbeat_timeout", ts, result, options);
+    }
+
+    audit(db, "runtime_scheduler.agent_offline", agent.id, {
+      last_active_at: agent.last_active_at,
+      heartbeat_timeout_ms: getHeartbeatTimeout(options),
+    });
+    options.publish?.("agent.offline", { id: agent.id, last_active_at: agent.last_active_at });
+  }
+
+  if (result.offline_agents.length > 0) {
+    options.logger?.warn?.(
+      `[runtime-scheduler] heartbeat timeout: marked ${result.offline_agents.length} agent(s) offline`,
+    );
+  }
+  return result;
+}
+
+export function startRuntimeScheduler(
+  ctx: {
+    db: Database;
+    sessions: Map<string, unknown>;
+    sessionQueue: SessionQueue;
+    bus?: { publish: (type: string, payload: unknown) => void };
+  },
+  options: RuntimeMaintenanceOptions & {
+    poll_interval_ms?: number;
+    heartbeat_interval_ms?: number;
+  } = {},
+) {
+  const logger = options.logger || console;
+  const pollIntervalMs = options.poll_interval_ms ?? RUNTIME_TASK_POLL_INTERVAL_MS;
+  const heartbeatIntervalMs = options.heartbeat_interval_ms ?? AGENT_HEARTBEAT_INTERVAL_MS;
+  const publish = options.publish || ((type: string, payload: unknown) => ctx.bus?.publish(type, payload));
+
+  const runPoll = () => {
+    try {
+      const result = pollClaimedTasks(ctx.db, {
+        ...options,
+        logger,
+        publish,
+        activeSessionIds: ctx.sessions.keys(),
+      });
+      releaseRuntimeQueues(ctx.sessionQueue, result.runtime_ids);
+    } catch (err) {
+      logger.error("[runtime-scheduler] task poll error:", err);
+    }
+  };
+
+  const runHeartbeat = () => {
+    try {
+      const heartbeat = sendHeartbeatsForRunningSessions(ctx.db, ctx.sessions.keys(), { ...options, logger });
+      const timeout = markHeartbeatTimedOutAgents(ctx.db, { ...options, logger, publish });
+      releaseRuntimeQueues(ctx.sessionQueue, [...heartbeat.runtime_ids, ...timeout.runtime_ids]);
+      if (heartbeat.heartbeat_agents.length > 0 || timeout.offline_agents.length > 0) {
+        logger.info(
+          `[runtime-scheduler] heartbeat: sent=${heartbeat.heartbeat_agents.length} offline=${timeout.offline_agents.length}`,
+        );
+      }
+    } catch (err) {
+      logger.error("[runtime-scheduler] heartbeat error:", err);
+    }
+  };
+
+  logger.info(
+    `[runtime-scheduler] started poll=${pollIntervalMs}ms heartbeat=${heartbeatIntervalMs}ms task_timeout=${getTaskTimeout(options)}ms heartbeat_timeout=${getHeartbeatTimeout(options)}ms`,
+  );
+
+  const pollTimer = setInterval(runPoll, pollIntervalMs);
+  const heartbeatTimer = setInterval(runHeartbeat, heartbeatIntervalMs);
+  pollTimer.unref?.();
+  heartbeatTimer.unref?.();
+
+  runPoll();
+  runHeartbeat();
+
+  return {
+    stop() {
+      clearInterval(pollTimer);
+      clearInterval(heartbeatTimer);
+    },
+  };
+}
+
+function recoverTask(
+  db: Database,
+  task: any,
+  reason: string,
+  ts: number,
+  result: RuntimeMaintenanceResult,
+  options: RuntimeMaintenanceOptions,
+) {
+  // Check recovery count — if already recovered 3+ times, abandon the task instead of looping
+  const MAX_RECOVERY_ATTEMPTS = 3;
+  const recoveryCount = (db.query(
+    "SELECT COUNT(*) as cnt FROM task_thread_item WHERE task_id = ? AND kind = 'scheduler_recovery'",
+  ).get(task.id) as any)?.cnt ?? 0;
+
+  if (recoveryCount >= MAX_RECOVERY_ATTEMPTS) {
+    // Too many recoveries — mark task as abandoned to break the loop
+    db.run("UPDATE session SET status = 'interrupted', ended_at = ? WHERE task_id = ? AND status = 'running'", [
+      ts,
+      task.id,
+    ]);
+    db.run(
+      "UPDATE task SET status = 'abandoned', assignee_agent_id = NULL, claim_token = NULL, updated_at = ? WHERE id = ?",
+      [ts, task.id],
+    );
+    if (task.assignee_agent_id) {
+      db.run(
+        `UPDATE agent SET status = 'idle', last_active_at = ? WHERE id = ? AND status != 'offline'
+         AND NOT EXISTS (SELECT 1 FROM session WHERE agent_id = ? AND status = 'running')`,
+        [ts, task.assignee_agent_id, task.assignee_agent_id],
+      );
+    }
+    result.recovered_tasks.push(task.id);
+    if (task.runtime_id) addUnique(result.runtime_ids, task.runtime_id);
+    db.run(
+      "INSERT INTO task_thread_item (id, task_id, kind, author, content, created_at) VALUES (?, ?, 'scheduler_recovery', 'system', ?, ?)",
+      [generateId("ti"), task.id, `Abandoned after ${recoveryCount} failed recovery attempts: ${reason}`, ts],
+    );
+    options.publish?.("task.abandoned", { id: task.id, reason, recovery_count: recoveryCount });
+    return;
+  }
+
+  const workflow = getWorkflowForTask(db, task.id);
+  db.run("UPDATE session SET status = 'interrupted', ended_at = ? WHERE task_id = ? AND status = 'running'", [
+    ts,
+    task.id,
+  ]);
+  db.run(
+    "UPDATE task SET status = ?, assignee_agent_id = NULL, claim_token = NULL, updated_at = ? WHERE id = ?",
+    [workflow.initial_status, ts, task.id],
+  );
+  if (task.assignee_agent_id) {
+    db.run(
+      `UPDATE agent
+       SET status = 'idle', last_active_at = ?
+       WHERE id = ?
+         AND status != 'offline'
+         AND NOT EXISTS (
+           SELECT 1 FROM session
+           WHERE agent_id = ?
+             AND status = 'running'
+         )`,
+      [ts, task.assignee_agent_id, task.assignee_agent_id],
+    );
+  }
+
+  result.recovered_tasks.push(task.id);
+  if (task.runtime_id) addUnique(result.runtime_ids, task.runtime_id);
+  audit(db, "runtime_scheduler.task_recovered", task.id, {
+    previous_status: task.status,
+    assignee_agent_id: task.assignee_agent_id || null,
+    reason,
+  });
+  db.run(
+    "INSERT INTO task_thread_item (id, task_id, kind, author, content, created_at) VALUES (?, ?, 'scheduler_recovery', 'system', ?, ?)",
+    [generateId("ti"), task.id, `Recovered by runtime scheduler: ${reason}`, ts],
+  );
+  options.publish?.("task.recovered", { id: task.id, reason, previous_status: task.status });
+}
+
+function emptyResult(): RuntimeMaintenanceResult {
+  return {
+    scanned: 0,
+    recovered_tasks: [],
+    interrupted_sessions: [],
+    offline_agents: [],
+    heartbeat_agents: [],
+    runtime_ids: [],
+  };
+}
+
+function releaseRuntimeQueues(queue: SessionQueue, runtimeIds: string[]) {
+  for (const runtimeId of new Set(runtimeIds)) {
+    notifyNextInQueue(queue, runtimeId);
+  }
+}
+
+function audit(db: Database, action: string, target: string, payload: unknown) {
+  db.run(
+    "INSERT INTO audit_log (id, actor, action, target, payload_json, created_at) VALUES (?, 'system', ?, ?, ?, ?)",
+    [generateId("audit"), action, target, JSON.stringify(payload), now()],
+  );
+}
+
+function getTimestamp(options: RuntimeMaintenanceOptions) {
+  return options.clock?.() ?? now();
+}
+
+function getTaskTimeout(options: RuntimeMaintenanceOptions) {
+  return options.task_timeout_ms ?? DEFAULT_TASK_TIMEOUT_MS;
+}
+
+function getHeartbeatTimeout(options: RuntimeMaintenanceOptions) {
+  return options.heartbeat_timeout_ms ?? DEFAULT_HEARTBEAT_TIMEOUT_MS;
+}
+
+function addUnique(values: string[], value: string) {
+  if (!values.includes(value)) values.push(value);
+}

package/src/ops/task-blocking.ts

@@ -0,0 +1,65 @@
+import type { Database } from "bun:sqlite";
+
+export interface TaskDependencySummary {
+  id: string;
+  title: string;
+  status: string;
+}
+
+export interface TaskBlockingDetails {
+  dependencies: TaskDependencySummary[];
+  blocking_dependencies: TaskDependencySummary[];
+  blocked_reason: string | null;
+}
+
+export function getTaskBlockingDetails(db: Database, taskIds: string[]): Record<string, TaskBlockingDetails> {
+  const result: Record<string, TaskBlockingDetails> = {};
+  const uniqueIds = [...new Set(taskIds.filter(Boolean))];
+  for (const id of uniqueIds) {
+    result[id] = {
+      dependencies: [],
+      blocking_dependencies: [],
+      blocked_reason: null,
+    };
+  }
+  if (uniqueIds.length === 0) return result;
+
+  const placeholders = uniqueIds.map(() => "?").join(",");
+  const dependencyRows = db.query(
+    `SELECT td.task_id, t.id, t.title, t.status
+     FROM task_dependency td
+     JOIN task t ON t.id = td.depends_on
+     WHERE td.task_id IN (${placeholders})
+     ORDER BY t.created_at ASC`,
+  ).all(...uniqueIds) as Array<{ task_id: string; id: string; title: string; status: string }>;
+
+  for (const row of dependencyRows) {
+    const dependency = {
+      id: row.id,
+      title: row.title,
+      status: row.status,
+    };
+    result[row.task_id].dependencies.push(dependency);
+    if (row.status !== "done") {
+      result[row.task_id].blocking_dependencies.push(dependency);
+    }
+  }
+
+  const reasonRows = db.query(
+    `SELECT task_id, content
+     FROM task_thread_item
+     WHERE task_id IN (${placeholders})
+       AND kind = 'workflow_transition'
+       AND content LIKE 'block:%'
+     ORDER BY created_at DESC`,
+  ).all(...uniqueIds) as Array<{ task_id: string; content: string }>;
+
+  for (const row of reasonRows) {
+    const details = result[row.task_id];
+    if (details && details.blocked_reason === null) {
+      details.blocked_reason = row.content.replace(/^block:\s*/i, "").trim() || null;
+    }
+  }
+
+  return result;
+}

package/src/ops/task-deps.ts

@@ -0,0 +1,37 @@
+import type { HubContext } from "../core/server";
+import { now } from "../core/db";
+
+/**
+ * After a task is marked done, check if any downstream tasks
+ * (that depend on it) can now be unblocked.
+ */
+export function unlockDependents(ctx: HubContext, completedTaskId: string, opts: {
+  doneStatus?: string;
+  blockedStatus?: string;
+  openStatus?: string;
+} = {}) {
+  const doneStatus = opts.doneStatus || "done";
+  const blockedStatus = opts.blockedStatus || "blocked";
+  const openStatus = opts.openStatus || "open";
+  const dependents = ctx.db.query(
+    "SELECT task_id FROM task_dependency WHERE depends_on = ?"
+  ).all(completedTaskId) as { task_id: string }[];
+
+  for (const { task_id } of dependents) {
+    // Check if ALL dependencies of this downstream task are done
+    const pending = ctx.db.query(`
+      SELECT td.depends_on FROM task_dependency td
+      JOIN task t ON t.id = td.depends_on
+      WHERE td.task_id = ? AND t.status != ?
+    `).all(task_id, doneStatus);
+
+    if (pending.length === 0) {
+      const task = ctx.db.query("SELECT * FROM task WHERE id = ?").get(task_id) as any;
+      if (task && task.status === blockedStatus) {
+        const ts = now();
+        ctx.db.run("UPDATE task SET status = ?, updated_at = ? WHERE id = ?", [openStatus, ts, task_id]);
+        ctx.bus.publish("task.unblocked", { id: task_id, unblocked_by: completedTaskId });
+      }
+    }
+  }
+}

package/src/ops/task-workspace.ts

@@ -0,0 +1,60 @@
+import { Database } from "bun:sqlite";
+import { existsSync, mkdirSync, rmSync } from "fs";
+import { join } from "path";
+import { now } from "../core/db";
+
+/**
+ * Creates an isolated working directory for a task.
+ * Structure: <hubDir>/workspaces/<taskId>/
+ *   ├── workspace/   — actual CWD for agent execution
+ *   ├── logs/        — execution logs
+ *   └── artifacts/   — build products marked for preservation
+ */
+export function createTaskWorkDir(db: Database, hubDir: string, taskId: string): string {
+  const baseDir = join(hubDir, "workspaces", taskId);
+  const workspaceDir = join(baseDir, "workspace");
+  const logsDir = join(baseDir, "logs");
+  const artifactsDir = join(baseDir, "artifacts");
+
+  mkdirSync(workspaceDir, { recursive: true });
+  mkdirSync(logsDir, { recursive: true });
+  mkdirSync(artifactsDir, { recursive: true });
+
+  db.run("UPDATE task SET work_dir = ?, updated_at = ? WHERE id = ?", [baseDir, now(), taskId]);
+
+  return baseDir;
+}
+
+/**
+ * Returns the working directory for a task, creating it if it doesn't exist.
+ */
+export function ensureTaskWorkDir(db: Database, hubDir: string, taskId: string): string {
+  const task = db.query("SELECT work_dir FROM task WHERE id = ?").get(taskId) as any;
+  if (task?.work_dir && existsSync(task.work_dir)) {
+    return task.work_dir;
+  }
+  return createTaskWorkDir(db, hubDir, taskId);
+}
+
+/**
+ * Removes the working directory for a task.
+ */
+export function removeTaskWorkDir(db: Database, taskId: string): boolean {
+  const task = db.query("SELECT work_dir FROM task WHERE id = ?").get(taskId) as any;
+  if (!task?.work_dir) return false;
+
+  if (existsSync(task.work_dir)) {
+    rmSync(task.work_dir, { recursive: true, force: true });
+  }
+  db.run("UPDATE task SET work_dir = NULL, updated_at = ? WHERE id = ?", [now(), taskId]);
+  return true;
+}
+
+/**
+ * Returns info about a task's working directory.
+ */
+export function getTaskWorkDirInfo(hubDir: string, taskId: string) {
+  const baseDir = join(hubDir, "workspaces", taskId);
+  const exists = existsSync(baseDir);
+  return { taskId, path: baseDir, exists };
+}