bopodev-api 0.1.1

package/src/services/heartbeat-service.ts

@@ -0,0 +1,706 @@
+import { and, desc, eq, inArray, sql } from "drizzle-orm";
+import { nanoid } from "nanoid";
+import { resolveAdapter } from "bopodev-agent-sdk";
+import type { AgentState, HeartbeatContext } from "bopodev-agent-sdk";
+import type { BopoDb } from "bopodev-db";
+import { agents, appendActivity, companies, goals, heartbeatRuns, issues, projects } from "bopodev-db";
+import { appendAuditEvent, appendCost } from "bopodev-db";
+import type { RealtimeHub } from "../realtime/hub";
+import { publishOfficeOccupantForAgent } from "../realtime/office-space";
+import { checkAgentBudget } from "./budget-service";
+
+export async function claimIssuesForAgent(
+  db: BopoDb,
+  companyId: string,
+  agentId: string,
+  heartbeatRunId: string,
+  maxItems = 5
+) {
+  const result = await db.execute(sql`
+    WITH candidate AS (
+      SELECT id
+      FROM issues
+      WHERE company_id = ${companyId}
+        AND assignee_agent_id = ${agentId}
+        AND status IN ('todo', 'in_progress')
+        AND is_claimed = false
+      ORDER BY updated_at ASC
+      LIMIT ${maxItems}
+      FOR UPDATE SKIP LOCKED
+    )
+    UPDATE issues i
+    SET is_claimed = true,
+        claimed_by_heartbeat_run_id = ${heartbeatRunId},
+        updated_at = CURRENT_TIMESTAMP
+    FROM candidate c
+    WHERE i.id = c.id
+    RETURNING i.id, i.project_id, i.title, i.body, i.status, i.priority, i.labels_json, i.tags_json;
+  `);
+
+  return (result.rows ?? []) as Array<{
+    id: string;
+    project_id: string;
+    title: string;
+    body: string | null;
+    status: string;
+    priority: string;
+    labels_json: string;
+    tags_json: string;
+  }>;
+}
+
+export async function releaseClaimedIssues(db: BopoDb, companyId: string, issueIds: string[]) {
+  if (issueIds.length === 0) {
+    return;
+  }
+  await db
+    .update(issues)
+    .set({ isClaimed: false, claimedByHeartbeatRunId: null, updatedAt: new Date() })
+    .where(and(eq(issues.companyId, companyId), inArray(issues.id, issueIds)));
+}
+
+export async function runHeartbeatForAgent(
+  db: BopoDb,
+  companyId: string,
+  agentId: string,
+  options?: { requestId?: string; trigger?: "manual" | "scheduler"; realtimeHub?: RealtimeHub }
+) {
+  const [agent] = await db
+    .select()
+    .from(agents)
+    .where(and(eq(agents.companyId, companyId), eq(agents.id, agentId)))
+    .limit(1);
+
+  if (!agent || agent.status === "paused" || agent.status === "terminated") {
+    return null;
+  }
+
+  const startedRuns = await db
+    .select({ id: heartbeatRuns.id, startedAt: heartbeatRuns.startedAt })
+    .from(heartbeatRuns)
+    .where(
+      and(
+        eq(heartbeatRuns.companyId, companyId),
+        eq(heartbeatRuns.agentId, agentId),
+        eq(heartbeatRuns.status, "started")
+      )
+    );
+  const staleRunThresholdMs = resolveStaleRunThresholdMs();
+  const nowTs = Date.now();
+  const staleRuns = startedRuns.filter((run) => {
+    const startedAt = run.startedAt.getTime();
+    return nowTs - startedAt >= staleRunThresholdMs;
+  });
+
+  if (staleRuns.length > 0) {
+    await recoverStaleHeartbeatRuns(db, companyId, agentId, staleRuns, {
+      requestId: options?.requestId,
+      trigger: options?.trigger ?? "manual",
+      staleRunThresholdMs
+    });
+  }
+
+  const budgetCheck = await checkAgentBudget(db, companyId, agentId);
+  const runId = nanoid(14);
+  if (budgetCheck.allowed) {
+    const claimed = await insertStartedRunAtomic(db, {
+      id: runId,
+      companyId,
+      agentId,
+      message: "Heartbeat started."
+    });
+    if (!claimed) {
+      const skippedRunId = nanoid(14);
+      await db.insert(heartbeatRuns).values({
+        id: skippedRunId,
+        companyId,
+        agentId,
+        status: "skipped",
+        finishedAt: new Date(),
+        message: "Heartbeat skipped: another run is already in progress for this agent."
+      });
+      await appendAuditEvent(db, {
+        companyId,
+        actorType: "system",
+        eventType: "heartbeat.skipped_overlap",
+        entityType: "heartbeat_run",
+        entityId: skippedRunId,
+        correlationId: options?.requestId ?? skippedRunId,
+        payload: { agentId, requestId: options?.requestId, trigger: options?.trigger ?? "manual" }
+      });
+      return skippedRunId;
+    }
+  } else {
+    await db.insert(heartbeatRuns).values({
+      id: runId,
+      companyId,
+      agentId,
+      status: "skipped",
+      message: "Heartbeat skipped due to budget hard-stop."
+    });
+  }
+
+  if (budgetCheck.allowed) {
+    await appendAuditEvent(db, {
+      companyId,
+      actorType: "system",
+      eventType: "heartbeat.started",
+      entityType: "heartbeat_run",
+      entityId: runId,
+      correlationId: options?.requestId ?? runId,
+      payload: {
+        agentId,
+        requestId: options?.requestId ?? null,
+        trigger: options?.trigger ?? "manual"
+      }
+    });
+  }
+
+  if (!budgetCheck.allowed) {
+    await appendAuditEvent(db, {
+      companyId,
+      actorType: "system",
+      eventType: "budget.hard_stop",
+      entityType: "agent",
+      entityId: agentId,
+      payload: { utilizationPct: budgetCheck.utilizationPct }
+    });
+    await publishOfficeOccupantForAgent(db, options?.realtimeHub, companyId, agentId);
+    return runId;
+  }
+
+  if (budgetCheck.utilizationPct >= 80) {
+    await appendAuditEvent(db, {
+      companyId,
+      actorType: "system",
+      eventType: "budget.soft_warning",
+      entityType: "agent",
+      entityId: agentId,
+      payload: { utilizationPct: budgetCheck.utilizationPct }
+    });
+  }
+
+  let issueIds: string[] = [];
+  let state: AgentState & {
+    runtime?: {
+      command?: string;
+      args?: string[];
+      cwd?: string;
+      timeoutMs?: number;
+      retryCount?: number;
+      retryBackoffMs?: number;
+    };
+  } = {};
+  let executionSummary = "";
+  let executionTrace: unknown = null;
+  let stateParseError: string | null = null;
+
+  try {
+    const workItems = await claimIssuesForAgent(db, companyId, agentId, runId);
+    issueIds = workItems.map((item) => item.id);
+    await publishOfficeOccupantForAgent(db, options?.realtimeHub, companyId, agentId);
+    const adapter = resolveAdapter(agent.providerType as "claude_code" | "codex" | "http" | "shell");
+    const parsedState = parseAgentState(agent.stateBlob);
+    state = parsedState.state;
+    stateParseError = parsedState.parseError;
+
+    const context = await buildHeartbeatContext(db, companyId, {
+      agentId,
+      agentName: agent.name,
+      agentRole: agent.role,
+      managerAgentId: agent.managerAgentId,
+      providerType: agent.providerType as "claude_code" | "codex" | "http" | "shell",
+      heartbeatRunId: runId,
+      state,
+      runtime: state.runtime,
+      workItems
+    });
+
+    const execution = await adapter.execute(context);
+    executionSummary = execution.summary;
+    executionTrace = execution.trace ?? null;
+
+    if (execution.tokenInput > 0 || execution.tokenOutput > 0 || execution.usdCost > 0) {
+      await appendCost(db, {
+        companyId,
+        providerType: agent.providerType,
+        tokenInput: execution.tokenInput,
+        tokenOutput: execution.tokenOutput,
+        usdCost: execution.usdCost.toFixed(6),
+        issueId: workItems[0]?.id ?? null,
+        projectId: workItems[0]?.project_id ?? null,
+        agentId
+      });
+    }
+
+    if (
+      execution.nextState ||
+      execution.usdCost > 0 ||
+      execution.tokenInput > 0 ||
+      execution.tokenOutput > 0 ||
+      execution.status !== "skipped"
+    ) {
+      await db
+        .update(agents)
+        .set({
+          stateBlob: JSON.stringify(execution.nextState ?? state),
+          usedBudgetUsd: sql`${agents.usedBudgetUsd} + ${execution.usdCost}`,
+          tokenUsage: sql`${agents.tokenUsage} + ${execution.tokenInput + execution.tokenOutput}`,
+          updatedAt: new Date()
+        })
+        .where(and(eq(agents.companyId, companyId), eq(agents.id, agentId)));
+    }
+
+    if (issueIds.length > 0 && execution.status === "ok") {
+      await db
+        .update(issues)
+        .set({ status: "in_review", updatedAt: new Date() })
+        .where(and(eq(issues.companyId, companyId), inArray(issues.id, issueIds)));
+
+      for (const issueId of issueIds) {
+        await appendActivity(db, {
+          companyId,
+          issueId,
+          actorType: "system",
+          eventType: "issue.sent_to_review",
+          payload: { heartbeatRunId: runId, agentId }
+        });
+      }
+    }
+
+    await db
+      .update(heartbeatRuns)
+      .set({
+        status: execution.status === "failed" ? "failed" : "completed",
+        finishedAt: new Date(),
+        message: execution.summary
+      })
+      .where(eq(heartbeatRuns.id, runId));
+
+    await appendAuditEvent(db, {
+      companyId,
+      actorType: "system",
+      eventType: "heartbeat.completed",
+      entityType: "heartbeat_run",
+      entityId: runId,
+      correlationId: options?.requestId ?? runId,
+      payload: {
+        agentId,
+        result: execution.summary,
+        issueIds,
+        usage: {
+          tokenInput: execution.tokenInput,
+          tokenOutput: execution.tokenOutput,
+          usdCost: execution.usdCost
+        },
+        trace: execution.trace ?? null,
+        diagnostics: {
+          stateParseError,
+          requestId: options?.requestId,
+          trigger: options?.trigger ?? "manual"
+        }
+      }
+    });
+  } catch (error) {
+    const classified = classifyHeartbeatError(error);
+    executionSummary = `Heartbeat failed (${classified.type}): ${classified.message}`;
+    await db
+      .update(heartbeatRuns)
+      .set({
+        status: "failed",
+        finishedAt: new Date(),
+        message: executionSummary
+      })
+      .where(eq(heartbeatRuns.id, runId));
+    await appendAuditEvent(db, {
+      companyId,
+      actorType: "system",
+      eventType: "heartbeat.failed",
+      entityType: "heartbeat_run",
+      entityId: runId,
+      correlationId: options?.requestId ?? runId,
+      payload: {
+        agentId,
+        issueIds,
+        errorType: classified.type,
+        errorMessage: classified.message,
+        trace: executionTrace,
+        diagnostics: {
+          stateParseError,
+          requestId: options?.requestId,
+          trigger: options?.trigger ?? "manual"
+        }
+      }
+    });
+  } finally {
+    try {
+      await releaseClaimedIssues(db, companyId, issueIds);
+    } catch (releaseError) {
+      await appendAuditEvent(db, {
+        companyId,
+        actorType: "system",
+        eventType: "heartbeat.release_failed",
+        entityType: "heartbeat_run",
+        entityId: runId,
+        correlationId: options?.requestId ?? runId,
+        payload: {
+          agentId,
+          issueIds,
+          error: String(releaseError)
+        }
+      });
+    }
+    await publishOfficeOccupantForAgent(db, options?.realtimeHub, companyId, agentId);
+  }
+
+  return runId;
+}
+
+async function insertStartedRunAtomic(
+  db: BopoDb,
+  input: { id: string; companyId: string; agentId: string; message: string }
+) {
+  const result = await db.execute(sql`
+    INSERT INTO heartbeat_runs (id, company_id, agent_id, status, message)
+    VALUES (${input.id}, ${input.companyId}, ${input.agentId}, 'started', ${input.message})
+    ON CONFLICT DO NOTHING
+    RETURNING id
+  `);
+  return (result.rows ?? []).length > 0;
+}
+
+async function recoverStaleHeartbeatRuns(
+  db: BopoDb,
+  companyId: string,
+  agentId: string,
+  staleRuns: Array<{ id: string; startedAt: Date }>,
+  input: { requestId?: string; trigger: "manual" | "scheduler"; staleRunThresholdMs: number }
+) {
+  const staleRunIds = staleRuns.map((run) => run.id);
+  if (staleRunIds.length === 0) {
+    return;
+  }
+
+  await db
+    .update(heartbeatRuns)
+    .set({
+      status: "failed",
+      finishedAt: new Date(),
+      message: "Heartbeat auto-failed after stale in-progress timeout."
+    })
+    .where(
+      and(
+        eq(heartbeatRuns.companyId, companyId),
+        eq(heartbeatRuns.agentId, agentId),
+        inArray(heartbeatRuns.id, staleRunIds),
+        eq(heartbeatRuns.status, "started")
+      )
+    );
+
+  const claimedIssueRows = await db
+    .select({ id: issues.id })
+    .from(issues)
+    .where(and(eq(issues.companyId, companyId), inArray(issues.claimedByHeartbeatRunId, staleRunIds), eq(issues.isClaimed, true)));
+  await releaseClaimedIssues(
+    db,
+    companyId,
+    claimedIssueRows.map((row) => row.id)
+  );
+
+  for (const staleRun of staleRuns) {
+    await appendAuditEvent(db, {
+      companyId,
+      actorType: "system",
+      eventType: "heartbeat.stale_recovered",
+      entityType: "heartbeat_run",
+      entityId: staleRun.id,
+      correlationId: input.requestId ?? staleRun.id,
+      payload: {
+        agentId,
+        trigger: input.trigger,
+        requestId: input.requestId ?? null,
+        staleRunThresholdMs: input.staleRunThresholdMs,
+        staleForMs: Date.now() - staleRun.startedAt.getTime()
+      }
+    });
+  }
+}
+
+export async function runHeartbeatSweep(
+  db: BopoDb,
+  companyId: string,
+  options?: { requestId?: string; realtimeHub?: RealtimeHub }
+) {
+  const companyAgents = await db.select().from(agents).where(eq(agents.companyId, companyId));
+  const recentRuns = await db
+    .select({ agentId: heartbeatRuns.agentId, startedAt: heartbeatRuns.startedAt })
+    .from(heartbeatRuns)
+    .where(eq(heartbeatRuns.companyId, companyId))
+    .orderBy(desc(heartbeatRuns.startedAt));
+  const latestRunByAgent = new Map<string, Date>();
+  for (const run of recentRuns) {
+    if (!latestRunByAgent.has(run.agentId)) {
+      latestRunByAgent.set(run.agentId, run.startedAt);
+    }
+  }
+
+  const now = new Date();
+  const runs: string[] = [];
+  let skippedNotDue = 0;
+  let skippedStatus = 0;
+  let failedStarts = 0;
+  const sweepStartedAt = Date.now();
+  for (const agent of companyAgents) {
+    if (agent.status !== "idle" && agent.status !== "running") {
+      skippedStatus += 1;
+      continue;
+    }
+    if (!isHeartbeatDue(agent.heartbeatCron, latestRunByAgent.get(agent.id) ?? null, now)) {
+      skippedNotDue += 1;
+      continue;
+    }
+    try {
+      const runId = await runHeartbeatForAgent(db, companyId, agent.id, {
+        trigger: "scheduler",
+        requestId: options?.requestId,
+        realtimeHub: options?.realtimeHub
+      });
+      if (runId) {
+        runs.push(runId);
+      }
+    } catch {
+      failedStarts += 1;
+    }
+  }
+  await appendAuditEvent(db, {
+    companyId,
+    actorType: "system",
+    eventType: "heartbeat.sweep.completed",
+    entityType: "company",
+    entityId: companyId,
+    correlationId: options?.requestId ?? null,
+    payload: {
+      runIds: runs,
+      startedCount: runs.length,
+      failedStarts,
+      skippedStatus,
+      skippedNotDue,
+      elapsedMs: Date.now() - sweepStartedAt,
+      requestId: options?.requestId ?? null
+    }
+  });
+  return runs;
+}
+
+async function buildHeartbeatContext(
+  db: BopoDb,
+  companyId: string,
+  input: {
+    agentId: string;
+    agentName: string;
+    agentRole: string;
+    managerAgentId: string | null;
+    providerType: "claude_code" | "codex" | "http" | "shell";
+    heartbeatRunId: string;
+    state: AgentState;
+    runtime?: { command?: string; args?: string[]; cwd?: string; timeoutMs?: number };
+    workItems: Array<{
+      id: string;
+      project_id: string;
+      title: string;
+      body: string | null;
+      status: string;
+      priority: string;
+      labels_json: string;
+      tags_json: string;
+    }>;
+  }
+): Promise<HeartbeatContext> {
+  const [company] = await db
+    .select({ name: companies.name, mission: companies.mission })
+    .from(companies)
+    .where(eq(companies.id, companyId))
+    .limit(1);
+  const projectIds = Array.from(new Set(input.workItems.map((item) => item.project_id)));
+  const projectRows =
+    projectIds.length > 0
+      ? await db
+          .select({ id: projects.id, name: projects.name })
+          .from(projects)
+          .where(and(eq(projects.companyId, companyId), inArray(projects.id, projectIds)))
+      : [];
+  const projectNameById = new Map(projectRows.map((row) => [row.id, row.name]));
+  const goalRows = await db
+    .select({
+      id: goals.id,
+      level: goals.level,
+      title: goals.title,
+      status: goals.status,
+      projectId: goals.projectId
+    })
+    .from(goals)
+    .where(eq(goals.companyId, companyId));
+
+  const activeCompanyGoals = goalRows
+    .filter((goal) => goal.status === "active" && goal.level === "company")
+    .map((goal) => goal.title);
+  const activeProjectGoals = goalRows
+    .filter(
+      (goal) =>
+        goal.status === "active" && goal.level === "project" && goal.projectId && projectIds.includes(goal.projectId)
+    )
+    .map((goal) => goal.title);
+  const activeAgentGoals = goalRows
+    .filter((goal) => goal.status === "active" && goal.level === "agent")
+    .map((goal) => goal.title);
+
+  return {
+    companyId,
+    agentId: input.agentId,
+    providerType: input.providerType,
+    heartbeatRunId: input.heartbeatRunId,
+    company: {
+      name: company?.name ?? "Unknown company",
+      mission: company?.mission ?? null
+    },
+    agent: {
+      name: input.agentName,
+      role: input.agentRole,
+      managerAgentId: input.managerAgentId
+    },
+    state: input.state,
+    runtime: input.runtime,
+    goalContext: {
+      companyGoals: activeCompanyGoals,
+      projectGoals: activeProjectGoals,
+      agentGoals: activeAgentGoals
+    },
+    workItems: input.workItems.map((item) => ({
+      issueId: item.id,
+      projectId: item.project_id,
+      projectName: projectNameById.get(item.project_id) ?? null,
+      title: item.title,
+      body: item.body,
+      status: item.status,
+      priority: item.priority,
+      labels: parseStringArray(item.labels_json),
+      tags: parseStringArray(item.tags_json)
+    }))
+  };
+}
+
+function parseStringArray(value: string | null) {
+  if (!value) {
+    return [];
+  }
+  try {
+    const parsed = JSON.parse(value) as unknown;
+    return Array.isArray(parsed) ? parsed.map((entry) => String(entry)) : [];
+  } catch {
+    return [];
+  }
+}
+
+function parseAgentState(stateBlob: string | null) {
+  if (!stateBlob) {
+    return { state: {} as AgentState, parseError: null };
+  }
+  try {
+    return {
+      state: JSON.parse(stateBlob) as AgentState & {
+        runtime?: {
+          command?: string;
+          args?: string[];
+          cwd?: string;
+          timeoutMs?: number;
+          retryCount?: number;
+          retryBackoffMs?: number;
+        };
+      },
+      parseError: null
+    };
+  } catch (error) {
+    return {
+      state: {} as AgentState,
+      parseError: String(error)
+    };
+  }
+}
+
+function classifyHeartbeatError(error: unknown) {
+  const message = String(error);
+  if (message.includes("ENOENT")) {
+    return { type: "runtime_missing", message };
+  }
+  if (message.includes("timeout")) {
+    return { type: "timeout", message };
+  }
+  return { type: "unknown", message };
+}
+
+function resolveStaleRunThresholdMs() {
+  const parsed = Number(process.env.BOPO_HEARTBEAT_STALE_RUN_MS ?? 10 * 60 * 1000);
+  if (!Number.isFinite(parsed) || parsed < 1_000) {
+    return 10 * 60 * 1000;
+  }
+  return parsed;
+}
+
+function isHeartbeatDue(cronExpression: string, lastRunAt: Date | null, now: Date) {
+  const normalizedNow = truncateToMinute(now);
+  if (!matchesCronExpression(cronExpression, normalizedNow)) {
+    return false;
+  }
+  if (!lastRunAt) {
+    return true;
+  }
+  return truncateToMinute(lastRunAt).getTime() !== normalizedNow.getTime();
+}
+
+function truncateToMinute(date: Date) {
+  const clone = new Date(date);
+  clone.setSeconds(0, 0);
+  return clone;
+}
+
+function matchesCronExpression(expression: string, date: Date) {
+  const parts = expression.trim().split(/\s+/);
+  if (parts.length !== 5) {
+    return false;
+  }
+
+  const [minute, hour, dayOfMonth, month, dayOfWeek] = parts as [string, string, string, string, string];
+  return (
+    matchesCronField(minute, date.getMinutes(), 0, 59) &&
+    matchesCronField(hour, date.getHours(), 0, 23) &&
+    matchesCronField(dayOfMonth, date.getDate(), 1, 31) &&
+    matchesCronField(month, date.getMonth() + 1, 1, 12) &&
+    matchesCronField(dayOfWeek, date.getDay(), 0, 6)
+  );
+}
+
+function matchesCronField(field: string, value: number, min: number, max: number) {
+  return field.split(",").some((part) => matchesCronPart(part.trim(), value, min, max));
+}
+
+function matchesCronPart(part: string, value: number, min: number, max: number): boolean {
+  if (part === "*") {
+    return true;
+  }
+
+  const stepMatch = part.match(/^\*\/(\d+)$/);
+  if (stepMatch) {
+    const step = Number(stepMatch[1]);
+    return Number.isInteger(step) && step > 0 ? (value - min) % step === 0 : false;
+  }
+
+  const rangeMatch = part.match(/^(\d+)-(\d+)$/);
+  if (rangeMatch) {
+    const start = Number(rangeMatch[1]);
+    const end = Number(rangeMatch[2]);
+    return start <= value && value <= end;
+  }
+
+  const exact = Number(part);
+  return Number.isInteger(exact) && exact >= min && exact <= max && exact === value;
+}