bopodev-api 0.1.23 → 0.1.25

package/src/services/governance-service.ts

@@ -1,10 +1,18 @@
 import { and, eq } from "drizzle-orm";
 import { mkdir } from "node:fs/promises";
 import { z } from "zod";
-import { AgentCreateRequestSchema, TemplateManifestDefault, TemplateManifestSchema } from "bopodev-contracts";
+import {
+  AGENT_ROLE_LABELS,
+  AgentCreateRequestSchema,
+  AgentRoleKeySchema,
+  TemplateManifestDefault,
+  TemplateManifestSchema
+} from "bopodev-contracts";
 import type { BopoDb } from "bopodev-db";
 import {
   approvalRequests,
+  agents,
+  appendAuditEvent,
   createAgent,
   createGoal,
   createIssue,
@@ -107,6 +115,32 @@ const applyTemplatePayloadSchema = z.object({
   templateVersion: z.string().min(1),
   variables: z.record(z.string(), z.unknown()).default({})
 });
+const overrideBudgetPayloadSchema = z
+  .object({
+    agentId: z.string().min(1).optional(),
+    projectId: z.string().min(1).optional(),
+    reason: z.string().optional(),
+    additionalBudgetUsd: z.number().positive().optional(),
+    revisedMonthlyBudgetUsd: z.number().positive().optional()
+  })
+  .refine(
+    (value) => Boolean(value.agentId) || Boolean(value.projectId),
+    "Budget override payload requires agentId or projectId."
+  )
+  .refine(
+    (value) => !(value.agentId && value.projectId),
+    "Budget override payload must target either agentId or projectId, not both."
+  )
+  .refine(
+    (value) =>
+      (typeof value.additionalBudgetUsd === "number" && value.additionalBudgetUsd > 0) ||
+      (typeof value.revisedMonthlyBudgetUsd === "number" && value.revisedMonthlyBudgetUsd > 0),
+    "Budget override payload requires additionalBudgetUsd or revisedMonthlyBudgetUsd."
+  );
+const pauseOrTerminateAgentPayloadSchema = z.object({
+  agentId: z.string().min(1),
+  reason: z.string().optional()
+});
 const AGENT_STARTUP_PROJECT_NAME = "Agent Onboarding";
 const AGENT_STARTUP_TASK_MARKER = "[bopodev:onboarding:agent-startup:v1]";
 
@@ -153,7 +187,7 @@ export async function resolveApproval(
     let execution:
       | {
           applied: boolean;
-          entityType?: "agent" | "goal" | "memory" | "template";
+          entityType?: "agent" | "goal" | "project" | "memory" | "template";
           entityId?: string;
           entity?: Record<string, unknown>;
         }
@@ -255,7 +289,8 @@ async function applyApprovalAction(db: BopoDb, companyId: string, action: string
     const existingAgents = await listAgents(db, companyId);
     const duplicate = existingAgents.find(
       (agent) =>
-        agent.role === parsed.data.role &&
+        ((parsed.data.roleKey && agent.roleKey === parsed.data.roleKey) ||
+          (!parsed.data.roleKey && parsed.data.role && agent.role === parsed.data.role)) &&
         (agent.managerAgentId ?? null) === (parsed.data.managerAgentId ?? null) &&
         agent.status !== "terminated"
     );
@@ -271,7 +306,9 @@ async function applyApprovalAction(db: BopoDb, companyId: string, action: string
     const agent = await createAgent(db, {
       companyId,
       managerAgentId: parsed.data.managerAgentId,
-      role: parsed.data.role,
+      role: resolveAgentRoleText(parsed.data.role, parsed.data.roleKey, parsed.data.title),
+      roleKey: normalizeRoleKey(parsed.data.roleKey),
+      title: normalizeTitle(parsed.data.title),
       name: parsed.data.name,
       providerType: parsed.data.providerType,
       heartbeatCron: parsed.data.heartbeatCron,
@@ -281,7 +318,13 @@ async function applyApprovalAction(db: BopoDb, companyId: string, action: string
       initialState: runtimeConfigToStateBlobPatch(runtimeConfig)
     });
     const startupProjectId = await ensureAgentStartupProject(db, companyId);
-    await ensureAgentStartupIssue(db, companyId, startupProjectId, agent.id, agent.role);
+    await ensureAgentStartupIssue(
+      db,
+      companyId,
+      startupProjectId,
+      agent.id,
+      resolveAgentDisplayTitle(agent.title, agent.roleKey, agent.role)
+    );
 
     return {
       applied: true,
@@ -347,8 +390,144 @@ async function applyApprovalAction(db: BopoDb, companyId: string, action: string
     };
   }
 
-  if (action === "pause_agent" || action === "terminate_agent" || action === "override_budget") {
-    return { applied: false };
+  if (action === "override_budget") {
+    const parsed = overrideBudgetPayloadSchema.safeParse(payload);
+    if (!parsed.success) {
+      throw new GovernanceError("Approval payload for budget override is invalid.");
+    }
+    if (parsed.data.agentId) {
+      const [agent] = await db
+        .select({
+          id: agents.id,
+          monthlyBudgetUsd: agents.monthlyBudgetUsd,
+          usedBudgetUsd: agents.usedBudgetUsd
+        })
+        .from(agents)
+        .where(and(eq(agents.companyId, companyId), eq(agents.id, parsed.data.agentId)))
+        .limit(1);
+      if (!agent) {
+        throw new GovernanceError("Agent not found for budget override request.");
+      }
+      const currentMonthlyBudget = Number(agent.monthlyBudgetUsd);
+      const currentUsedBudget = Number(agent.usedBudgetUsd);
+      const nextMonthlyBudget =
+        typeof parsed.data.revisedMonthlyBudgetUsd === "number"
+          ? parsed.data.revisedMonthlyBudgetUsd
+          : currentMonthlyBudget + (parsed.data.additionalBudgetUsd ?? 0);
+      if (!Number.isFinite(nextMonthlyBudget) || nextMonthlyBudget <= 0) {
+        throw new GovernanceError("Budget override must resolve to a positive monthly budget.");
+      }
+      if (nextMonthlyBudget <= currentUsedBudget) {
+        throw new GovernanceError("Budget override must exceed current used budget.");
+      }
+      await db
+        .update(agents)
+        .set({
+          monthlyBudgetUsd: nextMonthlyBudget.toFixed(4),
+          updatedAt: new Date()
+        })
+        .where(and(eq(agents.companyId, companyId), eq(agents.id, parsed.data.agentId)));
+      return {
+        applied: true,
+        entityType: "agent" as const,
+        entityId: parsed.data.agentId,
+        entity: {
+          agentId: parsed.data.agentId,
+          previousMonthlyBudgetUsd: currentMonthlyBudget,
+          monthlyBudgetUsd: nextMonthlyBudget,
+          usedBudgetUsd: currentUsedBudget,
+          reason: parsed.data.reason ?? null
+        }
+      };
+    }
+    const [project] = await db
+      .select({
+        id: projects.id,
+        monthlyBudgetUsd: projects.monthlyBudgetUsd,
+        usedBudgetUsd: projects.usedBudgetUsd
+      })
+      .from(projects)
+      .where(and(eq(projects.companyId, companyId), eq(projects.id, parsed.data.projectId!)))
+      .limit(1);
+    if (!project) {
+      throw new GovernanceError("Project not found for budget override request.");
+    }
+    const currentMonthlyBudget = Number(project.monthlyBudgetUsd);
+    const currentUsedBudget = Number(project.usedBudgetUsd);
+    const nextMonthlyBudget =
+      typeof parsed.data.revisedMonthlyBudgetUsd === "number"
+        ? parsed.data.revisedMonthlyBudgetUsd
+        : currentMonthlyBudget + (parsed.data.additionalBudgetUsd ?? 0);
+    if (!Number.isFinite(nextMonthlyBudget) || nextMonthlyBudget <= 0) {
+      throw new GovernanceError("Budget override must resolve to a positive monthly budget.");
+    }
+    if (nextMonthlyBudget <= currentUsedBudget) {
+      throw new GovernanceError("Budget override must exceed current used budget.");
+    }
+    await db
+      .update(projects)
+      .set({
+        monthlyBudgetUsd: nextMonthlyBudget.toFixed(4),
+        updatedAt: new Date()
+      })
+      .where(and(eq(projects.companyId, companyId), eq(projects.id, parsed.data.projectId!)));
+    await appendAuditEvent(db, {
+      companyId,
+      actorType: "system",
+      eventType: "project_budget.override_applied",
+      entityType: "project",
+      entityId: parsed.data.projectId!,
+      payload: {
+        previousMonthlyBudgetUsd: currentMonthlyBudget,
+        monthlyBudgetUsd: nextMonthlyBudget,
+        usedBudgetUsd: currentUsedBudget,
+        reason: parsed.data.reason ?? null
+      }
+    });
+    return {
+      applied: true,
+      entityType: "project" as const,
+      entityId: parsed.data.projectId!,
+      entity: {
+        projectId: parsed.data.projectId!,
+        previousMonthlyBudgetUsd: currentMonthlyBudget,
+        monthlyBudgetUsd: nextMonthlyBudget,
+        usedBudgetUsd: currentUsedBudget,
+        reason: parsed.data.reason ?? null
+      }
+    };
+  }
+
+  if (action === "pause_agent" || action === "terminate_agent") {
+    const parsed = pauseOrTerminateAgentPayloadSchema.safeParse(payload);
+    if (!parsed.success) {
+      throw new GovernanceError(`Approval payload for ${action} is invalid.`);
+    }
+    const nextStatus = action === "pause_agent" ? "paused" : "terminated";
+    const [updated] = await db
+      .update(agents)
+      .set({
+        status: nextStatus,
+        updatedAt: new Date()
+      })
+      .where(and(eq(agents.companyId, companyId), eq(agents.id, parsed.data.agentId)))
+      .returning({
+        id: agents.id,
+        status: agents.status
+      });
+    if (!updated) {
+      throw new GovernanceError("Agent not found for lifecycle governance action.");
+    }
+    return {
+      applied: true,
+      entityType: "agent" as const,
+      entityId: updated.id,
+      entity: {
+        id: updated.id,
+        status: updated.status,
+        reason: parsed.data.reason ?? null
+      }
+    };
   }
 
   if (action === "promote_memory_fact") {
@@ -516,9 +695,9 @@ async function ensureAgentStartupIssue(
   companyId: string,
   projectId: string,
   agentId: string,
-  role: string
+  roleTitle: string
 ) {
-  const title = `Set up ${role} operating files`;
+  const title = `Set up ${roleTitle} operating files`;
   const body = buildAgentStartupTaskBody(companyId, agentId);
   const existingIssues = await listIssues(db, companyId);
   const existing = existingIssues.find(
@@ -545,6 +724,51 @@ async function ensureAgentStartupIssue(
   return created.id;
 }
 
+function normalizeRoleKey(input: string | null | undefined) {
+  const normalized = input?.trim().toLowerCase();
+  if (!normalized) {
+    return null;
+  }
+  return AgentRoleKeySchema.safeParse(normalized).success ? normalized : null;
+}
+
+function normalizeTitle(input: string | null | undefined) {
+  const normalized = input?.trim();
+  return normalized ? normalized : null;
+}
+
+function resolveAgentRoleText(
+  legacyRole: string | undefined,
+  roleKeyInput: string | undefined,
+  titleInput: string | null | undefined
+) {
+  const normalizedLegacy = legacyRole?.trim();
+  if (normalizedLegacy) {
+    return normalizedLegacy;
+  }
+  const normalizedTitle = normalizeTitle(titleInput);
+  if (normalizedTitle) {
+    return normalizedTitle;
+  }
+  const roleKey = normalizeRoleKey(roleKeyInput);
+  if (roleKey) {
+    return AGENT_ROLE_LABELS[roleKey as keyof typeof AGENT_ROLE_LABELS];
+  }
+  return AGENT_ROLE_LABELS.general;
+}
+
+function resolveAgentDisplayTitle(title: string | null | undefined, roleKeyInput: string | null | undefined, role: string) {
+  const normalizedTitle = normalizeTitle(title);
+  if (normalizedTitle) {
+    return normalizedTitle;
+  }
+  const roleKey = normalizeRoleKey(roleKeyInput);
+  if (roleKey) {
+    return AGENT_ROLE_LABELS[roleKey as keyof typeof AGENT_ROLE_LABELS];
+  }
+  return role;
+}
+
 function buildAgentStartupTaskBody(companyId: string, agentId: string) {
   const agentWorkspaceRoot = resolveAgentFallbackWorkspacePath(companyId, agentId);
   const agentOperatingFolder = `${agentWorkspaceRoot}/operating`;

package/src/services/heartbeat-queue-service.ts

@@ -0,0 +1,318 @@
+import { and, eq } from "drizzle-orm";
+import {
+  cancelHeartbeatJob,
+  getHeartbeatRun,
+  claimNextHeartbeatJob,
+  enqueueHeartbeatJob,
+  issueComments,
+  markHeartbeatJobCompleted,
+  markHeartbeatJobDeadLetter,
+  markHeartbeatJobRetry,
+  updateIssueCommentRecipients,
+  type BopoDb
+} from "bopodev-db";
+import { parseIssueCommentRecipients } from "../lib/comment-recipients";
+import type { RealtimeHub } from "../realtime/hub";
+import { runHeartbeatForAgent } from "./heartbeat-service";
+
+type QueueJobPayload = {
+  sourceRunId?: string;
+  wakeContext?: {
+    reason?: string | null;
+    commentId?: string | null;
+    issueIds?: string[];
+  };
+  commentDispatch?: {
+    commentId: string;
+    issueId: string;
+    recipientId: string;
+  };
+};
+
+const activeCompanyQueueWorkers = new Set<string>();
+
+export async function enqueueHeartbeatQueueJob(
+  db: BopoDb,
+  input: {
+    companyId: string;
+    agentId: string;
+    jobType: "manual" | "scheduler" | "resume" | "redo" | "comment_dispatch";
+    payload?: QueueJobPayload;
+    priority?: number;
+    idempotencyKey?: string | null;
+    maxAttempts?: number;
+    availableAt?: Date;
+  }
+) {
+  return enqueueHeartbeatJob(db, {
+    companyId: input.companyId,
+    agentId: input.agentId,
+    jobType: input.jobType,
+    payload: input.payload ?? {},
+    priority: input.priority ?? 100,
+    idempotencyKey: input.idempotencyKey ?? null,
+    maxAttempts: input.maxAttempts ?? 10,
+    availableAt: input.availableAt
+  });
+}
+
+export function triggerHeartbeatQueueWorker(
+  db: BopoDb,
+  companyId: string,
+  options?: { requestId?: string; realtimeHub?: RealtimeHub; maxJobsPerSweep?: number }
+) {
+  if (activeCompanyQueueWorkers.has(companyId)) {
+    return;
+  }
+  activeCompanyQueueWorkers.add(companyId);
+  queueMicrotask(() => {
+    void runHeartbeatQueueSweep(db, companyId, options)
+      .catch((error) => {
+        // eslint-disable-next-line no-console
+        console.error("[heartbeat-queue] worker run failed", error);
+      })
+      .finally(() => {
+        activeCompanyQueueWorkers.delete(companyId);
+      });
+  });
+}
+
+export async function runHeartbeatQueueSweep(
+  db: BopoDb,
+  companyId: string,
+  options?: { requestId?: string; realtimeHub?: RealtimeHub; maxJobsPerSweep?: number }
+) {
+  const maxJobs = Math.max(1, Math.min(options?.maxJobsPerSweep ?? 50, 500));
+  let processed = 0;
+  while (processed < maxJobs) {
+    const job = await claimNextHeartbeatJob(db, companyId);
+    if (!job) {
+      break;
+    }
+    processed += 1;
+    try {
+      await processHeartbeatQueueJob(db, {
+        companyId,
+        job: {
+          id: job.id,
+          agentId: job.agentId,
+          jobType: job.jobType as "manual" | "scheduler" | "resume" | "redo" | "comment_dispatch",
+          attemptCount: job.attemptCount,
+          maxAttempts: job.maxAttempts,
+          payload: (job.payload ?? {}) as QueueJobPayload
+        },
+        requestId: options?.requestId,
+        realtimeHub: options?.realtimeHub
+      });
+    } catch (error) {
+      await handleQueueRetryOrDeadLetter(db, {
+        companyId,
+        jobId: job.id,
+        attemptCount: job.attemptCount,
+        maxAttempts: job.maxAttempts,
+        error: `Queue processing failed: ${String((error as Error)?.message ?? error)}`
+      });
+    }
+  }
+  return { processed };
+}
+
+async function processHeartbeatQueueJob(
+  db: BopoDb,
+  input: {
+    companyId: string;
+    job: {
+      id: string;
+      agentId: string;
+      jobType: "manual" | "scheduler" | "resume" | "redo" | "comment_dispatch";
+      attemptCount: number;
+      maxAttempts: number;
+      payload: QueueJobPayload;
+    };
+    requestId?: string;
+    realtimeHub?: RealtimeHub;
+  }
+) {
+  let runId: string | null = null;
+  try {
+    runId = await runHeartbeatForAgent(db, input.companyId, input.job.agentId, {
+      trigger: input.job.jobType === "manual" || input.job.jobType === "resume" || input.job.jobType === "redo" ? "manual" : "scheduler",
+      requestId: input.requestId,
+      realtimeHub: input.realtimeHub,
+      ...(input.job.jobType === "resume" || input.job.jobType === "redo" ? { mode: input.job.jobType } : {}),
+      ...(input.job.payload.sourceRunId ? { sourceRunId: input.job.payload.sourceRunId } : {}),
+      ...(input.job.payload.wakeContext ? { wakeContext: input.job.payload.wakeContext } : {})
+    });
+  } catch (error) {
+    await handleQueueRetryOrDeadLetter(db, {
+      companyId: input.companyId,
+      jobId: input.job.id,
+      attemptCount: input.job.attemptCount,
+      maxAttempts: input.job.maxAttempts,
+      error: `Heartbeat executor failed: ${String((error as Error)?.message ?? error)}`
+    });
+    return;
+  }
+
+  if (!runId) {
+    await handleQueueRetryOrDeadLetter(db, {
+      companyId: input.companyId,
+      jobId: input.job.id,
+      attemptCount: input.job.attemptCount,
+      maxAttempts: input.job.maxAttempts,
+      error: "Heartbeat execution returned no run id."
+    });
+    return;
+  }
+
+  const run = await getHeartbeatRun(db, input.companyId, runId);
+  if (!run) {
+    await handleQueueRetryOrDeadLetter(db, {
+      companyId: input.companyId,
+      jobId: input.job.id,
+      attemptCount: input.job.attemptCount,
+      maxAttempts: input.job.maxAttempts,
+      heartbeatRunId: runId,
+      error: "Heartbeat run record was not found after execution."
+    });
+    return;
+  }
+
+  if (run.status === "skipped" && String(run.message ?? "").toLowerCase().includes("already in progress")) {
+    await markHeartbeatJobRetry(db, {
+      companyId: input.companyId,
+      id: input.job.id,
+      heartbeatRunId: runId,
+      retryAt: new Date(Date.now() + resolveRetryDelayMs(input.job.attemptCount)),
+      error: "Agent busy, retry queued."
+    });
+    return;
+  }
+
+  if (run.status === "skipped") {
+    if (isProjectBudgetHardStopMessage(run.message)) {
+      await cancelHeartbeatJob(db, {
+        companyId: input.companyId,
+        id: input.job.id
+      });
+      return;
+    }
+    await handleQueueRetryOrDeadLetter(db, {
+      companyId: input.companyId,
+      jobId: input.job.id,
+      attemptCount: input.job.attemptCount,
+      maxAttempts: input.job.maxAttempts,
+      heartbeatRunId: runId,
+      error: run.message ?? "Heartbeat skipped."
+    });
+    await markCommentRecipientDeliveryIfNeeded(db, input.companyId, input.job.payload.commentDispatch, {
+      deliveryStatus: "failed",
+      dispatchedRunId: null,
+      dispatchedAt: null
+    });
+    return;
+  }
+
+  await markHeartbeatJobCompleted(db, {
+    companyId: input.companyId,
+    id: input.job.id,
+    heartbeatRunId: runId
+  });
+  await markCommentRecipientDeliveryIfNeeded(db, input.companyId, input.job.payload.commentDispatch, {
+    deliveryStatus: "dispatched",
+    dispatchedRunId: runId,
+    dispatchedAt: new Date().toISOString()
+  });
+}
+
+async function handleQueueRetryOrDeadLetter(
+  db: BopoDb,
+  input: {
+    companyId: string;
+    jobId: string;
+    attemptCount: number;
+    maxAttempts: number;
+    error: string;
+    heartbeatRunId?: string;
+  }
+) {
+  if (input.attemptCount >= input.maxAttempts) {
+    await markHeartbeatJobDeadLetter(db, {
+      companyId: input.companyId,
+      id: input.jobId,
+      heartbeatRunId: input.heartbeatRunId ?? null,
+      error: input.error
+    });
+    return;
+  }
+  await markHeartbeatJobRetry(db, {
+    companyId: input.companyId,
+    id: input.jobId,
+    heartbeatRunId: input.heartbeatRunId ?? null,
+    retryAt: new Date(Date.now() + resolveRetryDelayMs(input.attemptCount)),
+    error: input.error
+  });
+}
+
+function resolveRetryDelayMs(attemptCount: number) {
+  const baseDelayMs = Number(process.env.BOPO_HEARTBEAT_QUEUE_RETRY_BASE_MS ?? 1000);
+  const cappedAttempt = Math.max(0, Math.min(attemptCount, 8));
+  return Math.max(500, baseDelayMs) * 2 ** cappedAttempt;
+}
+
+function isProjectBudgetHardStopMessage(message: string | null | undefined) {
+  const normalized = String(message ?? "")
+    .trim()
+    .toLowerCase();
+  return normalized.includes("project budget hard-stop");
+}
+
+async function markCommentRecipientDeliveryIfNeeded(
+  db: BopoDb,
+  companyId: string,
+  dispatch: QueueJobPayload["commentDispatch"] | undefined,
+  input: {
+    deliveryStatus: "dispatched" | "failed";
+    dispatchedRunId: string | null;
+    dispatchedAt: string | null;
+  }
+) {
+  if (!dispatch) {
+    return;
+  }
+  const [comment] = await db
+    .select({ recipientsJson: issueComments.recipientsJson })
+    .from(issueComments)
+    .where(and(eq(issueComments.companyId, companyId), eq(issueComments.id, dispatch.commentId)))
+    .limit(1);
+  if (!comment) {
+    return;
+  }
+  const recipients = parseIssueCommentRecipients(comment.recipientsJson);
+  let changed = false;
+  const updatedRecipients = recipients.map((recipient) => {
+    if (recipient.recipientType !== "agent" || recipient.recipientId !== dispatch.recipientId) {
+      return recipient;
+    }
+    if (recipient.deliveryStatus !== "pending") {
+      return recipient;
+    }
+    changed = true;
+    return {
+      ...recipient,
+      deliveryStatus: input.deliveryStatus,
+      dispatchedRunId: input.dispatchedRunId,
+      dispatchedAt: input.dispatchedAt
+    };
+  });
+  if (!changed) {
+    return;
+  }
+  await updateIssueCommentRecipients(db, {
+    companyId,
+    issueId: dispatch.issueId,
+    id: dispatch.commentId,
+    recipients: updatedRecipients
+  });
+}
+