bopodev-api 0.1.26 → 0.1.28

package/src/server.ts

@@ -1,9 +1,8 @@
 import { createServer } from "node:http";
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
-import { sql } from "drizzle-orm";
 import { config as loadDotenv } from "dotenv";
-import { bootstrapDatabase, listCompanies } from "bopodev-db";
+import { bootstrapDatabase, listCompanies, resolveDefaultDbPath, sql } from "bopodev-db";
 import { checkRuntimeCommandHealth } from "bopodev-agent-sdk";
 import type { RuntimeCommandHealth } from "bopodev-agent-sdk";
 import { createApp } from "./app";
@@ -21,6 +20,8 @@ import {
 } from "./security/deployment-mode";
 import { ensureBuiltinPluginsRegistered } from "./services/plugin-runtime";
 import { ensureBuiltinTemplatesRegistered } from "./services/template-catalog";
+import { beginIssueCommentDispatchShutdown, waitForIssueCommentDispatchDrain } from "./services/comment-recipient-dispatch-service";
+import { beginHeartbeatQueueShutdown, waitForHeartbeatQueueDrain } from "./services/heartbeat-queue-service";
 import { createHeartbeatScheduler } from "./worker/scheduler";
 
 loadApiEnv();
@@ -32,8 +33,33 @@ async function main() {
   const publicBaseUrl = resolvePublicBaseUrl();
   validateDeploymentConfiguration(deploymentMode, allowedOrigins, allowedHostnames, publicBaseUrl);
   const dbPath = normalizeOptionalDbPath(process.env.BOPO_DB_PATH);
+  const usingExternalDatabase = Boolean(process.env.DATABASE_URL?.trim());
   const port = Number(process.env.PORT ?? 4020);
-  const { db } = await bootstrapDatabase(dbPath);
+  const effectiveDbPath = dbPath ?? resolveDefaultDbPath();
+  let db: Awaited<ReturnType<typeof bootstrapDatabase>>["db"];
+  let dbClient: Awaited<ReturnType<typeof bootstrapDatabase>>["client"];
+  try {
+    const boot = await bootstrapDatabase(dbPath);
+    db = boot.db;
+    dbClient = boot.client;
+  } catch (error) {
+    if (isProbablyDatabaseStartupError(error)) {
+      // eslint-disable-next-line no-console
+      console.error("[startup] Database bootstrap failed before the API could start.");
+      if (usingExternalDatabase) {
+        // eslint-disable-next-line no-console
+        console.error("[startup] Check DATABASE_URL connectivity, permissions, and migration state.");
+      } else {
+        // eslint-disable-next-line no-console
+        console.error(`[startup] Embedded Postgres data path: ${effectiveDbPath}`);
+        // eslint-disable-next-line no-console
+        console.error(
+          "[startup] Recovery: stop all API/node processes using this DB, back up the path above, delete the directory if it is corrupted, then restart. Or set BOPO_DB_PATH to a fresh path."
+        );
+      }
+    }
+    throw error;
+  }
   const existingCompanies = await listCompanies(db);
   await ensureBuiltinPluginsRegistered(
     db,
@@ -120,16 +146,81 @@ async function main() {
 
   const defaultCompanyId = process.env.BOPO_DEFAULT_COMPANY_ID;
   const schedulerCompanyId = await resolveSchedulerCompanyId(db, defaultCompanyId ?? null);
+  let scheduler: ReturnType<typeof createHeartbeatScheduler> | undefined;
   if (schedulerCompanyId && shouldStartScheduler()) {
-    createHeartbeatScheduler(db, schedulerCompanyId, realtimeHub);
+    scheduler = createHeartbeatScheduler(db, schedulerCompanyId, realtimeHub);
   } else if (schedulerCompanyId) {
     // eslint-disable-next-line no-console
     console.log("[startup] Scheduler disabled for this instance (BOPO_SCHEDULER_ROLE is follower/off).");
   }
+
+  let shutdownInFlight: Promise<void> | null = null;
+  function shutdown(signal: string) {
+    const shutdownTimeoutMs = Number(process.env.BOPO_SHUTDOWN_TIMEOUT_MS ?? 15_000);
+    const forcedExit = setTimeout(() => {
+      // eslint-disable-next-line no-console
+      console.error(`[shutdown] timed out after ${shutdownTimeoutMs}ms; forcing exit.`);
+      process.exit(process.exitCode ?? 1);
+    }, shutdownTimeoutMs);
+    forcedExit.unref();
+    shutdownInFlight ??= (async () => {
+      // eslint-disable-next-line no-console
+      console.log(`[shutdown] ${signal} — draining HTTP/background work before closing the embedded database…`);
+      beginHeartbeatQueueShutdown();
+      beginIssueCommentDispatchShutdown();
+      await Promise.allSettled([
+        scheduler?.stop() ?? Promise.resolve(),
+        new Promise<void>((resolve, reject) => {
+          server.close((err) => {
+            if (err) {
+              reject(err);
+              return;
+            }
+            resolve();
+          });
+        })
+      ]);
+      await Promise.allSettled([waitForHeartbeatQueueDrain(), waitForIssueCommentDispatchDrain()]);
+      try {
+        await realtimeHub.close();
+      } catch (closeError) {
+        // eslint-disable-next-line no-console
+        console.error("[shutdown] realtime hub close error", closeError);
+      }
+      try {
+        await closeDatabaseClient(dbClient);
+      } catch (closeDbError) {
+        // eslint-disable-next-line no-console
+        console.error("[shutdown] database close error", closeDbError);
+      }
+      // eslint-disable-next-line no-console
+      console.log("[shutdown] clean exit");
+      process.exitCode = 0;
+    })().catch((error) => {
+      // eslint-disable-next-line no-console
+      console.error("[shutdown] failed", error);
+      process.exitCode = 1;
+    });
+    return shutdownInFlight;
+  }
+
+  process.once("SIGINT", () => void shutdown("SIGINT"));
+  process.once("SIGTERM", () => void shutdown("SIGTERM"));
 }
 
 void main();
 
+async function closeDatabaseClient(client: unknown) {
+  if (!client || typeof client !== "object") {
+    return;
+  }
+  const closeFn = (client as { close?: unknown }).close;
+  if (typeof closeFn !== "function") {
+    return;
+  }
+  await (closeFn as () => Promise<void>)();
+}
+
 async function hasCodexAgentsConfigured(db: Awaited<ReturnType<typeof bootstrapDatabase>>["db"]) {
   const result = await db.execute(sql`
     SELECT id
@@ -137,7 +228,7 @@ async function hasCodexAgentsConfigured(db: Awaited<ReturnType<typeof bootstrapD
     WHERE provider_type = 'codex'
     LIMIT 1
   `);
-  return (result.rows ?? []).length > 0;
+  return result.length > 0;
 }
 
 async function hasOpenCodeAgentsConfigured(db: Awaited<ReturnType<typeof bootstrapDatabase>>["db"]) {
@@ -147,7 +238,7 @@ async function hasOpenCodeAgentsConfigured(db: Awaited<ReturnType<typeof bootstr
     WHERE provider_type = 'opencode'
     LIMIT 1
   `);
-  return (result.rows ?? []).length > 0;
+  return result.length > 0;
 }
 
 async function resolveSchedulerCompanyId(
@@ -161,7 +252,7 @@ async function resolveSchedulerCompanyId(
       WHERE id = ${configuredCompanyId}
       LIMIT 1
     `);
-    if ((configured.rows ?? []).length > 0) {
+    if (configured.length > 0) {
       return configuredCompanyId;
     }
     // eslint-disable-next-line no-console
@@ -174,7 +265,7 @@ async function resolveSchedulerCompanyId(
     ORDER BY created_at ASC
     LIMIT 1
   `);
-  const id = fallback.rows?.[0]?.id;
+  const id = fallback[0]?.id;
   return typeof id === "string" && id.length > 0 ? id : null;
 }
 
@@ -262,3 +353,16 @@ function normalizeOptionalDbPath(value: string | undefined) {
   const normalized = value?.trim();
   return normalized && normalized.length > 0 ? normalized : undefined;
 }
+
+function isProbablyDatabaseStartupError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  const cause = error instanceof Error ? error.cause : undefined;
+  const causeMessage = cause instanceof Error ? cause.message : String(cause ?? "");
+  return (
+    message.includes("database") ||
+    message.includes("postgres") ||
+    message.includes("migration") ||
+    causeMessage.includes("postgres") ||
+    causeMessage.includes("connection")
+  );
+}

package/src/services/attention-service.ts

@@ -1,7 +1,10 @@
-import { and, desc, eq, like } from "drizzle-orm";
 import {
+  and,
   issueComments,
   issues,
+  desc,
+  eq,
+  like,
   listApprovalRequests,
   listAttentionInboxStates,
   listHeartbeatRuns,
@@ -17,6 +20,40 @@ import type { BoardAttentionItem } from "bopodev-contracts";
 
 type AttentionStateRow = Awaited<ReturnType<typeof listAttentionInboxStates>>[number];
 
+/** Keep in sync with `RESOLVED_APPROVAL_INBOX_WINDOW_DAYS` in governance routes (resolved history in Inbox). */
+const RESOLVED_APPROVAL_ATTENTION_WINDOW_MS = 30 * 24 * 60 * 60 * 1000;
+
+type StoredApprovalRow = Awaited<ReturnType<typeof listApprovalRequests>>[number];
+
+function approvalIncludedInAttentionList(approval: Pick<StoredApprovalRow, "status" | "resolvedAt">): boolean {
+  if (approval.status === "pending") {
+    return true;
+  }
+  if (!approval.resolvedAt) {
+    return false;
+  }
+  return Date.now() - approval.resolvedAt.getTime() <= RESOLVED_APPROVAL_ATTENTION_WINDOW_MS;
+}
+
+function finalizeApprovalAttentionItem(item: BoardAttentionItem, approval: Pick<StoredApprovalRow, "status" | "resolvedAt" | "action">): BoardAttentionItem {
+  if (approval.status === "pending") {
+    return item;
+  }
+  const outcome =
+    approval.status === "approved" ? "Approved" : approval.status === "rejected" ? "Rejected" : "Overridden";
+  const title =
+    approval.action === "override_budget" ? `Budget hard-stop · ${outcome}` : `Approval · ${outcome}`;
+  const resolvedAtIso = approval.resolvedAt?.toISOString() ?? item.resolvedAt;
+  return {
+    ...item,
+    title,
+    state: "resolved",
+    resolvedAt: resolvedAtIso,
+    severity: "info",
+    sourceTimestamp: resolvedAtIso ?? item.sourceTimestamp
+  };
+}
+
 export async function listBoardAttentionItems(db: BopoDb, companyId: string, actorId: string): Promise<BoardAttentionItem[]> {
   const [approvals, blockedIssues, heartbeatRuns, stateRows, boardComments] = await Promise.all([
     listApprovalRequests(db, companyId),
@@ -42,7 +79,7 @@ export async function listBoardAttentionItems(db: BopoDb, companyId: string, act
   const items: BoardAttentionItem[] = [];
 
   for (const approval of approvals) {
-    if (approval.status !== "pending") {
+    if (!approvalIncludedInAttentionList(approval)) {
       continue;
     }
     const payload = parsePayload(approval.payloadJson);
@@ -55,25 +92,61 @@ export async function listBoardAttentionItems(db: BopoDb, companyId: string, act
       const usedBudget = asNumber(payload.usedBudgetUsd);
       const key = `budget:${approval.id}`;
       items.push(
+        finalizeApprovalAttentionItem(
+          withState(
+            {
+              key,
+              category: "budget_hard_stop",
+              severity: ageHours >= 12 ? "critical" : "warning",
+              requiredActor: "board",
+              title: "Budget hard-stop requires board decision",
+              contextSummary: projectId
+                ? `Project ${shortId(projectId)} is blocked by budget hard-stop.`
+                : agentId
+                  ? `Agent ${shortId(agentId)} is blocked by budget hard-stop.`
+                  : "Agent work is blocked by budget hard-stop.",
+              actionLabel: "Review budget override",
+              actionHref: "/governance",
+              impactSummary: "Heartbeat work stays paused until budget override is approved or rejected.",
+              evidence: {
+                approvalId: approval.id,
+                projectId: projectId ?? undefined,
+                agentId: agentId ?? undefined
+              },
+              sourceTimestamp: approval.createdAt.toISOString(),
+              state: "open",
+              seenAt: null,
+              acknowledgedAt: null,
+              dismissedAt: null,
+              resolvedAt: null
+            },
+            stateByKey.get(key),
+            `Budget utilization ${formatPercent(utilizationPct)} (${formatUsd(usedBudget)} / ${formatUsd(currentBudget)}).`
+          ),
+          approval
+        )
+      );
+      continue;
+    }
+
+    const key = `approval:${approval.id}`;
+    items.push(
+      finalizeApprovalAttentionItem(
         withState(
           {
             key,
-            category: "budget_hard_stop",
-            severity: ageHours >= 12 ? "critical" : "warning",
+            category: "approval_required",
+            severity: ageHours >= 24 ? "critical" : "warning",
             requiredActor: "board",
-            title: "Budget hard-stop requires board decision",
-            contextSummary: projectId
-              ? `Project ${shortId(projectId)} is blocked by budget hard-stop.`
-              : agentId
-                ? `Agent ${shortId(agentId)} is blocked by budget hard-stop.`
-                : "Agent work is blocked by budget hard-stop.",
-            actionLabel: "Review budget override",
+            title: "Approval required",
+            contextSummary: formatApprovalContext(approval.action, payload),
+            actionLabel: "Open approvals",
             actionHref: "/governance",
-            impactSummary: "Heartbeat work stays paused until budget override is approved or rejected.",
+            impactSummary: "Execution remains blocked until this governance decision is resolved.",
             evidence: {
               approvalId: approval.id,
-              projectId: projectId ?? undefined,
-              agentId: agentId ?? undefined
+              projectId: asString(payload.projectId) ?? undefined,
+              agentId: asString(payload.agentId) ?? undefined
             },
             sourceTimestamp: approval.createdAt.toISOString(),
             state: "open",
@@ -82,39 +155,9 @@ export async function listBoardAttentionItems(db: BopoDb, companyId: string, act
             dismissedAt: null,
             resolvedAt: null
           },
-          stateByKey.get(key),
-          `Budget utilization ${formatPercent(utilizationPct)} (${formatUsd(usedBudget)} / ${formatUsd(currentBudget)}).`
-        )
-      );
-      continue;
-    }
-
-    const key = `approval:${approval.id}`;
-    items.push(
-      withState(
-        {
-          key,
-          category: "approval_required",
-          severity: ageHours >= 24 ? "critical" : "warning",
-          requiredActor: "board",
-          title: "Approval required",
-          contextSummary: formatApprovalContext(approval.action, payload),
-          actionLabel: "Open approvals",
-          actionHref: "/governance",
-          impactSummary: "Execution remains blocked until this governance decision is resolved.",
-          evidence: {
-            approvalId: approval.id,
-            projectId: asString(payload.projectId) ?? undefined,
-            agentId: asString(payload.agentId) ?? undefined
-          },
-          sourceTimestamp: approval.createdAt.toISOString(),
-          state: "open",
-          seenAt: null,
-          acknowledgedAt: null,
-          dismissedAt: null,
-          resolvedAt: null
-        },
-        stateByKey.get(key)
+          stateByKey.get(key)
+        ),
+        approval
       )
     );
   }

package/src/services/budget-service.ts

@@ -1,6 +1,5 @@
-import { and, eq, sql } from "drizzle-orm";
 import type { BopoDb } from "bopodev-db";
-import { agents, projects } from "bopodev-db";
+import { agents, and, eq, projects, sql } from "bopodev-db";
 
 export interface BudgetCheckResult {
   allowed: boolean;

package/src/services/comment-recipient-dispatch-service.ts

@@ -1,21 +1,38 @@
-import { and, desc, eq, inArray, like } from "drizzle-orm";
 import {
+  and,
   agents,
+  desc,
+  eq,
+  inArray,
   issueComments,
+  like,
   updateIssueCommentRecipients,
   type BopoDb
 } from "bopodev-db";
+import { createDrainableWorkTracker } from "../lib/drainable-work";
 import { parseIssueCommentRecipients, type PersistedCommentRecipient } from "../lib/comment-recipients";
 import type { RealtimeHub } from "../realtime/hub";
 import { enqueueHeartbeatQueueJob, triggerHeartbeatQueueWorker } from "./heartbeat-queue-service";
 
 const COMMENT_DISPATCH_SWEEP_LIMIT = 100;
 const activeCompanyDispatchRuns = new Set<string>();
+const commentDispatchTracker = createDrainableWorkTracker();
 
 export async function runIssueCommentDispatchSweep(
   db: BopoDb,
   companyId: string,
   options?: { requestId?: string; realtimeHub?: RealtimeHub; limit?: number }
+) {
+  if (commentDispatchTracker.isShuttingDown()) {
+    return;
+  }
+  return commentDispatchTracker.track(runIssueCommentDispatchSweepInternal(db, companyId, options));
+}
+
+async function runIssueCommentDispatchSweepInternal(
+  db: BopoDb,
+  companyId: string,
+  options?: { requestId?: string; realtimeHub?: RealtimeHub; limit?: number }
 ) {
   const rows = await db
     .select({
@@ -34,6 +51,9 @@ export async function runIssueCommentDispatchSweep(
     .limit(options?.limit ?? COMMENT_DISPATCH_SWEEP_LIMIT);
 
   for (const row of rows) {
+    if (commentDispatchTracker.isShuttingDown()) {
+      return;
+    }
     const recipients = parseIssueCommentRecipients(row.recipientsJson);
     if (!recipients.some((recipient) => recipient.deliveryStatus === "pending")) {
       continue;
@@ -60,11 +80,15 @@ export function triggerIssueCommentDispatchWorker(
   companyId: string,
   options?: { requestId?: string; realtimeHub?: RealtimeHub; limit?: number }
 ) {
-  if (activeCompanyDispatchRuns.has(companyId)) {
+  if (commentDispatchTracker.isShuttingDown() || activeCompanyDispatchRuns.has(companyId)) {
     return;
   }
   activeCompanyDispatchRuns.add(companyId);
   queueMicrotask(() => {
+    if (commentDispatchTracker.isShuttingDown()) {
+      activeCompanyDispatchRuns.delete(companyId);
+      return;
+    }
     void runIssueCommentDispatchSweep(db, companyId, options)
       .catch((error) => {
         // eslint-disable-next-line no-console
@@ -156,3 +180,16 @@ async function dispatchCommentRecipients(
   return dispatchedRecipients;
 }
 
+export function beginIssueCommentDispatchShutdown() {
+  commentDispatchTracker.beginShutdown();
+}
+
+export async function waitForIssueCommentDispatchDrain() {
+  await commentDispatchTracker.drain();
+}
+
+export function resetIssueCommentDispatchShutdownForTests() {
+  activeCompanyDispatchRuns.clear();
+  commentDispatchTracker.resetForTests();
+}
+

package/src/services/governance-service.ts

@@ -1,4 +1,3 @@
-import { and, eq } from "drizzle-orm";
 import { mkdir } from "node:fs/promises";
 import { z } from "zod";
 import {
@@ -10,6 +9,7 @@ import {
 } from "bopodev-contracts";
 import type { BopoDb } from "bopodev-db";
 import {
+  and,
   approvalRequests,
   agents,
   appendAuditEvent,
@@ -26,6 +26,7 @@ import {
   listProjectWorkspaces,
   listProjects,
   projects,
+  eq,
   updateProjectWorkspace,
   updatePluginConfig
 } from "bopodev-db";
@@ -777,6 +778,7 @@ function buildAgentStartupTaskBody(companyId: string, agentId: string) {
     `Create your operating baseline before starting feature delivery work.`,
     "",
     `1. Create your operating folder at \`${agentOperatingFolder}/\`.`,
+    "   During heartbeats, prefer the absolute path in `$BOPODEV_AGENT_OPERATING_DIR` (set by the runtime) so files land under your agent folder even when the shell cwd is a project workspace.",
     "2. Author these files with your own responsibilities and working style:",
     `   - \`${agentOperatingFolder}/AGENTS.md\``,
     `   - \`${agentOperatingFolder}/HEARTBEAT.md\``,

package/src/services/heartbeat-queue-service.ts

@@ -1,6 +1,7 @@
-import { and, eq } from "drizzle-orm";
 import {
+  and,
   cancelHeartbeatJob,
+  eq,
   getHeartbeatRun,
   claimNextHeartbeatJob,
   enqueueHeartbeatJob,
@@ -11,6 +12,7 @@ import {
   updateIssueCommentRecipients,
   type BopoDb
 } from "bopodev-db";
+import { createDrainableWorkTracker } from "../lib/drainable-work";
 import { parseIssueCommentRecipients } from "../lib/comment-recipients";
 import type { RealtimeHub } from "../realtime/hub";
 import { runHeartbeatForAgent } from "./heartbeat-service";
@@ -30,6 +32,7 @@ type QueueJobPayload = {
 };
 
 const activeCompanyQueueWorkers = new Set<string>();
+const queueWorkTracker = createDrainableWorkTracker();
 
 export async function enqueueHeartbeatQueueJob(
   db: BopoDb,
@@ -61,11 +64,15 @@ export function triggerHeartbeatQueueWorker(
   companyId: string,
   options?: { requestId?: string; realtimeHub?: RealtimeHub; maxJobsPerSweep?: number }
 ) {
-  if (activeCompanyQueueWorkers.has(companyId)) {
+  if (queueWorkTracker.isShuttingDown() || activeCompanyQueueWorkers.has(companyId)) {
     return;
   }
   activeCompanyQueueWorkers.add(companyId);
   queueMicrotask(() => {
+    if (queueWorkTracker.isShuttingDown()) {
+      activeCompanyQueueWorkers.delete(companyId);
+      return;
+    }
     void runHeartbeatQueueSweep(db, companyId, options)
       .catch((error) => {
         // eslint-disable-next-line no-console
@@ -81,10 +88,21 @@ export async function runHeartbeatQueueSweep(
   db: BopoDb,
   companyId: string,
   options?: { requestId?: string; realtimeHub?: RealtimeHub; maxJobsPerSweep?: number }
+) {
+  if (queueWorkTracker.isShuttingDown()) {
+    return { processed: 0 };
+  }
+  return queueWorkTracker.track(runHeartbeatQueueSweepInternal(db, companyId, options));
+}
+
+async function runHeartbeatQueueSweepInternal(
+  db: BopoDb,
+  companyId: string,
+  options?: { requestId?: string; realtimeHub?: RealtimeHub; maxJobsPerSweep?: number }
 ) {
   const maxJobs = Math.max(1, Math.min(options?.maxJobsPerSweep ?? 50, 500));
   let processed = 0;
-  while (processed < maxJobs) {
+  while (processed < maxJobs && !queueWorkTracker.isShuttingDown()) {
     const job = await claimNextHeartbeatJob(db, companyId);
     if (!job) {
       break;
@@ -117,6 +135,19 @@ export async function runHeartbeatQueueSweep(
   return { processed };
 }
 
+export function beginHeartbeatQueueShutdown() {
+  queueWorkTracker.beginShutdown();
+}
+
+export async function waitForHeartbeatQueueDrain() {
+  await queueWorkTracker.drain();
+}
+
+export function resetHeartbeatQueueShutdownForTests() {
+  activeCompanyQueueWorkers.clear();
+  queueWorkTracker.resetForTests();
+}
+
 async function processHeartbeatQueueJob(
   db: BopoDb,
   input: {