@app-vitals/agent-runtime 0.0.0-stage → 0.1.0

package/cron-failure-reporter.ts

@@ -0,0 +1,107 @@
+/**
+ * agent/src/runtime/cron-failure-reporter.ts
+ *
+ * Extracted from index.ts's cron-sync scheduled callback so a thrown dispatch
+ * error (either the `dispatch === "loop"` path or the regular
+ * handleCronRequest path) surfaces the same way any other failed cron run
+ * does: an AgentCronRun row (visible in the admin run-history UI / Slack),
+ * not just raw pod stdout.
+ *
+ * Mirrors loop-orchestrator.ts's dispatch() catch block: createRun() is
+ * called for the tick, then on a thrown error completeRun() is called with
+ * outcome "failed" and { error: <message> }. Also calls
+ * sentryClient.captureException(err) — this is a genuinely unhandled failure
+ * (the whole tick aborted), not a swallowed/handled one, so it's captured as
+ * a proper Sentry Issue rather than relying on consoleLoggingIntegration to
+ * turn a console.error call into a lower-signal Sentry log line.
+ *
+ * `cronRunReporter` and `sentryClient` are both optional-by-convention at the
+ * call site (NoopCronRunReporter / undefined respectively) — this function
+ * itself requires a CronRunReporter (callers pass the Noop fallback) and
+ * treats sentryClient as optional so it behaves identically whether or not
+ * SENTRY_DSN is set.
+ *
+ * index.ts's catch wraps BOTH the loop-dispatch path (which calls into
+ * loop-orchestrator.ts's dispatch()) and the handleCronRequest path
+ * (cron-handler.ts) — both of those already call
+ * cronRunReporter.completeRun(..., "failed", ...) on their own runId before
+ * rethrowing. To avoid a duplicate AgentCronRun "failed" row per real
+ * failure, those callers tag the rethrown error via
+ * markCronRunFailureReported() first; reportCronFailure() always logs and
+ * captures to Sentry, but skips its own createRun/completeRun pair when the
+ * error is already tagged.
+ */
+
+import type { ErrorCapturingClient } from "@app-vitals/agent-lib/sentry";
+import { reportClaudeError } from "./claude.ts";
+import type { Clock } from "./clock.ts";
+import type { CronRunReporter } from "./cron-run-reporter.ts";
+
+interface ReportCronFailureDeps {
+  cronRunReporter: CronRunReporter;
+  sentryClient?: ErrorCapturingClient;
+  clock: Clock;
+}
+
+/**
+ * Symbol tag used to mark a thrown error as "an AgentCronRun failure row has
+ * already been recorded for this error" — set by callers (cron-handler.ts's
+ * runner-scope catch, loop-orchestrator.ts's dispatch() catch) that already
+ * called cronRunReporter.completeRun(..., "failed", ...) on their own runId
+ * before rethrowing. Symbol.for() (not Symbol()) so the tag survives across
+ * module instances if the error crosses a boundary.
+ */
+const CRON_RUN_ALREADY_REPORTED = Symbol.for(
+  "shipwright.cronRunAlreadyReported",
+);
+
+/**
+ * Marks a thrown error as already reported to CronRunReporter, so a later,
+ * broader catch (e.g. index.ts's cron-sync scheduled callback) can call
+ * reportCronFailure() for logging/Sentry purposes without also creating a
+ * duplicate AgentCronRun "failed" row.
+ */
+export function markCronRunFailureReported(err: unknown): void {
+  if (err && typeof err === "object") {
+    Reflect.set(err as object, CRON_RUN_ALREADY_REPORTED, true);
+  }
+}
+
+function isCronRunFailureReported(err: unknown): boolean {
+  return Boolean(
+    err &&
+      typeof err === "object" &&
+      Reflect.get(err as object, CRON_RUN_ALREADY_REPORTED),
+  );
+}
+
+/**
+ * Reports a cron tick's dispatch failure: logs locally (console.error),
+ * captures the error in Sentry (when a sentryClient is wired), and records a
+ * failed AgentCronRun via createRun/completeRun so the failure shows up in
+ * the admin run-history UI and metrics dashboard, not just raw pod logs.
+ */
+export async function reportCronFailure(
+  cronId: string,
+  err: unknown,
+  deps: ReportCronFailureDeps,
+): Promise<void> {
+  const { cronRunReporter, sentryClient, clock } = deps;
+  const message = err instanceof Error ? err.message : String(err);
+
+  console.error(`[cron] job ${cronId} failed:`, message);
+
+  reportClaudeError(sentryClient, err);
+
+  if (isCronRunFailureReported(err)) {
+    // A more specific layer (cron-handler.ts / loop-orchestrator.ts) already
+    // recorded a failed AgentCronRun on its own runId before rethrowing —
+    // skip creating a second, untagged row for the same failure.
+    return;
+  }
+
+  const runId = await cronRunReporter.createRun(cronId, clock.now());
+  await cronRunReporter.completeRun(cronId, runId, clock.now(), "failed", {
+    error: message,
+  });
+}

package/cron-failure-reporter.unit.test.ts

@@ -0,0 +1,244 @@
+/**
+ * agent/src/runtime/cron-failure-reporter.unit.test.ts
+ *
+ * Unit tests for reportCronFailure() — pure logic, no I/O. Verifies a thrown
+ * cron dispatch error results in a createRun/completeRun("failed") pair on
+ * the injected CronRunReporter and a captureException call on the injected
+ * ErrorCapturingClient, mirroring loop-orchestrator.ts's dispatch() pattern.
+ */
+
+import { describe, expect, it } from "bun:test";
+import { ClaudeTimeoutError } from "./claude.ts";
+import { FixedClock } from "./clock.ts";
+import {
+  markCronRunFailureReported,
+  reportCronFailure,
+} from "./cron-failure-reporter.ts";
+import type {
+  CronRunReporter,
+  ModelBreakdownEntry,
+} from "./cron-run-reporter.ts";
+
+const FIXED_TIME = new Date("2024-01-01T00:00:00.000Z");
+
+interface CreateRunCall {
+  cronId: string;
+  startedAt: Date;
+  phase?: string;
+}
+
+interface CompleteRunCall {
+  cronId: string;
+  runId: string | null;
+  completedAt: Date;
+  outcome: "completed" | "failed";
+  opts?: {
+    error?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    cacheReadTokens?: number;
+    cacheCreationTokens?: number;
+    modelBreakdown?: ModelBreakdownEntry[];
+  };
+  phase?: string;
+}
+
+function createFakeCronRunReporter(runId: string | null = "run-1") {
+  const createRunCalls: CreateRunCall[] = [];
+  const completeRunCalls: CompleteRunCall[] = [];
+
+  const reporter: CronRunReporter = {
+    async createRun(cronId, startedAt, phase) {
+      createRunCalls.push({ cronId, startedAt, phase });
+      return runId;
+    },
+    async completeRun(cronId, callRunId, completedAt, outcome, opts, phase) {
+      completeRunCalls.push({
+        cronId,
+        runId: callRunId,
+        completedAt,
+        outcome,
+        opts,
+        phase,
+      });
+    },
+    async skipRun() {
+      throw new Error("skipRun should not be called by reportCronFailure");
+    },
+    async recordProgress() {},
+    async recordSessionId() {},
+  };
+
+  return { reporter, createRunCalls, completeRunCalls };
+}
+
+function createFakeErrorCapturingClient() {
+  const capturedErrors: unknown[] = [];
+  const capturedMessages: string[] = [];
+  return {
+    client: {
+      captureException: (err: unknown) => {
+        capturedErrors.push(err);
+      },
+      captureMessage: (message: string) => {
+        capturedMessages.push(message);
+      },
+    },
+    capturedErrors,
+    capturedMessages,
+  };
+}
+
+describe("reportCronFailure", () => {
+  it("creates a run and completes it as failed with the error message", async () => {
+    const { reporter, createRunCalls, completeRunCalls } =
+      createFakeCronRunReporter();
+    const clock = FixedClock(FIXED_TIME);
+    const err = new Error("dispatch exploded");
+
+    await reportCronFailure("my-cron-id", err, {
+      cronRunReporter: reporter,
+      clock,
+    });
+
+    expect(createRunCalls).toHaveLength(1);
+    expect(createRunCalls[0]?.cronId).toBe("my-cron-id");
+    expect(createRunCalls[0]?.startedAt).toEqual(FIXED_TIME);
+
+    expect(completeRunCalls).toHaveLength(1);
+    expect(completeRunCalls[0]?.cronId).toBe("my-cron-id");
+    expect(completeRunCalls[0]?.runId).toBe("run-1");
+    expect(completeRunCalls[0]?.completedAt).toEqual(FIXED_TIME);
+    expect(completeRunCalls[0]?.outcome).toBe("failed");
+    expect(completeRunCalls[0]?.opts?.error).toBe("dispatch exploded");
+  });
+
+  it("stringifies a non-Error thrown value for the error field", async () => {
+    const { reporter, completeRunCalls } = createFakeCronRunReporter();
+    const clock = FixedClock(FIXED_TIME);
+
+    await reportCronFailure("my-cron-id", "just a string failure", {
+      cronRunReporter: reporter,
+      clock,
+    });
+
+    expect(completeRunCalls[0]?.outcome).toBe("failed");
+    expect(completeRunCalls[0]?.opts?.error).toBe("just a string failure");
+  });
+
+  it("calls captureException on the injected sentry client with the original error", async () => {
+    const { reporter } = createFakeCronRunReporter();
+    const { client, capturedErrors } = createFakeErrorCapturingClient();
+    const clock = FixedClock(FIXED_TIME);
+    const err = new Error("boom");
+
+    await reportCronFailure("my-cron-id", err, {
+      cronRunReporter: reporter,
+      sentryClient: client,
+      clock,
+    });
+
+    expect(capturedErrors).toHaveLength(1);
+    expect(capturedErrors[0]).toBe(err);
+  });
+
+  it("SENTRY-46: reports a ceiling-reason ClaudeTimeoutError via captureMessage, not captureException", async () => {
+    const { reporter } = createFakeCronRunReporter();
+    const { client, capturedErrors, capturedMessages } =
+      createFakeErrorCapturingClient();
+    const clock = FixedClock(FIXED_TIME);
+    const err = new ClaudeTimeoutError(3_600_000, "ceiling");
+
+    await reportCronFailure("my-cron-id", err, {
+      cronRunReporter: reporter,
+      sentryClient: client,
+      clock,
+    });
+
+    expect(capturedErrors).toHaveLength(0);
+    expect(capturedMessages).toHaveLength(1);
+    expect(capturedMessages[0]).toContain("3600s");
+  });
+
+  it("still reports an idle-reason ClaudeTimeoutError via captureException", async () => {
+    const { reporter } = createFakeCronRunReporter();
+    const { client, capturedErrors, capturedMessages } =
+      createFakeErrorCapturingClient();
+    const clock = FixedClock(FIXED_TIME);
+    const err = new ClaudeTimeoutError(1_500_000, "idle");
+
+    await reportCronFailure("my-cron-id", err, {
+      cronRunReporter: reporter,
+      sentryClient: client,
+      clock,
+    });
+
+    expect(capturedMessages).toHaveLength(0);
+    expect(capturedErrors).toHaveLength(1);
+    expect(capturedErrors[0]).toBe(err);
+  });
+
+  it("does not throw when no sentryClient is provided", async () => {
+    const { reporter } = createFakeCronRunReporter();
+    const clock = FixedClock(FIXED_TIME);
+
+    await expect(
+      reportCronFailure("my-cron-id", new Error("boom"), {
+        cronRunReporter: reporter,
+        clock,
+      }),
+    ).resolves.toBeUndefined();
+  });
+
+  it("still creates/completes a run for an error that was not marked as already-reported", async () => {
+    const { reporter, createRunCalls, completeRunCalls } =
+      createFakeCronRunReporter();
+    const clock = FixedClock(FIXED_TIME);
+    const err = new Error("unmarked failure");
+
+    await reportCronFailure("my-cron-id", err, {
+      cronRunReporter: reporter,
+      clock,
+    });
+
+    expect(createRunCalls).toHaveLength(1);
+    expect(completeRunCalls).toHaveLength(1);
+  });
+
+  it("skips createRun/completeRun but still logs and captures to Sentry when the error was already reported", async () => {
+    const { reporter, createRunCalls, completeRunCalls } =
+      createFakeCronRunReporter();
+    const { client, capturedErrors } = createFakeErrorCapturingClient();
+    const clock = FixedClock(FIXED_TIME);
+    const err = new Error("already reported by inner catch");
+    markCronRunFailureReported(err);
+
+    await reportCronFailure("my-cron-id", err, {
+      cronRunReporter: reporter,
+      sentryClient: client,
+      clock,
+    });
+
+    expect(createRunCalls).toHaveLength(0);
+    expect(completeRunCalls).toHaveLength(0);
+    expect(capturedErrors).toHaveLength(1);
+    expect(capturedErrors[0]).toBe(err);
+  });
+
+  it("does not mark non-object thrown values (e.g. strings) but still reports normally", async () => {
+    const { reporter, createRunCalls, completeRunCalls } =
+      createFakeCronRunReporter();
+
+    // markCronRunFailureReported on a primitive is a no-op; should not throw.
+    expect(() => markCronRunFailureReported("just a string")).not.toThrow();
+
+    const clock = FixedClock(FIXED_TIME);
+    await reportCronFailure("my-cron-id", "just a string", {
+      cronRunReporter: reporter,
+      clock,
+    });
+
+    expect(createRunCalls).toHaveLength(1);
+    expect(completeRunCalls).toHaveLength(1);
+  });
+});