@cursor/july 0.1.113 → 0.2.0

package/src/internal/reminder-runner.ts

@@ -7,7 +7,9 @@
 import { randomBytes } from "node:crypto";
 import type {
   AuthContext,
+  ChannelSession,
   DiscoveredReminder,
+  HostOtelApi,
   JsonObject,
   JsonValue,
   ReminderCreateInput,
@@ -17,6 +19,7 @@ import type {
   ReminderInfo,
   ReminderStatus,
   ReminderUntilContext,
+  TurnOutcome,
 } from "../types.js";
 import {
   channelContinuationIdentity,
@@ -47,6 +50,7 @@ import { APP_AUTH } from "./schedule-runner.js";
 import {
   type EngineSendOptions,
   HTTP_CHANNEL_ID,
+  SessionAuthError,
   SessionBusyError,
   type SessionEngine,
 } from "./session-engine.js";
@@ -87,6 +91,103 @@ export interface ReminderRunnerOptions {
   controlPlane?: ReminderControlPlane;
   /** Agent architecture; used with {@link controlPlane} for v2 protocol path. */
   architecture?: "v1" | "v2";
+  /**
+   * Hosted v2 guest (`--no-control-plane`, architecture v2). A pod starts
+   * for one delivery and stops when it settles, so a fire usually lands on
+   * a pod that holds no session for the reminder's continuation, and the
+   * pod may stop as soon as the fire's outcome is applied. The runner then
+   * opens a session on the stored continuation key and waits for the turn.
+   */
+  hostedGuest?: ReminderHostedGuestOptions;
+}
+
+export interface ReminderHostedGuestOptions {
+  /**
+   * Principal for the session a prompt fire opens when the pod holds none.
+   * Hosted admissions open sessions with the same value, so a session the
+   * engine restores from durable storage keeps its principal. Return
+   * `undefined` when no principal applies; the fire then skips with a
+   * reschedule so the control plane keeps the reminder armed.
+   */
+  fireAuth: () => AuthContext | null | undefined;
+}
+
+/**
+ * Reschedule applied when a hosted fire cannot open a session (no
+ * principal). A one-shot `skipped` outcome without a reschedule completes
+ * the reminder, which would lose the check-in for good.
+ */
+export const REMINDER_SESSION_OPEN_RETRY_DELAY = "5m";
+
+/**
+ * `reason` tag on `agent_sdk.reminder.fire_outcome`. Prompt fires report
+ * how the session was reached: `session_joined` (held by this pod),
+ * `session_restored` (read back from durable storage), `session_opened`
+ * (fresh session on the stored key), `session_gone` (no session and none
+ * could be opened). `fire_threw` pairs with kind `error`: the fire raised
+ * and the hosted admission retries it.
+ */
+export type ReminderFireOutcomeReason =
+  | "session_gone"
+  | "session_opened"
+  | "session_restored"
+  | "session_joined"
+  | "handler_missing"
+  | "handler_lost_on_restart"
+  | "missing_handler"
+  | "run_stop"
+  | "run_skip"
+  | "run_delivered"
+  | "until_fn"
+  | "busy"
+  | "followup_failed"
+  | "fire_threw";
+
+export const REMINDER_FIRE_OUTCOME_METRIC = "agent_sdk.reminder.fire_outcome";
+
+/** What one `executeFire` reports: the CP outcome plus the telemetry tags. */
+interface FireExecution {
+  outcome: ReminderExecutionOutcome;
+  reason: ReminderFireOutcomeReason;
+  /** Set when the hosted guest awaited the turn before reporting. */
+  turnStatus?: TurnOutcome["status"];
+}
+
+interface PromptDelivery {
+  result: ReminderFireResult;
+  reason: ReminderFireOutcomeReason;
+  turnStatus?: TurnOutcome["status"];
+}
+
+/** Out-params `sendFollowup` fills in for the fire's telemetry. */
+interface FollowupSettle {
+  turnStatus?: TurnOutcome["status"];
+  skipReason?: Extract<ReminderFireOutcomeReason, "busy" | "followup_failed">;
+}
+
+/** How a prompt fire reached (or failed to reach) its session. */
+type FollowupTarget =
+  | {
+      auth: AuthContext | null;
+      reached: Extract<
+        ReminderFireOutcomeReason,
+        "session_joined" | "session_restored"
+      >;
+    }
+  | {
+      auth: undefined;
+      /**
+       * `session_gone`: no session anywhere. `session_unreadable`: the durable
+       * index has one but the local re-read missed; never open a fresh
+       * session over it.
+       */
+      reached: "session_gone" | "session_unreadable";
+    };
+
+/** Guest-side read surface shared by v1 timed-work and the v2 CP client. */
+interface ReminderReader {
+  getReminder(id: string): Promise<ReminderRecord | undefined>;
+  listReminders(): Promise<ReminderRecord[]>;
 }
 
 /**
@@ -169,6 +270,29 @@ function toInfo(record: ReminderRecord): ReminderInfo {
   };
 }
 
+function stopFire(
+  reason: Extract<
+    ReminderFireOutcomeReason,
+    | "handler_missing"
+    | "handler_lost_on_restart"
+    | "missing_handler"
+    | "until_fn"
+  >
+): FireExecution {
+  return { outcome: { kind: "stop", reason }, reason };
+}
+
+function runReason(result: ReminderFireResult): ReminderFireOutcomeReason {
+  switch (result.action) {
+    case "stop":
+      return "run_stop";
+    case "skip":
+      return "run_skip";
+    case "delivered":
+      return "run_delivered";
+  }
+}
+
 function buildFirePrompt(record: ReminderRecord): string {
   const recurring =
     record.scheduleKind === "every" || record.scheduleKind === "cron";
@@ -205,6 +329,12 @@ export class ReminderRunner {
   private readonly rejectAnonymousHandlers: boolean;
   private readonly controlPlane: ReminderControlPlane | undefined;
   private readonly architecture: "v1" | "v2" | undefined;
+  private readonly hostedGuest: ReminderHostedGuestOptions | undefined;
+  /**
+   * A control-plane read failure is logged once per outage, not per call.
+   * The next successful read re-arms the log.
+   */
+  private controlPlaneReadFailureLogged = false;
 
   constructor(
     private readonly engine: SessionEngine,
@@ -215,6 +345,7 @@ export class ReminderRunner {
     this.rejectAnonymousHandlers = options.rejectAnonymousHandlers === true;
     this.controlPlane = options.controlPlane;
     this.architecture = options.architecture;
+    this.hostedGuest = options.hostedGuest;
     for (const reminder of options.namedHandlers ?? []) {
       this.namedHandlers.set(reminder.name, reminder.definition);
       this.namedHandlerNames.set(reminder.definition, reminder.name);
@@ -226,29 +357,62 @@ export class ReminderRunner {
     return this.controlPlane !== undefined && this.architecture === "v2";
   }
 
+  /**
+   * Where reminder documents are read from. Under the CP protocol that is
+   * the v2 control-plane client; the v1 timed-work route answers 405 for a
+   * v2 deployment, which used to leave `list()` with only this process's
+   * in-memory map (breaking arm idempotence and `cancelForContinuation`).
+   */
+  private get reminderReader(): ReminderReader | undefined {
+    if (this.usesControlPlaneProtocol) {
+      return this.controlPlane;
+    }
+    return this.platformTimers;
+  }
+
+  private logControlPlaneReadFailure(what: string, error: unknown): void {
+    if (this.usesControlPlaneProtocol && this.controlPlaneReadFailureLogged) {
+      return;
+    }
+    this.controlPlaneReadFailureLogged = true;
+    this.logger(
+      `[agent-sdk] reminders: control-plane ${what} failed: ${
+        error instanceof Error ? error.message : String(error)
+      }`
+    );
+  }
+
   /**
    * Load reminder documents from the control plane into memory. Does not
    * arm in-process timers — the CP owns clocks.
    */
   async hydrate(): Promise<void> {
-    if (this.platformTimers === undefined) {
+    const reader = this.reminderReader;
+    if (reader === undefined) {
       return;
     }
     let remote: ReminderRecord[] = [];
     try {
-      remote = await this.platformTimers.listReminders();
+      remote = await reader.listReminders();
     } catch (error) {
-      this.logger(
-        `[agent-sdk] reminders: control-plane list failed: ${
-          error instanceof Error ? error.message : String(error)
-        }`
-      );
+      this.logControlPlaneReadFailure("list", error);
       return;
     }
+    this.controlPlaneReadFailureLogged = false;
     this.records.clear();
     for (const record of remote) {
       this.records.set(record.id, record);
     }
+    if (this.usesControlPlaneProtocol) {
+      // The CP owns status; a fire whose handler is missing reports `stop`
+      // from executeFire instead of the guest writing a disarm.
+      if (remote.length > 0) {
+        this.logger(
+          `[agent-sdk] reminders: hydrated ${remote.length} reminder(s) from control plane`
+        );
+      }
+      return;
+    }
     let disarmedOrphan = 0;
     for (const record of [...this.records.values()]) {
       if (record.status !== "armed") {
@@ -330,9 +494,11 @@ export class ReminderRunner {
   }
 
   private async getRecord(id: string): Promise<ReminderRecord | undefined> {
-    if (this.platformTimers !== undefined) {
+    const reader = this.reminderReader;
+    if (reader !== undefined) {
       try {
-        const remote = await this.platformTimers.getReminder(id);
+        const remote = await reader.getReminder(id);
+        this.controlPlaneReadFailureLogged = false;
         if (remote === undefined) {
           this.records.delete(id);
           return undefined;
@@ -340,31 +506,33 @@ export class ReminderRunner {
         this.records.set(id, remote);
         return remote;
       } catch (error) {
-        this.logger(
-          `[agent-sdk] reminders: control-plane get failed for ${id}: ${
-            error instanceof Error ? error.message : String(error)
-          }`
-        );
+        this.logControlPlaneReadFailure(`get for ${id}`, error);
         return this.records.get(id);
       }
     }
     return this.records.get(id);
   }
 
+  /**
+   * Under the CP protocol a list failure propagates: the in-memory map is
+   * only what this process created, and returning it would let a caller
+   * arm duplicates or miss rows to cancel. v1 keeps the best-effort form.
+   */
   private async allRecords(): Promise<ReminderRecord[]> {
-    if (this.platformTimers !== undefined) {
+    const reader = this.reminderReader;
+    if (reader !== undefined) {
       try {
-        const remote = await this.platformTimers.listReminders();
+        const remote = await reader.listReminders();
+        this.controlPlaneReadFailureLogged = false;
         this.records.clear();
         for (const record of remote) {
           this.records.set(record.id, record);
         }
       } catch (error) {
-        this.logger(
-          `[agent-sdk] reminders: control-plane list failed: ${
-            error instanceof Error ? error.message : String(error)
-          }`
-        );
+        this.logControlPlaneReadFailure("list", error);
+        if (this.usesControlPlaneProtocol) {
+          throw error;
+        }
       }
     }
     return [...this.records.values()];
@@ -697,25 +865,46 @@ export class ReminderRunner {
   ): Promise<ReminderExecutionOutcome> {
     const record = this.recordFromDescriptor(descriptor);
     this.records.set(record.id, record);
+    let fire: FireExecution;
+    try {
+      fire = await this.runFire(record);
+    } catch (error) {
+      // The hosted admission maps the throw to a retryable failure; count
+      // it so the retry loop is visible beside the settled outcomes.
+      this.recordFireOutcome(record, {
+        kind: "error",
+        reason: "fire_threw",
+        detail: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+    this.recordFireOutcome(record, {
+      kind: fire.outcome.kind,
+      reason: fire.reason,
+      turnStatus: fire.turnStatus,
+    });
+    return fire.outcome;
+  }
 
+  private async runFire(record: ReminderRecord): Promise<FireExecution> {
     const live = this.handlers.get(record.id) ?? {};
     if (record.handler !== undefined) {
       if (this.resolveNamedHandler(record.handler) === undefined) {
-        return { kind: "stop", reason: "handler_missing" };
+        return stopFire("handler_missing");
       }
     } else if (
       (record.hasRunHandler === true || record.hasUntilFn === true) &&
       live.run === undefined &&
       live.untilFn === undefined
     ) {
-      return { kind: "stop", reason: "handler_lost_on_restart" };
+      return stopFire("handler_lost_on_restart");
     }
 
     if (live.untilFn !== undefined) {
       try {
         const done = await live.untilFn(this.buildUntilContext(record));
         if (done) {
-          return { kind: "stop", reason: "until_fn" };
+          return stopFire("until_fn");
         }
       } catch (error) {
         this.logger(
@@ -728,50 +917,94 @@ export class ReminderRunner {
 
     if (await this.isSessionBusy(record.channelId, record.continuationToken)) {
       if (record.busyPolicy === "skipIfBusy") {
-        this.logger(
-          `[agent-sdk] reminder "${record.id}" (${record.purpose}) skipped: session busy`
-        );
-        return { kind: "skipped" };
+        return { outcome: { kind: "skipped" }, reason: "busy" };
       }
       await this.whenSessionIdle(record.channelId, record.continuationToken);
     }
 
     let result: ReminderFireResult;
+    let reason: ReminderFireOutcomeReason;
+    let turnStatus: TurnOutcome["status"] | undefined;
     if (record.handler !== undefined) {
       const named = this.resolveNamedHandler(record.handler);
       if (named === undefined) {
-        return { kind: "stop", reason: "handler_missing" };
+        return stopFire("handler_missing");
       }
       result = await named.run(this.buildFireContext(record));
+      reason = runReason(result);
     } else if (live.run !== undefined) {
       result = await live.run(this.buildFireContext(record));
+      reason = runReason(result);
     } else if (record.prompt !== undefined) {
-      result = await this.deliverPrompt(record);
+      const delivery = await this.deliverPromptFire(record);
+      result = delivery.result;
+      reason = delivery.reason;
+      turnStatus = delivery.turnStatus;
     } else {
-      return { kind: "stop", reason: "missing_handler" };
+      return stopFire("missing_handler");
     }
 
     if (result.action === "stop") {
-      return { kind: "stop", reason: "run_stop" };
-    }
-    if (result.action === "skip") {
-      return {
-        kind: "skipped",
-        reschedule:
-          result.reschedule !== undefined
-            ? toWireReschedule(result.reschedule)
-            : undefined,
-      };
+      return { outcome: { kind: "stop", reason }, reason };
     }
+    const reschedule =
+      result.reschedule !== undefined
+        ? toWireReschedule(result.reschedule)
+        : undefined;
     return {
-      kind: "delivered",
-      reschedule:
-        result.reschedule !== undefined
-          ? toWireReschedule(result.reschedule)
-          : undefined,
+      outcome: {
+        kind: result.action === "skip" ? "skipped" : "delivered",
+        reschedule,
+      },
+      reason,
+      turnStatus,
     };
   }
 
+  /**
+   * One structured line and one `agent_sdk.reminder.fire_outcome` count per
+   * fire. Emitted through the host OTel meter, the same path hosted v2
+   * exports turn/tool metrics on, so a fire that ends in `stop` is visible
+   * beside the control plane's `timed_work.fire` count.
+   */
+  private recordFireOutcome(
+    record: ReminderRecord,
+    fire: {
+      kind: ReminderExecutionOutcome["kind"] | "error";
+      reason: ReminderFireOutcomeReason;
+      turnStatus?: TurnOutcome["status"];
+      detail?: string;
+    }
+  ): void {
+    this.logger(
+      `[agent-sdk] reminder fire_outcome kind=${fire.kind} reason=${fire.reason}` +
+        ` reminderId=${record.id} purpose=${record.purpose}` +
+        ` channelId=${record.channelId} fireCount=${record.fireCount}` +
+        (fire.turnStatus === undefined ? "" : ` turn=${fire.turnStatus}`) +
+        (fire.detail === undefined
+          ? ""
+          : ` detail=${JSON.stringify(fire.detail)}`)
+    );
+    // Test doubles may omit `host.otel`; the metric is best-effort.
+    const otel: HostOtelApi | undefined = this.engine.host.otel;
+    if (otel === undefined) {
+      return;
+    }
+    try {
+      otel.increment(REMINDER_FIRE_OUTCOME_METRIC, {
+        kind: fire.kind,
+        reason: fire.reason,
+        purpose: record.purpose,
+      });
+    } catch (error) {
+      this.logger(
+        `[agent-sdk] reminder fire_outcome metric failed: ${
+          error instanceof Error ? error.message : String(error)
+        }`
+      );
+    }
+  }
+
   private recordFromDescriptor(
     descriptor: ReminderFireDescriptor
   ): ReminderRecord {
@@ -1033,19 +1266,121 @@ export class ReminderRunner {
   private async deliverPrompt(
     record: ReminderRecord
   ): Promise<ReminderFireResult> {
+    return (await this.deliverPromptFire(record)).result;
+  }
+
+  /**
+   * Where a prompt fire joins. The pod's own store first; on a hosted guest
+   * the durable mirror next, because the pod that armed the reminder is
+   * usually gone and the session lives only in storage. A restored session
+   * keeps its own principal (channel admissions may carry a sender), so the
+   * fire joins under that, never under {@link ReminderHostedGuestOptions.fireAuth}.
+   */
+  private async resolveFollowupTarget(
+    record: ReminderRecord
+  ): Promise<FollowupTarget> {
+    const local = await this.resolveFollowupAuth(
+      record.channelId,
+      record.continuationToken
+    );
+    if (local !== undefined) {
+      return { auth: local, reached: "session_joined" };
+    }
+    if (this.hostedGuest === undefined) {
+      return { auth: undefined, reached: "session_gone" };
+    }
+    // Materializes the durable record locally; a storage error propagates
+    // and the hosted admission retries the fire.
+    const restored = await this.engine.hasContinuationSession(
+      record.channelId,
+      record.continuationToken
+    );
+    if (!restored) {
+      return { auth: undefined, reached: "session_gone" };
+    }
     const auth = await this.resolveFollowupAuth(
       record.channelId,
       record.continuationToken
     );
-    if (auth === undefined) {
-      // Under CP protocol, return stop and let the CP apply/CAS; do not
-      // mutate authoritative status from the guest mid-fire.
-      if (!this.usesControlPlaneProtocol) {
-        await this.disarm(record, "session_gone");
+    return auth === undefined
+      ? { auth: undefined, reached: "session_unreadable" }
+      : { auth, reached: "session_restored" };
+  }
+
+  private async deliverPromptFire(
+    record: ReminderRecord
+  ): Promise<PromptDelivery> {
+    const target = await this.resolveFollowupTarget(record);
+    if (target.auth !== undefined) {
+      return this.promptDelivery(record, target.auth, target.reached);
+    }
+    if (target.reached === "session_unreadable") {
+      this.logger(
+        `[agent-sdk] reminder "${record.id}" (${record.purpose}) durable session did not materialize locally; not opening a new one`
+      );
+      return { result: { action: "stop" }, reason: "session_gone" };
+    }
+    if (
+      this.hostedGuest !== undefined &&
+      this.channelUsesControlPlaneContinuation(record.channelId)
+    ) {
+      // No session on this pod and none in durable storage. Open one on the
+      // stored key the way an inbound event on a fresh key does, instead of
+      // consuming the fire with no turn.
+      const fireAuth = this.hostedGuest.fireAuth();
+      if (fireAuth === undefined) {
+        return {
+          result: {
+            action: "skip",
+            reschedule: { delay: REMINDER_SESSION_OPEN_RETRY_DELAY },
+          },
+          reason: "session_gone",
+        };
+      }
+      try {
+        return await this.promptDelivery(record, fireAuth, "session_opened");
+      } catch (error) {
+        if (!(error instanceof SessionAuthError)) {
+          throw error;
+        }
+        // A session landed on this key under another principal between the
+        // lookup and the send. That does not change on retry, so stop
+        // rather than let the control plane re-admit until its attempt cap.
+        this.logger(
+          `[agent-sdk] reminder "${record.id}" (${record.purpose}) cannot join session on its continuation: ${error.message}`
+        );
+        return { result: { action: "stop" }, reason: "session_gone" };
       }
-      return { action: "stop" };
     }
-    return this.sendFollowup(record, buildFirePrompt(record), auth);
+    // Under CP protocol, return stop and let the CP apply/CAS; do not
+    // mutate authoritative status from the guest mid-fire.
+    if (!this.usesControlPlaneProtocol) {
+      await this.disarm(record, "session_gone");
+    }
+    return { result: { action: "stop" }, reason: "session_gone" };
+  }
+
+  private async promptDelivery(
+    record: ReminderRecord,
+    auth: AuthContext | null,
+    delivered: Extract<
+      ReminderFireOutcomeReason,
+      "session_joined" | "session_restored" | "session_opened"
+    >
+  ): Promise<PromptDelivery> {
+    const settle: FollowupSettle = {};
+    const result = await this.sendFollowup(
+      record,
+      buildFirePrompt(record),
+      auth,
+      settle
+    );
+    return {
+      result,
+      reason:
+        result.action === "skip" ? (settle.skipReason ?? "busy") : delivered,
+      turnStatus: settle.turnStatus,
+    };
   }
 
   /**
@@ -1086,10 +1421,33 @@ export class ReminderRunner {
     return continuationSource(identity) === "control-plane";
   }
 
+  /**
+   * Local and v1 track the turn in the background: the process outlives the
+   * fire. A hosted v2 guest waits for the turn and its durable writes first,
+   * because the control plane may stop the pod as soon as the fire's
+   * outcome is applied, and `delivered` must mean the turn ran.
+   */
+  private async settleFollowup(
+    session: ChannelSession,
+    settle: FollowupSettle | undefined
+  ): Promise<void> {
+    const completion = this.engine.waitForCompletion(session.id);
+    if (this.hostedGuest === undefined) {
+      this.engine.trackBackground(completion);
+      return;
+    }
+    const outcome = await completion;
+    await this.engine.drainSessionDurability(session.id);
+    if (settle !== undefined) {
+      settle.turnStatus = outcome.status;
+    }
+  }
+
   private async sendFollowup(
     record: ReminderRecord,
     message: string,
-    auth: AuthContext | null
+    auth: AuthContext | null,
+    settle?: FollowupSettle
   ): Promise<ReminderFireResult> {
     const sendOptions = this.followupSendOptions(record, auth);
     try {
@@ -1098,11 +1456,14 @@ export class ReminderRunner {
         message,
         sendOptions
       );
-      this.engine.trackBackground(this.engine.waitForCompletion(session.id));
+      await this.settleFollowup(session, settle);
       return { action: "delivered" };
     } catch (error) {
       if (error instanceof SessionBusyError) {
         if (record.busyPolicy === "skipIfBusy") {
+          if (settle !== undefined) {
+            settle.skipReason = "busy";
+          }
           return { action: "skip" };
         }
         await this.whenSessionIdle(record.channelId, record.continuationToken);
@@ -1112,9 +1473,7 @@ export class ReminderRunner {
             message,
             sendOptions
           );
-          this.engine.trackBackground(
-            this.engine.waitForCompletion(session.id)
-          );
+          await this.settleFollowup(session, settle);
           return { action: "delivered" };
         } catch (retryError) {
           this.logger(
@@ -1124,6 +1483,9 @@ export class ReminderRunner {
                 : String(retryError)
             }`
           );
+          if (settle !== undefined) {
+            settle.skipReason = "followup_failed";
+          }
           return { action: "skip" };
         }
       }
@@ -1148,12 +1510,13 @@ export class ReminderRunner {
       appAuth: APP_AUTH,
       artifacts: this.engine.artifacts,
       async followup(args: { message: string }) {
-        const auth = await self.resolveFollowupAuth(
-          record.channelId,
-          record.continuationToken
-        );
+        const { auth } = await self.resolveFollowupTarget(record);
         if (auth === undefined) {
-          await self.disarm(record, "session_gone");
+          // Under the CP protocol the guest never writes reminder status;
+          // the fire's `stop` outcome carries the reason instead.
+          if (!self.usesControlPlaneProtocol) {
+            await self.disarm(record, "session_gone");
+          }
           throw new Error(
             `Reminder "${record.id}" followup: session gone for continuation`
           );