deepline 0.3.431 → 0.3.433

package/dist/bundling-sources/sdk/src/release.ts

@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.431',
+  version: '0.3.433',
   updateSummary: {
     introducedIn: '0.3.90',
     summary:

package/dist/bundling-sources/shared_libs/monitors/billing/credit-recovery.ts

@@ -2,8 +2,9 @@ import { parseStoredMonitorBillingState } from './state';
 
 /**
  * Only these event-time billing denials are healed by a restored Deepline
- * balance. A monthly spending cap and an opaque billing failure deliberately
- * remain explicit-recovery states.
+ * balance. A monthly spending cap is healed by the cap admitting again
+ * (`isMonitorCapSuspensionReason`), and an opaque billing failure deliberately
+ * remains an explicit-recovery state.
  */
 export function isCreditExhaustionSuspensionReason(
   reason: string | undefined,
@@ -14,6 +15,60 @@ export function isCreditExhaustionSuspensionReason(
   );
 }
 
+/**
+ * The billing policy sweep's credit pause. When the sweep finds an exhausted
+ * wallet it deactivates a direct monitor with this reason, and the repair that
+ * converges it records the same reason as the billing suspension's. The sweep
+ * then defers a `billing_suspended` row to credit recovery and never revives
+ * it itself, so unless credit recovery accepts it, a top-up has no automatic
+ * path back (the reason's own resolution promises one).
+ *
+ * Accepted only with the policy's provenance on the row: the same suspension
+ * reason under any other deactivation reason is not proof the wallet caused
+ * it.
+ *
+ * KAR-43: an exhausted wallet switches every Monitor off through this policy
+ * pause, Fleet members included, so a Fleet the policy suspended is credit
+ * restorable too (`isCreditRestorableFleetSuspensionReason`).
+ */
+export const POLICY_CREDIT_SUSPENSION_REASON = 'insufficient_balance';
+
+/** The policy's spending-cap pause (KAR-43). */
+export const POLICY_CAP_SUSPENSION_REASON = 'monthly_cap_exceeded';
+
+/**
+ * A Fleet's billing suspension that a restored balance heals: an event-time
+ * credit denial or the policy's own credit pause. The Fleet's suspension
+ * record is written only by Fleet billing lifecycle, so its reason is its
+ * provenance.
+ */
+export function isCreditRestorableFleetSuspensionReason(
+  reason: string | undefined,
+): boolean {
+  return (
+    isCreditExhaustionSuspensionReason(reason) ||
+    reason === POLICY_CREDIT_SUSPENSION_REASON
+  );
+}
+
+/** A spending-cap suspension, healed once this period's spend is under the cap. */
+export function isMonitorCapSuspensionReason(
+  reason: string | undefined,
+): boolean {
+  return reason === POLICY_CAP_SUSPENSION_REASON;
+}
+
+function isDirectCreditSuspension(input: {
+  suspensionReason: string | undefined;
+  deactivationReason: string | undefined;
+}): boolean {
+  return (
+    isCreditExhaustionSuspensionReason(input.suspensionReason) ||
+    (input.suspensionReason === POLICY_CREDIT_SUSPENSION_REASON &&
+      input.deactivationReason === POLICY_CREDIT_SUSPENSION_REASON)
+  );
+}
+
 /**
  * A billing suspension is not itself proof that TamRadar accepted the delete.
  * Keep this test beside the recovery selector so every automatic path rejects
@@ -44,9 +99,10 @@ export function hasConfirmedProviderMonitorUpstreamCleanup(
 }
 
 /**
- * Narrow selector for direct event-time suspensions that credit restoration
- * may re-provision. It intentionally does not include policy-only pauses,
- * manual intent, spending-cap denials, or cleanup still in flight.
+ * Narrow selector for direct credit-exhaustion suspensions that credit
+ * restoration may re-provision: an event-time suspension, or the policy
+ * sweep's own credit pause. It intentionally does not include manual intent,
+ * spending-cap denials, opaque billing failures, or cleanup still in flight.
  */
 export function isCreditRestorableDirectMonitorSuspension(input: {
   fleetId?: string;
@@ -61,7 +117,10 @@ export function isCreditRestorableDirectMonitorSuspension(input: {
   if (
     billingState.state !== 'billing_suspended' ||
     !billingState.reactivation_required ||
-    !isCreditExhaustionSuspensionReason(billingState.suspension_reason) ||
+    !isDirectCreditSuspension({
+      suspensionReason: billingState.suspension_reason,
+      deactivationReason: input.deactivationReason,
+    }) ||
     !hasConfirmedProviderMonitorUpstreamCleanup(input.providerBinding)
   ) {
     return false;
@@ -75,3 +134,35 @@ export function isCreditRestorableDirectMonitorSuspension(input: {
     input.deactivationReason === 'insufficient_balance'
   );
 }
+
+/**
+ * Narrow selector for direct spending-cap suspensions that may be re-provisioned
+ * once the cap admits again (KAR-43): the cap is raised, or the next billing
+ * period begins. The caller proves the cap admits; this proves the row was
+ * switched off for the cap, by the platform, and that upstream cleanup is
+ * confirmed. A customer stop, an opaque billing failure and an unfinished
+ * cleanup are never selected.
+ */
+export function isCapRestorableDirectMonitorSuspension(input: {
+  fleetId?: string;
+  status: string;
+  desired?: 'active' | 'inactive';
+  deactivationReason?: string;
+  billingState: unknown;
+  providerBinding: unknown;
+}): boolean {
+  if (input.fleetId !== undefined || input.status !== 'disabled') return false;
+  const billingState = parseStoredMonitorBillingState(input.billingState);
+  if (
+    billingState.state !== 'billing_suspended' ||
+    !billingState.reactivation_required ||
+    !isMonitorCapSuspensionReason(billingState.suspension_reason) ||
+    !hasConfirmedProviderMonitorUpstreamCleanup(input.providerBinding)
+  ) {
+    return false;
+  }
+  return (
+    input.desired !== 'inactive' ||
+    input.deactivationReason === POLICY_CAP_SUSPENSION_REASON
+  );
+}

package/dist/bundling-sources/shared_libs/monitors/billing/state.ts

@@ -187,6 +187,39 @@ export function monitorBillingRequiresExplicitReactivation(
   );
 }
 
+/**
+ * Whether a monthly-billed monitor is still inside the period it has already
+ * paid for (KAR-43 decision 8).
+ *
+ * A subscription monitor (TamRadar `contact_job_changes`) is billed per month,
+ * not per finding: the first month at create, then renewed by the
+ * `charge-reconcile` cron 24 hours before the provider's `next_charge_at`.
+ * That renewal is its funding gate. Until the paid period ends, a negative
+ * balance or an exceeded cap must not switch it off: its findings add no
+ * charges, the customer keeps the month they paid for, and a recreate would
+ * charge Deepline again (TamRadar has no reactivate endpoint).
+ *
+ * `provider_next_charge_at` is the provider-owned period end; `next_renewal_at`
+ * is Deepline's own periodic renewal. Either one in the future proves a paid
+ * period. An unparseable anchor proves nothing.
+ */
+export function monitorWithinPaidSubscriptionPeriod(
+  value: unknown,
+  now: number,
+): boolean {
+  const state = parseStoredMonitorBillingState(value);
+  if (state.state !== 'active' && state.state !== 'billing_pending') {
+    return false;
+  }
+  return [state.provider_next_charge_at, state.next_renewal_at].some(
+    (anchor) => {
+      if (!anchor) return false;
+      const paidThrough = Date.parse(anchor);
+      return Number.isFinite(paidThrough) && now < paidThrough;
+    },
+  );
+}
+
 export function publicMonitorBillingSummary(
   state: StoredMonitorBillingState,
   customerState: ProviderMonitorCustomerState,

package/dist/bundling-sources/shared_libs/monitors/deactivation-reasons.ts

@@ -1,4 +1,7 @@
-import { monitorBillingRequiresExplicitReactivation } from './billing/state';
+import {
+  monitorBillingRequiresExplicitReactivation,
+  monitorWithinPaidSubscriptionPeriod,
+} from './billing/state';
 
 /**
  * Why a monitor is desired-inactive.
@@ -62,9 +65,9 @@ export const MONITOR_DEACTIVATION_REASONS: readonly MonitorDeactivationReasonDef
       id: 'monthly_cap_exceeded',
       owner: 'policy',
       summary:
-        'Deactivated because the organization reached its monthly monitor spending cap.',
+        'Deactivated because the organization reached its monthly spending cap.',
       resolution:
-        'Raise or reset the monthly cap, then explicitly reactivate the monitors you want restored.',
+        "Raise the cap, or wait for the next billing period. Monitors are reactivated automatically once this period's spend is under the cap again.",
     },
     {
       id: 'billing_failure',
@@ -159,8 +162,15 @@ export type MonitorPolicyRow = {
   desired?: 'active' | 'inactive' | undefined;
   deactivationReason?: string | undefined;
   billingState?: unknown;
+  fleetId?: string | undefined;
 };
 
+/** The two spend reasons policy writes and reverts (KAR-43). */
+export const SPEND_POLICY_REASONS: ReadonlySet<string> = new Set([
+  'insufficient_balance',
+  'monthly_cap_exceeded',
+]);
+
 export type MonitorPolicyDecision =
   | { kind: 'deactivate'; reason: string }
   | { kind: 'reactivate' }
@@ -182,22 +192,59 @@ export type MonitorPolicyDecision =
 export function decideMonitorBalancePolicy(input: {
   row: MonitorPolicyRow;
   balance: number | null;
+  /**
+   * This billing period's recorded spend is at or over the workspace's
+   * monthly cap (KAR-43). Absent means no cap applies.
+   */
+  capExceeded?: boolean;
+  /** Decision time, for the paid-subscription-period exclusion. */
+  now?: number;
 }): MonitorPolicyDecision {
   const admits = balanceAdmitsMonitors(input.balance);
+  const capExceeded = input.capExceeded === true;
   const reason = input.row.deactivationReason;
-  if (!admits) {
+  // An exhausted wallet outranks an exceeded cap: a top-up alone must not
+  // leave a monitor that the cap still holds off labelled as a credit pause.
+  const offReason = !admits
+    ? 'insufficient_balance'
+    : capExceeded
+      ? 'monthly_cap_exceeded'
+      : null;
+  if (offReason !== null) {
     if (input.row.desired === 'inactive') {
       // Already off. Only re-label when policy already owned the reason: a
       // customer's own deactivation must not be re-attributed to billing.
-      return reason === 'insufficient_balance'
+      // A row whose event-time suspension owns recovery keeps its reason.
+      return reason === offReason
         ? { kind: 'none' }
-        : isPolicyDeactivation(reason)
-          ? { kind: 'deactivate', reason: 'insufficient_balance' }
+        : isPolicyDeactivation(reason) &&
+            !monitorBillingRequiresExplicitReactivation(input.row.billingState)
+          ? { kind: 'deactivate', reason: offReason }
           : { kind: 'none' };
     }
-    return { kind: 'deactivate', reason: 'insufficient_balance' };
+    // KAR-43 decision 8: a monthly-billed monitor inside the period it has
+    // already paid for keeps running. Its renewal charge is its funding gate.
+    //
+    // DIRECT MONITORS ONLY. A Fleet's billing suspension is one fleet-wide
+    // lifecycle: the Fleet becomes `billing_suspended`, and only then is it
+    // eligible for automatic reactivation, once EVERY member is disabled. A
+    // member left running would hold its Fleet in `billing_suspending`, which
+    // nothing recovers, so a Fleet member is switched off with its Fleet. The
+    // slot-month rule (`monitorPaidThroughAdmission`) still makes its
+    // recreate inside the paid period free to the customer.
+    if (
+      input.row.fleetId === undefined &&
+      monitorWithinPaidSubscriptionPeriod(
+        input.row.billingState,
+        input.now ?? Date.now(),
+      )
+    ) {
+      return { kind: 'none' };
+    }
+    return { kind: 'deactivate', reason: offReason };
   }
-  // The balance is healthy again. Revive only what THIS policy turned off.
+  // The balance and the cap admit again. Revive only what THIS policy turned
+  // off.
   //
   // ASYMMETRIC ON `null`, DELIBERATELY. An unreadable wallet is permissive for
   // deactivation - "we do not know" must not turn a customer's monitors off -
@@ -209,10 +256,47 @@ export function decideMonitorBalancePolicy(input: {
   if (
     input.balance !== null &&
     input.row.desired === 'inactive' &&
-    reason === 'insufficient_balance' &&
+    reason !== undefined &&
+    SPEND_POLICY_REASONS.has(reason) &&
     !monitorBillingRequiresExplicitReactivation(input.row.billingState)
   ) {
     return { kind: 'reactivate' };
   }
   return { kind: 'none' };
 }
+
+// ---------------------------------------------------------------------------
+// The shutdown notification (KAR-43)
+// ---------------------------------------------------------------------------
+
+/** The two reasons a workspace is told its Monitors were switched off. */
+export type MonitorShutdownNotificationReason =
+  | 'insufficient_credits'
+  | 'monthly_cap_exceeded';
+
+/**
+ * The notification subject for "this workspace's Monitors". The shutdown
+ * notification is one per shutdown, not one per monitor, so its ordering
+ * subject is the workspace's Monitors as a whole.
+ */
+export const WORKSPACE_MONITORS_NOTIFICATION_SUBJECT = 'workspace-monitors';
+
+/**
+ * Collapse every billing reason that switches a Monitor off into the two the
+ * customer acts on: add credits, or raise the cap. Event-time denials, the
+ * policy's credit pause and the cap pause all name the same remedy. Anything
+ * else (an opaque billing failure, a customer stop) is not this notification.
+ */
+export function monitorShutdownNotificationReason(
+  reason: string | null | undefined,
+): MonitorShutdownNotificationReason | null {
+  if (
+    reason === 'insufficient_credits' ||
+    reason === 'insufficient_credits_after_provider_event' ||
+    reason === 'insufficient_balance'
+  ) {
+    return 'insufficient_credits';
+  }
+  if (reason === 'monthly_cap_exceeded') return 'monthly_cap_exceeded';
+  return null;
+}

package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-dead-letter.ts

@@ -100,6 +100,9 @@ type DeadLetterMessageInput = Pick<
   radarKey?: string | null;
   eventIdempotencyKey?: string | null;
   rawBody?: string;
+  postgresSchedulerSchema?: string | null;
+  runtimeApiBaseUrl?: string | null;
+  deadLetterId?: string | null;
 };
 
 async function markCapturedRawWriteFailed(
@@ -132,6 +135,12 @@ async function markCapturedRawWriteFailed(
 // null-keyed poison events stay distinct (the partial index only enforces
 // uniqueness WHERE event_idempotency_key IS NOT NULL). Also marks the captured
 // raw row write_failed when one exists for the key.
+//
+// A redriven event that fails again returns its row to `open` with the
+// attempts added up, so a row never claims `replaying` or `replayed` for an
+// event that was not processed. A redrive message names its row
+// (`deadLetterId`), which also covers a null-keyed event the index cannot
+// match. A `discarded` row stays discarded.
 export async function recordSignalRadarDeadLetter(input: {
   client: InboundPgClient;
   message: DeadLetterMessageInput;
@@ -151,18 +160,53 @@ export async function recordSignalRadarDeadLetter(input: {
     headers: message.headers ?? {},
     receivedAt: message.receivedAt,
     admission: message.admission ?? null,
+    // A Preview callback runs on its own scheduler lane and calls back to its
+    // own origin; a redrive must keep both.
+    postgresSchedulerSchema: message.postgresSchedulerSchema ?? null,
+    runtimeApiBaseUrl: message.runtimeApiBaseUrl ?? null,
   };
 
+  if (message.deadLetterId) {
+    const reopened = await client.query(
+      `UPDATE dl_meta.signal_radar_inbound_dead_letter
+          SET status = CASE WHEN status = 'discarded' THEN status ELSE 'open' END,
+              attempts = attempts + $2,
+              last_error = $3,
+              updated_at = now()
+        WHERE dead_letter_id = $1
+        RETURNING dead_letter_id`,
+      [message.deadLetterId, input.attempts, lastError.slice(0, 8000)],
+    );
+    if ((reopened.rows ?? []).length > 0) {
+      if (eventIdempotencyKey) {
+        await markCapturedRawWriteFailed(
+          client,
+          eventIdempotencyKey,
+          lastError,
+        );
+      }
+      return;
+    }
+  }
+
   if (eventIdempotencyKey) {
     await client.query(
-      `INSERT INTO dl_meta.signal_radar_inbound_dead_letter
+      `INSERT INTO dl_meta.signal_radar_inbound_dead_letter AS dead_letter
          (dead_letter_id, provider, radar_key, org_id, event_idempotency_key,
           payload, headers, attempts, last_error, status)
        VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7::jsonb, $8, $9, 'open')
        ON CONFLICT (event_idempotency_key)
          WHERE event_idempotency_key IS NOT NULL
          DO UPDATE SET
-           attempts = EXCLUDED.attempts,
+           status = CASE
+             WHEN dead_letter.status IN ('replaying', 'replayed') THEN 'open'
+             ELSE dead_letter.status
+           END,
+           attempts = CASE
+             WHEN dead_letter.status IN ('replaying', 'replayed')
+               THEN dead_letter.attempts + EXCLUDED.attempts
+             ELSE EXCLUDED.attempts
+           END,
            last_error = EXCLUDED.last_error,
            payload = EXCLUDED.payload,
            headers = EXCLUDED.headers,

package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-message.ts

@@ -35,4 +35,8 @@ export type SignalRadarInboundMessage = {
   // Optional radar hint the worker resolves durably from Neon; present only on
   // dead-letter/replay envelopes where it was already known.
   radarKey?: string | null;
+  // The tenant dead-letter row this message redrives. The worker settles that
+  // row from the outcome: `replayed` once processed, back to `open` when it
+  // dead-letters again. Absent on every first delivery.
+  deadLetterId?: string | null;
 };

package/dist/bundling-sources/shared_libs/observability/queue-health.ts

@@ -219,19 +219,19 @@ export const OPERATIONS_QUEUE_CATALOG = [
   ),
   queue(
     'signal-radar-inbound-legacy-dlq',
-    'Signal-radar legacy inbound dead-letters',
+    'Signal-radar inbound dead letters',
     'provider_queue',
     'monitors',
-    'Tenant-scoped legacy poison-callback records; counts can overlap raw write failures',
+    'Tenant-scoped inbound callbacks that exhausted their deliveries; redrive (retry) or retire one item; counts can overlap raw receipts',
     300_000,
     900_000,
   ),
   queue(
     'signal-radar-inbound-typed-write-failures',
-    'Signal-radar typed-write failures',
+    'Signal-radar raw receipts not written',
     'provider_queue',
     'monitors',
-    'Tenant-scoped raw callback receipts whose typed write failed; records can overlap legacy dead-letters',
+    'Read-only evidence: tenant raw callback receipts not written as rows, including deliberate raw-only captures; act on the dead-letter item',
     300_000,
     900_000,
   ),

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -909,6 +909,42 @@
     "failureAlert": "convex_action_wrapper",
     "domainReportRequired": true
   },
+  {
+    "id": "convex.monitor-cap-change",
+    "name": "Monitor spending-cap change",
+    "scheduler": "convex_scheduled_action",
+    "cadence": "runAfter(0) when a workspace that has Monitors changes or removes its spending cap",
+    "timezone": "UTC",
+    "source": "packages/convex/scheduledJobsNode.ts",
+    "locator": "reactToCreditBalanceChange",
+    "owner": "billing",
+    "criticality": "critical",
+    "purpose": "Re-decide a workspace's Monitors against its changed spending cap: revive cap-paused Monitors through the fenced billing-recovery lifecycle once the cap admits, or switch them off when spend already exceeds a lowered cap.",
+    "successDefinition": "The bounded policy pass completes and submits any durable operation it needs, or records a valid no-op.",
+    "failureDefinition": "The policy pass cannot run. The hourly provider-monitor policy sweep remains the backstop.",
+    "startGraceMs": 120000,
+    "runTimeoutMs": 300000,
+    "failureAlert": "convex_action_wrapper",
+    "domainReportRequired": true
+  },
+  {
+    "id": "convex.monitor-spend-shutdown",
+    "name": "Monitor spend shutdown",
+    "scheduler": "convex_scheduled_action",
+    "cadence": "runAfter(0) from the per-workspace notification worker when a queued balance change goes below 0 or a spending-cap denial is recorded while a Monitor runs, and from a failed automatic recharge",
+    "timezone": "UTC",
+    "source": "packages/convex/scheduledJobsNode.ts",
+    "locator": "reactToCreditBalanceChange",
+    "owner": "billing",
+    "criticality": "critical",
+    "purpose": "Switch every Monitor in a workspace off for credit or the spending cap as soon as spend makes it necessary, instead of waiting for the hourly policy sweep. Direct monthly-billed Monitors inside their paid period are left running.",
+    "successDefinition": "The balance or cap is re-checked and the bounded policy pass submits its durable operations, or a valid no-op is recorded (balance back above 0, cap admits, automatic recharge in progress, no running Monitor).",
+    "failureDefinition": "The re-check or a policy page cannot run. The hourly provider-monitor policy sweep remains the backstop.",
+    "startGraceMs": 120000,
+    "runTimeoutMs": 300000,
+    "failureAlert": "convex_action_wrapper",
+    "domainReportRequired": true
+  },
   {
     "id": "inngest.workflow-run-watchdog",
     "name": "Workflow run watchdog",

package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts

@@ -7,6 +7,7 @@ import type {
 import { PRODUCT_NOTIFICATION_EVENT_CATALOG } from './contract';
 import {
   ProductNotificationEvent,
+  ProductNotificationTargetPath,
   getProductNotificationEventDefinition,
   productNotificationTriggerIsAllowed,
   type ProductNotificationEventName,
@@ -136,6 +137,30 @@ function validateInsufficientCreditDisable(
   }
 }
 
+function validateMonthlyCapMonitorDisable(trigger: ProductNotificationTrigger) {
+  if (
+    trigger.kind !== 'state_transition' ||
+    trigger.targetState !== 'disabled' ||
+    trigger.reasonCode !== 'monthly_cap_exceeded'
+  ) {
+    throw new Error(
+      'Spending-cap Monitor rules must enter disabled for monthly_cap_exceeded.',
+    );
+  }
+}
+
+/**
+ * KAR-43 decision 7: a shutdown notification goes only to a workspace that
+ * has at least one Monitor. A workspace-wide shutdown event names how many
+ * Monitors it switched off; a per-Monitor event is itself a Monitor.
+ */
+function switchedOffAMonitor(payload: ProductNotificationFacts): boolean {
+  return (
+    payload.affectedMonitorCount === undefined ||
+    Number(payload.affectedMonitorCount) >= 1
+  );
+}
+
 function validateFailedOutcome(trigger: ProductNotificationTrigger) {
   if (trigger.kind === 'consecutive_outcomes' && trigger.outcome !== 'failed') {
     throw new Error('Failure rules require failed consecutive outcomes.');
@@ -321,14 +346,19 @@ const PRODUCT_NOTIFICATION_TYPE_REGISTRY = [
     requiredFilter: {},
     allowedTriggers: ['state_transition'],
     catalog: {
-      name: 'Monitor disabled due to insufficient credits',
+      name: 'Monitors turned off due to insufficient credits',
       category: 'credits',
       description:
-        'Notify when Deepline disables a Monitor because the workspace ran out of credits.',
+        "Notify when Deepline turns off the workspace's Monitors because it ran out of credits. Adding credits turns them back on.",
       allowMultiple: false,
       editor: { kind: 'none' },
     },
     rule: {
+      // KAR-43: on by default for workspace owners, once per shutdown. The
+      // workspace condition fires on the first Monitor switched off and
+      // rearms when the balance rises.
+      default: { ruleKey: 'default.monitor_disabled', groupKey: 'default' },
+      eligible: switchedOffAMonitor,
       conditionSubject: 'workspace',
       rearm: {
         eventName: ProductNotificationEvent.CreditsBalanceChanged,
@@ -345,15 +375,72 @@ const PRODUCT_NOTIFICATION_TYPE_REGISTRY = [
       }),
       validate: validateInsufficientCreditDisable,
       present: ({ payload }) => ({
-        title: 'Monitor disabled due to insufficient credits',
+        title: 'Monitors turned off: out of credits',
         summary: [
-          `${payload.monitorName} was disabled because your `,
+          'Deepline turned off the Monitors in your ',
           { workspace: true },
-          ' ran out of credits.',
+          ' because it ran out of credits.',
         ],
-        details: [],
-        targetUrl: String(payload.targetUrl),
-        subjectKind: 'monitor',
+        details: [
+          'Add credits to turn them back on. Monitors resume automatically once the balance is above zero.',
+          'Events that arrive while Monitors are off are not delivered later.',
+        ],
+        targetUrl: ProductNotificationTargetPath.Billing,
+        subjectKind: 'workspace_monitors',
+        subjectId: String(payload.monitorId),
+        severity: 'error',
+      }),
+    },
+  },
+  {
+    id: 'monitor.disabled.monthly_cap_exceeded',
+    eventName: ProductNotificationEvent.MonitorLifecycleChanged,
+    requiredFilter: {},
+    allowedTriggers: ['state_transition'],
+    catalog: {
+      name: 'Monitors turned off at the spending cap',
+      category: 'credits',
+      description:
+        "Notify when Deepline turns off the workspace's Monitors because it reached its monthly spending cap. Raising the cap, or the next billing period, turns them back on.",
+      allowMultiple: false,
+      editor: { kind: 'none' },
+    },
+    rule: {
+      default: {
+        ruleKey: 'default.monitor_disabled_monthly_cap',
+        groupKey: 'default',
+      },
+      eligible: switchedOffAMonitor,
+      conditionSubject: 'workspace',
+      // Monitors coming back under the cap rearm it for the next shutdown.
+      rearm: {
+        eventName: ProductNotificationEvent.MonitorLifecycleChanged,
+        matches: (payload) =>
+          payload.currentState === 'active' &&
+          payload.reasonCode === 'monthly_cap_exceeded',
+      },
+      create: (_input, base) => ({
+        ...base,
+        trigger: {
+          kind: 'state_transition',
+          targetState: 'disabled',
+          reasonCode: 'monthly_cap_exceeded',
+        },
+      }),
+      validate: validateMonthlyCapMonitorDisable,
+      present: ({ payload }) => ({
+        title: 'Monitors turned off: spending cap reached',
+        summary: [
+          'Deepline turned off the Monitors in your ',
+          { workspace: true },
+          ' because it reached its monthly spending cap.',
+        ],
+        details: [
+          'Raise the cap to turn them back on. Otherwise they resume automatically when the next billing period begins.',
+          'Events that arrive while Monitors are off are not delivered later.',
+        ],
+        targetUrl: ProductNotificationTargetPath.Billing,
+        subjectKind: 'workspace_monitors',
         subjectId: String(payload.monitorId),
         severity: 'error',
       }),

package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts

@@ -20,7 +20,7 @@ import {
  * they create sparse copy-on-write overrides.
  */
 export const CURRENT_PRODUCT_NOTIFICATION_DEFAULT_POLICY = {
-  version: 10,
+  version: 11,
   groups: [
     {
       groupKey: 'default',

package/dist/bundling-sources/shared_libs/product-notifications/events.ts

@@ -128,7 +128,11 @@ export const PRODUCT_NOTIFICATION_EVENT_DEFINITIONS: readonly ProductNotificatio
         targetUrl: 'string',
       },
       orderingSubjectFact: 'monitorId',
-      optionalFacts: { previousReasonCode: 'string' },
+      optionalFacts: {
+        previousReasonCode: 'string',
+        /** A workspace-wide shutdown names how many Monitors it switched off. */
+        affectedMonitorCount: 'number',
+      },
       allowedTriggers: ['state_transition'],
     },
     {