deepline 0.3.74 → 0.3.76

package/dist/bundling-sources/shared_libs/monitors/deactivation-reasons.ts

@@ -0,0 +1,186 @@
+/**
+ * Why a monitor is desired-inactive.
+ *
+ * ONE ENUMERATION, ONE MODULE
+ * ---------------------------
+ * `deactivationReason` distinguishes customer intent from platform action, and
+ * that distinction is only worth anything if the set of reasons is closed. A
+ * reason invented at a call site is a reason no drift output explains, no
+ * billing reader recognises, and no UI can phrase. So every reason is declared
+ * here, with who owns it and whether the platform may revive the monitor when
+ * the condition clears.
+ *
+ * ABSENT IS NOT A REASON
+ * ----------------------
+ * A monitor with `desired: 'inactive'` and NO reason is the customer's own
+ * decision. The platform must never revive it, never explain it, and never
+ * count it as an outage. That is the whole point of the field: "you turned
+ * this off" and "we did, and here is why" must not be the same state.
+ */
+
+export type MonitorDeactivationOwner = 'policy' | 'fleet';
+
+/**
+ * The delivery window, in days, that `monitors health` reports against.
+ *
+ * Stated here in the unit a customer reads so the CLI, the API envelope and
+ * the Convex predicate cannot disagree about what "silent" means. The
+ * millisecond form the predicate uses is derived from it.
+ */
+export const SILENT_MONITOR_WINDOW_DAYS = 3;
+
+export type MonitorDeactivationReasonDefinition = {
+  id: string;
+  /** Who writes it. Only `policy` reasons are automatically reverted. */
+  owner: MonitorDeactivationOwner;
+  /** One line, customer-facing. This is what drift output prints. */
+  summary: string;
+  /** What the customer does about it. */
+  resolution: string;
+};
+
+export const MONITOR_DEACTIVATION_REASONS: readonly MonitorDeactivationReasonDefinition[] =
+  [
+    {
+      id: 'insufficient_balance',
+      owner: 'policy',
+      summary:
+        'Deactivated because the organization has no Deepline credit left.',
+      resolution:
+        'Add credits. Monitors are reactivated automatically once the balance is positive again.',
+    },
+    {
+      id: 'entitlement_expired',
+      owner: 'policy',
+      summary:
+        'Deactivated because the entitlement that admitted this monitor has expired.',
+      resolution:
+        'Restore the entitlement. Monitors are reactivated automatically once it is in force.',
+    },
+    {
+      id: 'fleet_removed',
+      owner: 'fleet',
+      summary:
+        'Deactivated because the fleet that owned this monitor no longer wants it.',
+      resolution:
+        'Re-add the source row and re-sync the fleet, or leave it deactivated.',
+    },
+  ];
+
+const BY_ID = new Map(
+  MONITOR_DEACTIVATION_REASONS.map((reason) => [reason.id, reason]),
+);
+
+export const MONITOR_DEACTIVATION_REASON_IDS: readonly string[] =
+  MONITOR_DEACTIVATION_REASONS.map((reason) => reason.id);
+
+export function monitorDeactivationReason(
+  id: string | undefined | null,
+): MonitorDeactivationReasonDefinition | null {
+  if (!id) return null;
+  return BY_ID.get(id) ?? null;
+}
+
+/**
+ * Refuse an undeclared reason loudly.
+ *
+ * A writer that could store an unknown string would make the enumeration
+ * advisory, and an advisory enumeration is the state this field exists to
+ * prevent.
+ */
+export function requireMonitorDeactivationReason(
+  id: string,
+): MonitorDeactivationReasonDefinition {
+  const reason = BY_ID.get(id);
+  if (!reason) {
+    throw new Error(
+      `Unknown monitor deactivation reason '${id}'. Declared reasons: ${MONITOR_DEACTIVATION_REASON_IDS.join(', ')}.`,
+    );
+  }
+  return reason;
+}
+
+/** True when the platform, not the customer, turned this monitor off. */
+export function isPolicyDeactivation(id: string | undefined | null): boolean {
+  return monitorDeactivationReason(id)?.owner === 'policy';
+}
+
+// ---------------------------------------------------------------------------
+// The balance policy
+// ---------------------------------------------------------------------------
+
+/**
+ * The strictly-positive rule, stated once.
+ *
+ * A negative balance is permitted where provider truth settles late, but an
+ * organization at or below zero opens no new charges. A monitor is a standing
+ * charge, so the same rule governs whether it may keep running.
+ */
+export function balanceAdmitsMonitors(balance: number | null): boolean {
+  // A wallet nobody has seeded is not a wallet at zero. It is an organization
+  // whose balance has never been read, and deactivating every monitor it owns
+  // on the strength of an absent record would be the worst possible reading of
+  // "we do not know".
+  if (balance === null) return true;
+  return balance > 0;
+}
+
+export type MonitorPolicyRow = {
+  desired?: 'active' | 'inactive' | undefined;
+  deactivationReason?: string | undefined;
+};
+
+export type MonitorPolicyDecision =
+  | { kind: 'deactivate'; reason: string }
+  | { kind: 'reactivate' }
+  | { kind: 'none' };
+
+/**
+ * What policy wants for one monitor, given the organization's balance.
+ *
+ * Pure, so the rule can be driven in every direction by a unit test without a
+ * wallet, a Convex runtime, or a monitor. The three cases that matter are all
+ * about NOT touching something:
+ *
+ *   - a customer deactivation (inactive, no reason) is never revived;
+ *   - a fleet deactivation is never revived by policy either - the fleet owns
+ *     that intent and a re-sync is what restores it;
+ *   - a monitor already deactivated for this exact reason is left alone, so a
+ *     repeated sweep writes nothing.
+ */
+export function decideMonitorBalancePolicy(input: {
+  row: MonitorPolicyRow;
+  balance: number | null;
+}): MonitorPolicyDecision {
+  const admits = balanceAdmitsMonitors(input.balance);
+  const reason = input.row.deactivationReason;
+  if (!admits) {
+    if (input.row.desired === 'inactive') {
+      // Already off. Only re-label when policy already owned the reason: a
+      // customer's own deactivation must not be re-attributed to billing.
+      return reason === 'insufficient_balance'
+        ? { kind: 'none' }
+        : isPolicyDeactivation(reason)
+          ? { kind: 'deactivate', reason: 'insufficient_balance' }
+          : { kind: 'none' };
+    }
+    return { kind: 'deactivate', reason: 'insufficient_balance' };
+  }
+  // The balance is healthy again. Revive only what THIS policy turned off.
+  //
+  // ASYMMETRIC ON `null`, DELIBERATELY. An unreadable wallet is permissive for
+  // deactivation - "we do not know" must not turn a customer's monitors off -
+  // and restrictive here, because reviving is not a state flip. The provider
+  // has no activate endpoint, so revival RE-CREATES the monitor and CHARGES for
+  // it, and an absent record is not evidence that anybody paid. An organization
+  // whose wallet cannot be read is revived by the next pass that can read it,
+  // or by the customer running the reactivate verb.
+  if (
+    input.balance !== null &&
+    input.row.desired === 'inactive' &&
+    reason === 'insufficient_balance'
+  ) {
+    return { kind: 'reactivate' };
+  }
+  return { kind: 'none' };
+}

package/dist/bundling-sources/shared_libs/monitors/monitor-fleet-limits.ts

@@ -14,3 +14,44 @@
  * the contract module's copy and never learns this file exists.
  */
 export const MONITOR_FLEET_MAX_MEMBERS = 1_000;
+
+/**
+ * A claim may carry at most this many independent (organization, fleet)
+ * batches.
+ *
+ * A lease starts when the claim commits, not when a particular batch reaches
+ * the provider.  Letting a 100-item claim contain 100 one-member fleets meant
+ * a sequential dispatcher could leave the later leases expired before it ever
+ * started their work.  A second tick then legitimately reclaimed them while
+ * the first was still settling, which is exactly the write contention a lease
+ * is meant to prevent.  Dispatch these groups concurrently, bounded by this
+ * same number, so every claimed group begins within one bounded pass. The app
+ * and Convex both use the bounded dispatcher below so a rolling deployment
+ * remains safe while an older claim mutation is still live.
+ */
+export const MONITOR_FLEET_MAX_DISPATCH_GROUPS_PER_TICK = 4;
+
+/** Run independent fleet dispatches without letting one lease wait behind all others. */
+export async function dispatchMonitorFleetGroups<T, R>(
+  groups: readonly T[],
+  dispatch: (group: T) => Promise<R>,
+): Promise<R[]> {
+  const results: R[] = Array.from({ length: groups.length });
+  let next = 0;
+  const workers = Array.from(
+    {
+      length: Math.min(
+        MONITOR_FLEET_MAX_DISPATCH_GROUPS_PER_TICK,
+        groups.length,
+      ),
+    },
+    async () => {
+      while (next < groups.length) {
+        const index = next++;
+        results[index] = await dispatch(groups[index]!);
+      }
+    },
+  );
+  await Promise.all(workers);
+  return results;
+}

package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts

@@ -90,17 +90,12 @@ export type AsyncOperationContractWire = {
     finishAction?: AsyncOperationActionCall;
     cancelAction?: AsyncOperationActionCall;
   };
-  capacity: {
-    scope:
-      | 'organization_provider'
-      | 'organization_connection'
-      | 'organization_action';
-    limit:
-      | { kind: 'provider_registry' }
-      | { kind: 'fixed'; maxConcurrentJobs: number };
-    onConflict: 'wait' | 'fail';
-    leaseTtlMs: number;
-  };
+  /**
+   * A provider may acknowledge a created job with its handle but no lifecycle
+   * status. This rule applies only to that launch response; unknown poll
+   * responses still use terminal.defaultOutcome.
+   */
+  launch?: { unclassifiedJobOutcome: 'running' };
   job: { idPaths: readonly string[] };
   polling: {
     initialDelayMs: number;
@@ -112,6 +107,8 @@ export type AsyncOperationContractWire = {
     timeoutMs: number;
     /** Optional public start-input field that overrides timeoutMs for this call. */
     timeoutInputField?: string;
+    /** Optional public start-input field that bounds poll attempts for this call. */
+    maxAttemptsInputField?: string;
   };
   terminal: {
     rules: readonly AsyncOperationTerminalRule[];
@@ -124,7 +121,8 @@ export type AsyncOperationContractWire = {
         kind: 'input_flag';
         field: string;
         default: boolean;
-        durableValue?: boolean;
+        /** Value sent only to the provider launch action when a Play polls. */
+        launchValue?: boolean;
       }
     | { kind: 'caller_managed' };
 };
@@ -260,13 +258,6 @@ export function defineAsyncOperation(
       validateResultValueSource(source, `result.objectFields.${field}`);
     }
   }
-  positiveInteger(contract.capacity.leaseTtlMs, 'capacity.leaseTtlMs');
-  if (contract.capacity.limit.kind === 'fixed') {
-    positiveInteger(
-      contract.capacity.limit.maxConcurrentJobs,
-      'capacity.limit.maxConcurrentJobs',
-    );
-  }
   nonNegativeInteger(contract.polling.initialDelayMs, 'polling.initialDelayMs');
   positiveInteger(contract.polling.intervalMs, 'polling.intervalMs');
   positiveInteger(contract.polling.timeoutMs, 'polling.timeoutMs');
@@ -276,6 +267,12 @@ export function defineAsyncOperation(
   if (contract.polling.timeoutInputField !== undefined) {
     required(contract.polling.timeoutInputField, 'polling.timeoutInputField');
   }
+  if (contract.polling.maxAttemptsInputField !== undefined) {
+    required(
+      contract.polling.maxAttemptsInputField,
+      'polling.maxAttemptsInputField',
+    );
+  }
   if (
     contract.polling.backoffMultiplier !== undefined &&
     (!Number.isFinite(contract.polling.backoffMultiplier) ||
@@ -379,7 +376,11 @@ function positiveStartInputNumber(
 export function resolveAsyncOperationPolling(input: {
   contract: AsyncOperationContract;
   startInput: Readonly<Record<string, unknown>>;
-}): { intervalMs: number; timeoutMs: number } {
+}): { intervalMs: number; timeoutMs: number; maxAttempts?: number } {
+  const maxAttempts = positiveStartInputNumber(
+    input.startInput,
+    input.contract.polling.maxAttemptsInputField,
+  );
   return {
     intervalMs:
       positiveStartInputNumber(
@@ -391,6 +392,7 @@ export function resolveAsyncOperationPolling(input: {
         input.startInput,
         input.contract.polling.timeoutInputField,
       ) ?? input.contract.polling.timeoutMs,
+    ...(maxAttempts ? { maxAttempts: Math.floor(maxAttempts) } : {}),
   };
 }
 
@@ -429,12 +431,6 @@ export function defineCallerManagedAsyncOperation(input: {
         ? { finishAction: companion(input.finishAction) }
         : {}),
     },
-    capacity: {
-      scope: 'organization_provider',
-      limit: { kind: 'provider_registry' },
-      onConflict: 'wait',
-      leaseTtlMs: 15 * 60_000,
-    },
     job: {
       idPaths: input.jobIdPaths ?? [
         'job_id',
@@ -527,14 +523,21 @@ export function asyncOperationShouldWait(
   return typeof value === 'boolean' ? value : contract.wait.default;
 }
 
-export function asyncOperationDurableStartInput(
+/**
+ * A Play starts a provider job without asking provider-local code to wait.
+ *
+ * This only shapes the launch request. The external handle remains in the
+ * active lifecycle call stack; it is never written into or recovered from a
+ * result receipt. Receipts cache terminal tool output only.
+ */
+export function asyncOperationLaunchInput(
   contract: AsyncOperationContractWire,
   input: Record<string, unknown>,
 ): Record<string, unknown> {
   if (contract.wait.kind !== 'input_flag') return input;
   return {
     ...input,
-    [contract.wait.field]: contract.wait.durableValue ?? false,
+    [contract.wait.field]: contract.wait.launchValue ?? false,
   };
 }
 
@@ -621,6 +624,18 @@ export function classifyAsyncOperationResult(
   return { outcome: contract.terminal.defaultOutcome };
 }
 
+export function hasExplicitAsyncOperationTerminalMatch(
+  contract: AsyncOperationContractWire,
+  result: unknown,
+): boolean {
+  return contract.terminal.rules.some((rule) => {
+    const matches = rule.conditions.map((condition) =>
+      conditionMatches(result, condition),
+    );
+    return rule.mode === 'any' ? matches.some(Boolean) : matches.every(Boolean);
+  });
+}
+
 function isEmptyAsyncResult(value: unknown): boolean {
   return (
     value == null ||
@@ -753,13 +768,57 @@ export function asyncOperationPollDelayMs(
   return Math.min(delay, contract.polling.maxIntervalMs ?? delay);
 }
 
+function retryablePollRateLimit(error: unknown): {
+  retryAfterMs: number | null;
+} | null {
+  if (!error || typeof error !== 'object' || Array.isArray(error)) {
+    return null;
+  }
+  const record = error as Record<string, unknown>;
+  const status = record.status ?? record.statusCode;
+  // The Play transport emits this typed retryable envelope after it has
+  // already informed the shared provider pacer. Direct provider runtimes may
+  // deliberately expose provider-specific incomplete-result responses, so do
+  // not intercept their untyped 429s here.
+  if (status !== 429 || record.retryable !== true) {
+    return null;
+  }
+  const retryAfterMs = record.retryAfterMs;
+  return {
+    retryAfterMs:
+      typeof retryAfterMs === 'number' &&
+      Number.isFinite(retryAfterMs) &&
+      retryAfterMs >= 0
+        ? Math.ceil(retryAfterMs)
+        : null,
+  };
+}
+
+function asyncOperationRateLimitPollDelayMs(input: {
+  contract: AsyncOperationContractWire;
+  pollAttempt: number;
+  rateLimitAttempt: number;
+  retryAfterMs: number | null;
+}): number {
+  // A rejected poll did not observe the remote job, so it must not consume the
+  // caller's max_polls budget. It does, however, need a real delay even when
+  // the first normal poll was immediate; otherwise concurrent rows retry in a
+  // tight herd. Never shorten an explicit provider Retry-After.
+  const localBackoffMs = asyncOperationPollDelayMs(
+    input.contract,
+    Math.max(1, input.pollAttempt + input.rateLimitAttempt),
+  );
+  return Math.max(input.retryAfterMs ?? 0, localBackoffMs);
+}
+
 /**
  * Runtime-neutral executor for a declared async lifecycle.
  *
  * Provider runtimes and Play execution intentionally share this state machine:
  * the provider contract owns polling inputs, terminal classification, result
- * pagination, and materialization. Callers own transport, durable receipts,
- * retry policy, billing, and recovery persistence around each action.
+ * pagination, and materialization. Callers own transport retries, billing,
+ * and terminal-result caching. This executor never persists a provider handle
+ * or turns a receipt into provider-job state.
  */
 export async function executeAsyncOperationLifecycle<Response>(input: {
   contract: AsyncOperationContractWire;
@@ -774,6 +833,10 @@ export async function executeAsyncOperationLifecycle<Response>(input: {
     phase: 'poll' | 'finish' | 'finish_page';
     pollAttempt: number;
     page?: number;
+    /** Remaining lifecycle budget for this physical provider request. */
+    timeoutMs: number;
+    /** Aborts a physical provider request when that budget expires. */
+    abortSignal: AbortSignal;
   }) => Promise<Response>;
   responseValue: (response: Response) => unknown;
   sleep?: (ms: number) => Promise<void>;
@@ -799,6 +862,7 @@ export async function executeAsyncOperationLifecycle<Response>(input: {
       jobId: string;
       pollAttempt: number;
       previousPollResult: unknown;
+      reason: 'timeout' | 'max_attempts';
     }
 > {
   const jobId =
@@ -825,31 +889,125 @@ export async function executeAsyncOperationLifecycle<Response>(input: {
   };
   let pollAttempt = 0;
   let previousPollResult: unknown = undefined;
+  let nextPollDelayMs: number | null = null;
+  let consecutiveRateLimitAttempts = 0;
+  const timedOut = () => ({
+    kind: 'timed_out' as const,
+    jobId,
+    pollAttempt,
+    previousPollResult,
+    reason: 'timeout' as const,
+  });
+  const executeActionWithinDeadline = async (request: {
+    action: string;
+    input: Record<string, unknown>;
+    phase: 'poll' | 'finish' | 'finish_page';
+    pollAttempt: number;
+    page?: number;
+  }): Promise<
+    { kind: 'response'; response: Response } | { kind: 'timed_out' }
+  > => {
+    const remainingTimeoutMs = Math.max(
+      0,
+      polling.timeoutMs - (now() - startedAt),
+    );
+    if (remainingTimeoutMs <= 0) return { kind: 'timed_out' };
+    const abortSignal = AbortSignal.timeout(Math.ceil(remainingTimeoutMs));
+    try {
+      const response = await input.executeAction({
+        ...request,
+        jobId,
+        timeoutMs: Math.ceil(remainingTimeoutMs),
+        abortSignal,
+      });
+      return now() - startedAt >= polling.timeoutMs || abortSignal.aborted
+        ? { kind: 'timed_out' }
+        : { kind: 'response', response };
+    } catch (error) {
+      if (abortSignal.aborted) return { kind: 'timed_out' };
+      throw error;
+    }
+  };
   while (true) {
+    if (
+      polling.maxAttempts !== undefined &&
+      pollAttempt >= polling.maxAttempts
+    ) {
+      return {
+        kind: 'timed_out',
+        jobId,
+        pollAttempt,
+        previousPollResult,
+        reason: 'max_attempts',
+      };
+    }
     if (now() - startedAt >= polling.timeoutMs) {
-      return { kind: 'timed_out', jobId, pollAttempt, previousPollResult };
+      return {
+        kind: 'timed_out',
+        jobId,
+        pollAttempt,
+        previousPollResult,
+        reason: 'timeout',
+      };
     }
     const delayMs =
-      pollAttempt === 0 && input.pollImmediately === true
+      nextPollDelayMs ??
+      (pollAttempt === 0 && input.pollImmediately === true
         ? 0
-        : asyncOperationPollDelayMs(effectiveContract, pollAttempt);
-    if (delayMs > 0) await sleep(delayMs);
-    let pollResponse: Response | null = null;
-    for (const action of input.contract.lifecycle.pollActions) {
-      pollResponse = await input.executeAction({
-        action: action.action,
-        input: buildAsyncOperationActionInput({
-          mapping: action.input,
-          jobId,
-          startInput: input.startInput,
-          startResult: input.startResult,
-          pollResult: previousPollResult,
-        }),
+        : asyncOperationPollDelayMs(effectiveContract, pollAttempt));
+    nextPollDelayMs = null;
+    const remainingTimeoutMs = Math.max(
+      0,
+      polling.timeoutMs - (now() - startedAt),
+    );
+    if (delayMs > 0 && remainingTimeoutMs > 0) {
+      await sleep(Math.min(delayMs, remainingTimeoutMs));
+    }
+    // A provider Retry-After can exceed the caller's remaining wait budget.
+    // Never issue a late poll after either a normal cadence or rate-limit
+    // delay has consumed that budget.
+    if (now() - startedAt >= polling.timeoutMs) {
+      return {
+        kind: 'timed_out',
         jobId,
-        phase: 'poll',
         pollAttempt,
+        previousPollResult,
+        reason: 'timeout',
+      };
+    }
+    let pollResponse: Response | null = null;
+    let currentPollResult = previousPollResult;
+    try {
+      for (const action of input.contract.lifecycle.pollActions) {
+        const actionExecution = await executeActionWithinDeadline({
+          action: action.action,
+          input: buildAsyncOperationActionInput({
+            mapping: action.input,
+            jobId,
+            startInput: input.startInput,
+            startResult: input.startResult,
+            pollResult: currentPollResult,
+          }),
+          phase: 'poll',
+          pollAttempt,
+        });
+        if (actionExecution.kind === 'timed_out') return timedOut();
+        pollResponse = actionExecution.response;
+        currentPollResult = input.responseValue(pollResponse);
+      }
+      previousPollResult = currentPollResult;
+      consecutiveRateLimitAttempts = 0;
+    } catch (error) {
+      const rateLimit = retryablePollRateLimit(error);
+      if (!rateLimit) throw error;
+      consecutiveRateLimitAttempts += 1;
+      nextPollDelayMs = asyncOperationRateLimitPollDelayMs({
+        contract: effectiveContract,
+        pollAttempt,
+        rateLimitAttempt: consecutiveRateLimitAttempts,
+        retryAfterMs: rateLimit.retryAfterMs,
       });
-      previousPollResult = input.responseValue(pollResponse);
+      continue;
     }
     if (!pollResponse) {
       throw new Error(
@@ -899,13 +1057,14 @@ export async function executeAsyncOperationLifecycle<Response>(input: {
           startResult: input.startResult,
           pollResult: previousPollResult,
         });
-        finishResponse = await input.executeAction({
+        const actionExecution = await executeActionWithinDeadline({
           action: finishAction.action,
           input: finishInput,
-          jobId,
           phase: 'finish',
           pollAttempt,
         });
+        if (actionExecution.kind === 'timed_out') return timedOut();
+        finishResponse = actionExecution.response;
         finishResult = input.responseValue(finishResponse);
         const pagination = finishAction.pagination;
         if (pagination) {
@@ -937,10 +1096,8 @@ export async function executeAsyncOperationLifecycle<Response>(input: {
             // response instead of issuing unbounded empty page reads.
             return (
               !pagination.lastPagePath ||
-              readAsyncOperationPath(
-                currentResult,
-                pagination.lastPagePath,
-              ) !== false
+              readAsyncOperationPath(currentResult, pagination.lastPagePath) !==
+                false
             );
           };
           while (!isLastPage()) {
@@ -950,14 +1107,15 @@ export async function executeAsyncOperationLifecycle<Response>(input: {
                 `Async operation ${input.contract.lifecycle.startAction} exceeded its ${pagination.maxPages}-page result limit.`,
               );
             }
-            const pageResponse = await input.executeAction({
+            const actionExecution = await executeActionWithinDeadline({
               action: finishAction.action,
               input: { ...finishInput, [pagination.pageInputField]: page },
-              jobId,
               phase: 'finish_page',
               pollAttempt,
               page,
             });
+            if (actionExecution.kind === 'timed_out') return timedOut();
+            const pageResponse = actionExecution.response;
             currentResult = input.responseValue(pageResponse);
             const pageItems = readAsyncOperationPath(
               currentResult,