deepline 0.3.115 → 0.3.117

package/dist/bundling-sources/shared_libs/monitors/fleets/verdict.ts

@@ -8,23 +8,32 @@
  * the fact; this is the single reading of them.
  */
 
-export type FleetConvergenceCounts = {
-  pending: number;
-  submitting: number;
-  rate_limited: number;
-  settled: number;
-  blocked: number;
-  failed: number;
-};
+import type { ProviderMonitorCustomerState } from '@deepline/monitors/provider-monitor-state';
+
+export const FLEET_CONVERGENCE_STATES = [
+  'pending',
+  'submitting',
+  'rate_limited',
+  'settled',
+  'blocked',
+  'failed',
+] as const;
+
+export type FleetConvergenceState = (typeof FLEET_CONVERGENCE_STATES)[number];
+
+export type FleetConvergenceCounts = Record<FleetConvergenceState, number>;
 
 /** The lifecycle a fleet row records. */
-export type FleetLifecycleStatus =
-  | 'active'
-  | 'deactivating'
-  | 'deactivated'
-  | 'billing_suspending'
-  | 'billing_suspended'
-  | 'billing_reactivating';
+export const FLEET_LIFECYCLE_STATUSES = [
+  'active',
+  'deactivating',
+  'deactivated',
+  'billing_suspending',
+  'billing_suspended',
+  'billing_reactivating',
+] as const;
+
+export type FleetLifecycleStatus = (typeof FLEET_LIFECYCLE_STATUSES)[number];
 
 /** What a caller is told the fleet is doing. */
 export type FleetStatusVerdict =
@@ -37,6 +46,34 @@ export type FleetStatusVerdict =
   | 'billing_suspended'
   | 'billing_reactivating';
 
+/**
+ * The Fleet's stable customer state. Convergence and lifecycle transitions are
+ * reported by the durable Fleet operation, not by growing this union.
+ */
+export function fleetCustomerState(input: {
+  lifecycle: FleetLifecycleStatus;
+  verdict: FleetStatusVerdict;
+  convergedGeneration: number | null;
+}): ProviderMonitorCustomerState {
+  switch (input.lifecycle) {
+    case 'billing_suspending':
+    case 'billing_suspended':
+    case 'billing_reactivating':
+      return 'billing_suspended';
+    case 'deactivating':
+    case 'deactivated':
+      return 'disabled';
+    case 'active':
+      // A new Fleet is not active merely because its desired lifecycle says
+      // so. It becomes active after one generation has been provider-proven.
+      // A later convergence may keep serving that last proven generation.
+      return input.verdict !== 'degraded' &&
+        (input.verdict === 'converged' || input.convergedGeneration !== null)
+        ? 'active'
+        : 'disabled';
+  }
+}
+
 const ZERO_FLEET_COUNTS: FleetConvergenceCounts = {
   pending: 0,
   submitting: 0,

package/dist/bundling-sources/shared_libs/monitors/monitor-job-state.ts

@@ -0,0 +1,44 @@
+/**
+ * Shared vocabulary for the public Monitor Job projection.
+ *
+ * A job reports progress toward provider truth. None of these values are a
+ * monitor lifecycle state, and no cron/admin path may invent another job
+ * vocabulary. Convex validators repeat these literals at the storage ABI
+ * boundary; the contract tests keep that deliberate mirror aligned.
+ */
+export const MONITOR_JOB_STATES = [
+  'queued',
+  'running',
+  'cancelling',
+  'succeeded',
+  'failed',
+  'cancelled',
+] as const;
+
+export type MonitorJobState = (typeof MONITOR_JOB_STATES)[number];
+
+export const MONITOR_JOB_PHASES = [
+  'queued',
+  'provider_create',
+  'provider_readback',
+  'reconciliation',
+  'completed',
+] as const;
+
+export type MonitorJobPhase = (typeof MONITOR_JOB_PHASES)[number];
+
+export const MONITOR_JOB_RECONCILIATION_STATES = [
+  'pending',
+  'aligned',
+  'partial',
+  'mismatched',
+] as const;
+
+export type MonitorJobReconciliationState =
+  (typeof MONITOR_JOB_RECONCILIATION_STATES)[number];
+
+export const TERMINAL_MONITOR_JOB_STATES = [
+  'succeeded',
+  'failed',
+  'cancelled',
+] as const satisfies readonly MonitorJobState[];

package/dist/bundling-sources/shared_libs/monitors/package.json

@@ -53,6 +53,8 @@
     "./lifecycle-notifications.js": "./lifecycle-notifications.ts",
     "./monitor-fleet-limits": "./monitor-fleet-limits.ts",
     "./monitor-fleet-limits.js": "./monitor-fleet-limits.ts",
+    "./monitor-job-state": "./monitor-job-state.ts",
+    "./monitor-job-state.js": "./monitor-job-state.ts",
     "./org-monitor-limits": "./org-monitor-limits.ts",
     "./org-monitor-limits.js": "./org-monitor-limits.ts",
     "./provider-callback-verification": "./provider-callback-verification.ts",

package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts

@@ -21,6 +21,10 @@ export const PROVIDER_MONITOR_BATCH_OPERATION_KINDS = [
   // operation is the durable, observable hand-off to the fenced Fleet
   // executor; it is not a second lifecycle verb or a raw provider create.
   'converge',
+  // A named Batch owns a bounded mixed plan (sync/start/stop) under one
+  // durable operation. It is intentionally distinct from manifest deploy:
+  // omission/removal semantics belong to the Batch desired-set owner.
+  'named_batch',
 ] as const;
 
 export type ProviderMonitorBatchOperationKind =
@@ -66,12 +70,25 @@ export type ProviderMonitorBatchScope =
   | { kind: 'target'; target: ProviderMonitorBatchTarget }
   | { kind: 'fleet'; fleetId: string }
   | ProviderMonitorBatchManifestScope
+  | {
+      kind: 'named_batch';
+      batchId: string;
+      generation: number;
+      intent: 'sync' | 'start' | 'stop';
+    }
   | ProviderMonitorBatchGlobalAuditScope;
 
 export type ProviderMonitorBatchOperationRequest = {
   kind: ProviderMonitorBatchOperationKind;
   scope: ProviderMonitorBatchScope;
   dryRun: boolean;
+  /**
+   * Optional ownership mode for exact customer lifecycle requests. Omitted
+   * means the system/policy runner may operate the selected definitions. The
+   * field is deliberately additive so older operation rows and callers keep
+   * their original fingerprint and behavior.
+   */
+  control?: 'customer_exact' | 'system';
   /**
    * The application deployment that accepted this operation. A provider
    * callback is part of the create side effect, so a continuation must retain
@@ -93,6 +110,17 @@ export type ProviderMonitorBatchOperationState =
   | 'failed'
   | 'cancelled';
 
+/** Per-monitor evidence written by a durable batch operation. */
+export const PROVIDER_MONITOR_BATCH_RESULT_STATES = [
+  'done',
+  'retry',
+  'blocked',
+  'superseded',
+] as const;
+
+export type ProviderMonitorBatchResultState =
+  (typeof PROVIDER_MONITOR_BATCH_RESULT_STATES)[number];
+
 const IDEMPOTENCY_KEY = /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/;
 
 function canonicalJson(value: unknown): string {
@@ -176,14 +204,40 @@ export function validateProviderMonitorBatchOperationRequest(
   if (!isProviderMonitorBatchOperationKind(request.kind)) {
     throw new Error(`Unknown monitor batch operation '${request.kind}'.`);
   }
+  if (
+    request.control !== undefined &&
+    request.control !== 'customer_exact' &&
+    request.control !== 'system'
+  ) {
+    throw new Error(`Unknown monitor batch control '${request.control}'.`);
+  }
   const callbackBaseUrl = normalizeCallbackBaseUrl(request.callbackBaseUrl);
-  if (callbackBaseUrl !== undefined && request.kind !== 'deploy') {
-    throw new Error('callbackBaseUrl is only valid for a batch deploy.');
+  if (
+    callbackBaseUrl !== undefined &&
+    request.kind !== 'deploy' &&
+    request.kind !== 'named_batch'
+  ) {
+    throw new Error(
+      'callbackBaseUrl is only valid for a batch deploy or named Batch sync.',
+    );
   }
   if (!request.scope || typeof request.scope !== 'object') {
     throw new Error('A monitor batch operation requires exactly one scope.');
   }
   const scope = request.scope;
+  if (request.kind === 'named_batch' && scope.kind !== 'named_batch') {
+    throw new Error('A named_batch operation requires a named Batch scope.');
+  }
+  if (
+    callbackBaseUrl !== undefined &&
+    request.kind === 'named_batch' &&
+    scope.kind === 'named_batch' &&
+    scope.intent !== 'sync'
+  ) {
+    throw new Error(
+      'callbackBaseUrl is only valid for a named Batch sync operation.',
+    );
+  }
   if (scope.kind === 'keys') {
     const keys = [...new Set(scope.monitorKeys.map((key) => key.trim()))];
     if (
@@ -294,6 +348,23 @@ export function validateProviderMonitorBatchOperationRequest(
       ...(callbackBaseUrl === undefined ? {} : { callbackBaseUrl }),
     };
   }
+  if (scope.kind === 'named_batch') {
+    const batchId = scope.batchId.trim();
+    if (!batchId || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(batchId)) {
+      throw new Error('A named Batch scope requires a valid batch id.');
+    }
+    if (!Number.isSafeInteger(scope.generation) || scope.generation < 1) {
+      throw new Error('A named Batch scope requires a positive generation.');
+    }
+    if (request.kind !== 'named_batch') {
+      throw new Error('A named Batch scope is only valid for named_batch.');
+    }
+    return {
+      ...request,
+      scope: { ...scope, batchId },
+      ...(callbackBaseUrl === undefined ? {} : { callbackBaseUrl }),
+    };
+  }
   if (scope.kind === 'global_audit') {
     const runId = scope.runId.trim();
     const monitorKey = scope.monitorKey?.trim();
@@ -304,7 +375,9 @@ export function validateProviderMonitorBatchOperationRequest(
       throw new Error('A global-audit scope is only valid for repair.');
     }
     if (scope.monitorKey !== undefined && !monitorKey) {
-      throw new Error('A global-audit monitor key, when supplied, must be non-empty.');
+      throw new Error(
+        'A global-audit monitor key, when supplied, must be non-empty.',
+      );
     }
     return {
       ...request,

package/dist/bundling-sources/shared_libs/monitors/provider-monitor-state.ts

@@ -1,9 +1,22 @@
 /**
- * The customer-visible state of a monitor. Provider work and audit evidence
- * intentionally live on separate axes; neither may add a third value here.
+ * The complete customer-visible lifecycle of a monitor.
+ *
+ * Provider work, retries, and audit evidence intentionally live on separate
+ * axes. They must never become additional values in this union.
+ */
+export type ProviderMonitorCustomerState =
+  | 'active'
+  | 'billing_suspended'
+  | 'disabled';
+
+/**
+ * The old two-value desired-state type. Keep it for control-plane callers and
+ * the deprecated `status` response alias while clients migrate to `state`.
  */
 export type ProviderMonitorState = 'active' | 'inactive';
 
+export type ProviderMonitorStatusAlias = ProviderMonitorState;
+
 declare const providerMonitorResourceIdBrand: unique symbol;
 
 /**
@@ -64,7 +77,9 @@ export function providerMonitorOperationalState(input: {
   desired: ProviderMonitorState;
   providerResourceId: string | null | undefined;
 }): OperationalProviderMonitorState {
-  const providerResourceId = providerMonitorResourceId(input.providerResourceId);
+  const providerResourceId = providerMonitorResourceId(
+    input.providerResourceId,
+  );
   if (input.desired === 'active' && providerResourceId !== null) {
     return {
       state: 'active',
@@ -81,6 +96,62 @@ export function providerMonitorOperationalState(input: {
   };
 }
 
+/**
+ * Project untrusted persisted billing state onto the only billing lifecycle
+ * values that are meaningful to a customer. Cleanup progress is a job detail;
+ * an unresolved billing admission is unavailable, not a new public state.
+ */
+export function providerMonitorCustomerStateForBillingState(
+  billingState: unknown,
+): ProviderMonitorCustomerState {
+  switch (billingState) {
+    case 'active':
+      return 'active';
+    case 'billing_suspended':
+    case 'upstream_disable_pending':
+    case 'upstream_disable_failed':
+      return 'billing_suspended';
+    case 'disabled':
+    case 'billing_pending':
+    case 'preflight_denied':
+    default:
+      return 'disabled';
+  }
+}
+
+/**
+ * The single public monitor-state projection. A billing suspension owns the
+ * off state. Otherwise `active` is operational proof: a committed active
+ * lifecycle and a non-empty provider resource binding. Internal billing
+ * bookkeeping such as an unresolved usage charge must not make a usable
+ * provider resource look disabled.
+ */
+export function providerMonitorCustomerState(input: {
+  lifecycleStatus?: string | null;
+  billingState?: unknown;
+  providerResourceId?: string | null | undefined;
+}): ProviderMonitorCustomerState {
+  const billingState = providerMonitorCustomerStateForBillingState(
+    input.billingState,
+  );
+  if (billingState === 'billing_suspended') return billingState;
+
+  const providerResourceId = providerMonitorResourceId(
+    input.providerResourceId,
+  );
+  if (input.lifecycleStatus === 'active' && providerResourceId !== null) {
+    return 'active';
+  }
+  return 'disabled';
+}
+
+/** Deprecated compatibility projection for the old `status`/`active` keys. */
+export function providerMonitorStatusAlias(
+  state: ProviderMonitorCustomerState,
+): ProviderMonitorStatusAlias {
+  return state === 'active' ? 'active' : 'inactive';
+}
+
 /**
  * Runtime boundary for code that is about to publish or administer an active
  * resource. Its return type is the same proof exposed to callers: an active
@@ -102,9 +173,13 @@ export function requireActiveProviderMonitorBinding(input: {
 }
 
 /**
- * Translate the historical control-record lifecycle label at the sole
- * compatibility boundary. New code persists `desired` and returns this type;
- * this mapping exists only while the online backfill drains existing rows.
+ * Recover desired intent from a historical control-record lifecycle label at
+ * the sole compatibility boundary. This does not prove that a monitor is
+ * operationally active: customer projections must still require a committed
+ * provider resource through {@link providerMonitorCustomerState}.
+ *
+ * New code persists `desired`; this mapping exists only while the online
+ * backfill drains existing rows.
  */
 export function providerMonitorStateForLegacyStatus(
   status: string | undefined,

package/dist/bundling-sources/shared_libs/monitors/provider-monitor-status.ts

@@ -1,10 +1,26 @@
-export type ProviderMonitorStatus =
-  | 'replacement_pending'
-  | 'replacement_active_old_cleanup_pending'
-  | 'replacement_cleanup_failed'
-  | 'active'
-  | 'disabled'
-  | 'deleted'
-  | 'deleting'
-  | 'error'
-  | 'pending_create';
+/** Current values allowed in providerMonitorDefinitions.status. */
+export const PROVIDER_MONITOR_PERSISTED_STATUSES = [
+  'active',
+  'disabled',
+  'deleted',
+  'deleting',
+  'error',
+  'pending_create',
+] as const;
+
+/** Compatibility-only inputs normalized by current writers before storage. */
+export const PROVIDER_MONITOR_LEGACY_REPLACEMENT_STATUSES = [
+  'replacement_pending',
+  'replacement_active_old_cleanup_pending',
+  'replacement_cleanup_failed',
+] as const;
+
+export const PROVIDER_MONITOR_STATUSES = [
+  ...PROVIDER_MONITOR_LEGACY_REPLACEMENT_STATUSES,
+  ...PROVIDER_MONITOR_PERSISTED_STATUSES,
+] as const;
+
+export type ProviderMonitorStatus = (typeof PROVIDER_MONITOR_STATUSES)[number];
+
+export type PersistedProviderMonitorStatus =
+  (typeof PROVIDER_MONITOR_PERSISTED_STATUSES)[number];

package/dist/bundling-sources/shared_libs/observability/queue-health.ts

@@ -456,6 +456,26 @@ export type QueueHealth = {
   readonly worstAgeRatio?: number;
 };
 
+/**
+ * Small operator-facing state set for the dashboard and admin CLI.
+ *
+ * This is deliberately derived from the source-owned depth buckets. It is
+ * not a replacement for the queue's durable state machine and therefore does
+ * not create a new persisted state or a new retry policy.
+ */
+export type QueueOperatorState =
+  | 'draining'
+  | 'stalled'
+  | 'held'
+  | 'quarantined'
+  | 'clear'
+  | 'unknown';
+
+export type QueueOperatorStateSnapshot = {
+  readonly state: QueueOperatorState;
+  readonly reason: string;
+};
+
 export const QUEUE_HEALTH_TELEMETRY_EVENT = 'operations.queue.health' as const;
 export const QUEUE_HEALTH_TELEMETRY_SCHEMA_VERSION = 1 as const;
 const UNOBSERVED_SIGNAL_REPEAT_MS = 5 * 60_000;
@@ -475,6 +495,8 @@ export type QueueHealthSignal = {
   readonly adapter: string;
   readonly observedAt: number;
   readonly health: QueueHealth;
+  readonly operatorState: QueueOperatorState;
+  readonly operatorReason: string;
   readonly previousHealth?: QueueHealth;
   readonly warningDefinition: string;
   readonly failureDefinition: string;
@@ -503,6 +525,11 @@ export function deriveQueueHealthSignal(input: {
 }): QueueHealthSignal | null {
   const now = input.now ?? input.observation.observedAt;
   const health = deriveQueueHealth(input.definition, input.observation, now);
+  const operator = deriveQueueOperatorState(
+    input.definition,
+    input.observation,
+    now,
+  );
   const previousHealth = input.previousObservation
     ? deriveQueueHealth(input.definition, input.previousObservation, now)
     : undefined;
@@ -546,6 +573,8 @@ export function deriveQueueHealthSignal(input: {
     adapter: input.observation.adapter,
     observedAt: input.observation.observedAt,
     health,
+    operatorState: operator.state,
+    operatorReason: operator.reason,
     ...(previousHealth === undefined ? {} : { previousHealth }),
     warningDefinition: input.definition.warningDefinition,
     failureDefinition: input.definition.failureDefinition,
@@ -694,6 +723,105 @@ export function deriveQueueHealth(
   };
 }
 
+/**
+ * Collapse the detailed queue buckets into a small, actionable operator
+ * vocabulary. The precedence is intentional: quarantined and held work must
+ * not be hidden by a large ready count, while an old ready-only queue is
+ * called stalled only when no in-flight work is observed.
+ */
+export function deriveQueueOperatorState(
+  definition: QueueDefinition,
+  observation: QueueObservation | undefined,
+  now = Date.now(),
+): QueueOperatorStateSnapshot {
+  const health = deriveQueueHealth(definition, observation, now);
+  if (
+    observation === undefined ||
+    health.status === 'unobserved' ||
+    health.reason === 'invalid_observation'
+  ) {
+    return {
+      state: 'unknown',
+      reason: 'Queue observation is missing or stale.',
+    };
+  }
+
+  const ready = measuredValue(observation.depth.ready);
+  const delayed = measuredValue(observation.depth.delayed);
+  const inFlight = measuredValue(observation.depth.inFlight);
+  const blocked = measuredValue(observation.depth.blocked);
+  const deadLettered = measuredValue(observation.depth.deadLettered);
+  const oldestReadyMs = measuredValue(observation.oldestAgeMs.ready);
+
+  if (deadLettered !== undefined && deadLettered > 0) {
+    return {
+      state: 'quarantined',
+      reason: 'Dead-lettered work requires an owner recovery path.',
+    };
+  }
+  if (blocked !== undefined && blocked > 0) {
+    return {
+      state: 'held',
+      reason: 'Blocked work is intentionally held for owner review.',
+    };
+  }
+
+  if (
+    Object.values(observation.depth).some(
+      (measurement) =>
+        measurement.kind === 'lower_bound' && measurement.value === 0,
+    )
+  ) {
+    return {
+      state: 'unknown',
+      reason: 'A bounded read cannot prove that the queue is clear.',
+    };
+  }
+  if (
+    Object.values(observation.depth).some(
+      (measurement) => measurement.kind === 'unavailable',
+    )
+  ) {
+    return {
+      state: 'unknown',
+      reason: 'A depth bucket is unavailable; queue state cannot be proven.',
+    };
+  }
+
+  if (
+    ready !== undefined &&
+    ready > 0 &&
+    inFlight === 0 &&
+    oldestReadyMs !== undefined &&
+    oldestReadyMs >= definition.ageSlo.failMs
+  ) {
+    return {
+      state: 'stalled',
+      reason:
+        'Ready work is past its failure age SLO and no in-flight work is observed.',
+    };
+  }
+
+  if (
+    (ready !== undefined && ready > 0) ||
+    (delayed !== undefined && delayed > 0) ||
+    (inFlight !== undefined && inFlight > 0)
+  ) {
+    return {
+      state: 'draining',
+      reason:
+        health.reason === 'age_breached'
+          ? 'Work is active, but its oldest item is past the age SLO.'
+          : 'Work is eligible, scheduled, or currently in flight.',
+    };
+  }
+
+  return {
+    state: 'clear',
+    reason: 'No active, held, or quarantined work is observed.',
+  };
+}
+
 // Legacy v1 snapshot helpers remain exported for the webhook/cron collectors
 // during the v1 -> v2 observability rollout. They are intentionally isolated
 // from the scheduler-neutral v2 contract above.

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -975,6 +975,24 @@
     "failureAlert": "convex_action_wrapper",
     "domainReportRequired": true
   },
+  {
+    "id": "convex.monitor-provider-repair-recovery-tick",
+    "name": "Monitor provider repair recovery tick",
+    "scheduler": "convex_cron",
+    "cadence": "every 60s",
+    "timezone": "UTC",
+    "source": "packages/convex/crons.ts",
+    "locator": "monitor provider repair recovery tick",
+    "owner": "monitors",
+    "criticality": "critical",
+    "purpose": "Reclaim and wake durable provider monitor repair operations without delaying ordinary Fleet convergence.",
+    "successDefinition": "Due repair operations are re-issued or durably accounted; no due repairs is a valid no-op.",
+    "failureDefinition": "A repair lease or continuation is stranded, or its recovery state cannot persist.",
+    "startGraceMs": 120000,
+    "runTimeoutMs": 1800000,
+    "failureAlert": "convex_action_wrapper",
+    "domainReportRequired": true
+  },
   {
     "id": "convex.recover-play-webhook-receipts",
     "name": "Recover Play webhook receipts",

package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts

@@ -15,6 +15,7 @@ export const MANUALLY_RUNNABLE_CONVEX_CRON_JOB_IDS = [
   'convex.cleanup-expired-domain-health-cache',
   'convex.salesforce-oauth-refresh-maintenance',
   'convex.monitor-fleet-convergence-tick',
+  'convex.monitor-provider-repair-recovery-tick',
   'convex.monitor-deployment-job-tick',
   'convex.provider-monitor-global-audit-daily',
   'convex.recover-notification-delivery-work',

package/dist/bundling-sources/shared_libs/plays/share/static-pipeline-summary.ts

@@ -328,8 +328,8 @@ function previewDetailFor(substep: PlayStaticSubstep): string | null {
         ? `${substep.steps.length} fallbacks`
         : 'first match wins';
     case 'step_suite': {
-      const count = stepSuiteToolIds(substep).length;
-      return count > 0 ? `${count} providers, first match wins` : null;
+      const count = effectiveStepCount(substep);
+      return `${count} step${count === 1 ? '' : 's'}`;
     }
     case 'control_flow': {
       const count = substep.steps.reduce(