deepline 0.3.120 → 0.3.122

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.120',
+  version: '0.3.122',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/monitors/billing/credit-recovery.ts

@@ -0,0 +1,77 @@
+import { parseStoredMonitorBillingState } from './state';
+
+/**
+ * Only these event-time billing denials are healed by a restored Deepline
+ * balance. A monthly spending cap and an opaque billing failure deliberately
+ * remain explicit-recovery states.
+ */
+export function isCreditExhaustionSuspensionReason(
+  reason: string | undefined,
+): boolean {
+  return (
+    reason === 'insufficient_credits' ||
+    reason === 'insufficient_credits_after_provider_event'
+  );
+}
+
+/**
+ * A billing suspension is not itself proof that TamRadar accepted the delete.
+ * Keep this test beside the recovery selector so every automatic path rejects
+ * an unfinished cleanup, including a retired binding that still needs work.
+ */
+export function hasConfirmedProviderMonitorUpstreamCleanup(
+  providerBinding: unknown,
+): boolean {
+  if (!providerBinding || typeof providerBinding !== 'object') return false;
+  const binding = providerBinding as Record<string, unknown>;
+  const suspension = binding.billing_suspension;
+  if (!suspension || typeof suspension !== 'object') return false;
+  const upstreamDisabledAt = (suspension as Record<string, unknown>)
+    .upstream_disabled_at;
+  if (typeof upstreamDisabledAt !== 'string' || !upstreamDisabledAt.length) {
+    return false;
+  }
+  const retiredBindings = binding.retired_bindings;
+  if (!Array.isArray(retiredBindings) || retiredBindings.length === 0) {
+    return true;
+  }
+  const cleanup = binding.cleanup;
+  const cleanupStatus =
+    cleanup && typeof cleanup === 'object'
+      ? (cleanup as Record<string, unknown>).status
+      : null;
+  return cleanupStatus === 'old_cleanup_deleted';
+}
+
+/**
+ * Narrow selector for direct event-time suspensions that credit restoration
+ * may re-provision. It intentionally does not include policy-only pauses,
+ * manual intent, spending-cap denials, or cleanup still in flight.
+ */
+export function isCreditRestorableDirectMonitorSuspension(input: {
+  fleetId?: string;
+  status: string;
+  desired?: 'active' | 'inactive';
+  deactivationReason?: string;
+  billingState: unknown;
+  providerBinding: unknown;
+}): boolean {
+  if (input.fleetId !== undefined || input.status !== 'disabled') return false;
+  const billingState = parseStoredMonitorBillingState(input.billingState);
+  if (
+    billingState.state !== 'billing_suspended' ||
+    !billingState.reactivation_required ||
+    !isCreditExhaustionSuspensionReason(billingState.suspension_reason) ||
+    !hasConfirmedProviderMonitorUpstreamCleanup(input.providerBinding)
+  ) {
+    return false;
+  }
+  // An inactive desired state with no policy provenance is the customer's
+  // current choice. Historical policy rows can be identified exactly by the
+  // closed control-plane reason; an already-active desired state is a prior
+  // restoration intent that still requires the normal reactivation lifecycle.
+  return (
+    input.desired !== 'inactive' ||
+    input.deactivationReason === 'insufficient_balance'
+  );
+}

package/dist/bundling-sources/shared_libs/monitors/package.json

@@ -11,6 +11,8 @@
     "./analytics-public.js": "./analytics-public.ts",
     "./billing/charges": "./billing/charges.ts",
     "./billing/charges.js": "./billing/charges.ts",
+    "./billing/credit-recovery": "./billing/credit-recovery.ts",
+    "./billing/credit-recovery.js": "./billing/credit-recovery.ts",
     "./billing/policy-builders": "./billing/policy-builders.ts",
     "./billing/policy-builders.js": "./billing/policy-builders.ts",
     "./billing/pricing": "./billing/pricing.ts",

package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts

@@ -65,6 +65,11 @@ export type ProviderMonitorBatchGlobalAuditScope = {
   monitorKey?: string;
 };
 
+/** Internal-only scope for credit-restoration reactivation. */
+export type ProviderMonitorBatchCreditRecoveryScope = {
+  kind: 'credit_recovery';
+};
+
 export type ProviderMonitorBatchScope =
   | { kind: 'keys'; monitorKeys: readonly string[] }
   | { kind: 'target'; target: ProviderMonitorBatchTarget }
@@ -76,7 +81,8 @@ export type ProviderMonitorBatchScope =
       generation: number;
       intent: 'sync' | 'start' | 'stop';
     }
-  | ProviderMonitorBatchGlobalAuditScope;
+  | ProviderMonitorBatchGlobalAuditScope
+  | ProviderMonitorBatchCreditRecoveryScope;
 
 export type ProviderMonitorBatchOperationRequest = {
   kind: ProviderMonitorBatchOperationKind;
@@ -389,6 +395,18 @@ export function validateProviderMonitorBatchOperationRequest(
       ...(callbackBaseUrl === undefined ? {} : { callbackBaseUrl }),
     };
   }
+  if (scope.kind === 'credit_recovery') {
+    if (request.kind !== 'reactivate') {
+      throw new Error(
+        'A credit-recovery scope is only valid for reactivation.',
+      );
+    }
+    return {
+      ...request,
+      scope: { kind: 'credit_recovery' },
+      ...(callbackBaseUrl === undefined ? {} : { callbackBaseUrl }),
+    };
+  }
   throw new Error(
     'A monitor batch operation requires exactly one supported scope.',
   );

package/dist/bundling-sources/shared_libs/observability/dlq.ts

@@ -177,6 +177,345 @@ export type DlqApplyResult = {
   readonly verificationRequired: boolean;
 };
 
+/**
+ * Queue-item control is deliberately a separate port from an inventory. A
+ * count and a bounded index description are not permission to manufacture a
+ * row scan: owners register this reader only when they can return exact item
+ * evidence without widening their data-access boundary.
+ */
+export const QUEUE_ITEM_CONTROL_CONTRACT_VERSION = 1 as const;
+
+/** The common state vocabulary exposed by every queue connector. */
+export const QUEUE_ITEM_STATES = [
+  'ready',
+  'delayed',
+  'running',
+  'held',
+  'quarantined',
+  'archived',
+  'completed',
+  'cancelled',
+  'unknown',
+] as const;
+
+export type QueueItemState = (typeof QUEUE_ITEM_STATES)[number];
+
+export type QueueItemAction =
+  | 'archive'
+  | 'quarantine'
+  | 'retry'
+  | 'hold'
+  | 'resume'
+  | 'cancel'
+  | 'update';
+
+export const QUEUE_ITEM_ACTIONS = [
+  'archive',
+  'quarantine',
+  'retry',
+  'hold',
+  'resume',
+  'cancel',
+  'update',
+] as const satisfies readonly QueueItemAction[];
+
+export type QueueItemActionAvailability = {
+  readonly action: QueueItemAction;
+  readonly allowed: boolean;
+  readonly reason?: string;
+};
+
+export type QueueItemFieldType =
+  | 'string'
+  | 'number'
+  | 'boolean'
+  | 'datetime'
+  | 'json';
+
+export type QueueItemFieldDefinition = {
+  readonly name: string;
+  readonly type: QueueItemFieldType;
+  readonly required: boolean;
+  readonly mutable: boolean;
+  readonly description?: string;
+};
+
+export type QueueItemLegacyAction = Exclude<DlqPathAction, 'inspect'>;
+export type QueueItemActionInput = QueueItemAction | QueueItemLegacyAction;
+
+export type QueueItemLastError = {
+  readonly code: string;
+  readonly message: string;
+};
+
+/** Names used in the API documentation and in connector implementations. */
+export type ActionAvailability = QueueItemActionAvailability;
+export type FieldDefinition = QueueItemFieldDefinition;
+
+/** A normalized item returned by an owner connector. `nativeState` is always
+ * retained so an operator can distinguish a connector mapping from the
+ * backend's actual state. */
+export type QueueItem = {
+  readonly queueId: string;
+  readonly id: string;
+  /** Opaque source revision used for compare-and-set mutation. */
+  readonly revision: string;
+  readonly observedAt: string;
+  readonly state: QueueItemState;
+  readonly nativeState: string;
+  readonly lastError?: QueueItemLastError;
+  readonly actions: readonly ActionAvailability[];
+  readonly editableFields: readonly FieldDefinition[];
+  readonly createdAt?: string;
+  readonly updatedAt?: string;
+  readonly summary?: string;
+};
+
+/** Keep old DLQ verbs working at the boundary while exposing a smaller,
+ * operator-readable control vocabulary. */
+export function normalizeQueueItemAction(
+  action: QueueItemActionInput,
+): QueueItemAction {
+  const aliases: Readonly<Record<QueueItemLegacyAction, QueueItemAction>> = {
+    redrive: 'retry',
+    discard: 'archive',
+    mark_done: 'archive',
+    handoff: 'resume',
+    recover: 'retry',
+    reconcile: 'update',
+    replay: 'retry',
+    retry: 'retry',
+    quarantine: 'quarantine',
+    release: 'cancel',
+    settle: 'resume',
+  };
+  return action in aliases
+    ? aliases[action as QueueItemLegacyAction]
+    : (action as QueueItemAction);
+}
+
+/**
+ * Legacy bounded evidence kept for callers that only consume the original
+ * `id/state/revision` projection. New connectors must return `QueueItem`.
+ */
+export type QueueItemEvidence = {
+  readonly id: string;
+  readonly state: string;
+  readonly revision: string;
+  /** Additive fields understood by newer readers; old producers may omit them. */
+  readonly nativeState?: string;
+  readonly lastError?: QueueItemLastError;
+  readonly actions?: readonly QueueItemActionAvailability[];
+  readonly editableFields?: readonly QueueItemFieldDefinition[];
+  readonly createdAt?: number;
+  readonly updatedAt?: number;
+  readonly summary?: string;
+};
+
+export type QueueItemUnavailable = {
+  readonly code: 'item_reader_unavailable';
+  readonly reason: string;
+};
+
+export type QueueItemList = {
+  readonly contractVersion: typeof QUEUE_ITEM_CONTROL_CONTRACT_VERSION;
+  readonly queueId: string;
+  readonly environment: string;
+  readonly observedAt: number;
+  readonly items: readonly QueueItem[];
+  readonly nextCursor: string | null;
+  readonly limit: number;
+  readonly cursor: string | null;
+  readonly unavailable?: QueueItemUnavailable;
+};
+
+export type QueueItemLookup =
+  | { readonly kind: 'found'; readonly item: QueueItem }
+  | { readonly kind: 'not_found' }
+  | {
+      readonly kind: 'unavailable';
+      readonly unavailable: QueueItemUnavailable;
+    };
+
+export type QueueItemActionRequest = {
+  readonly action: QueueItemAction;
+  readonly sourceAction?: QueueItemActionInput;
+  readonly idempotencyKey: string;
+  readonly expectedRevision: string;
+  readonly reason: string;
+  /** Connector-declared mutable fields for the `update` action only. */
+  readonly fields?: Readonly<Record<string, unknown>>;
+  /** Authenticated platform-admin identity forwarded to the owner connector. */
+  readonly actorId?: string;
+};
+
+export type QueueItemActionOperation = {
+  readonly contractVersion: typeof QUEUE_ITEM_CONTROL_CONTRACT_VERSION;
+  readonly operationId: string;
+  readonly queueId: string;
+  readonly itemId: string;
+  readonly environment: string;
+  readonly action: QueueItemAction;
+  readonly idempotencyKey: string;
+  readonly expectedRevision: string;
+  /** `pending` means the owner accepted the request but has not yet verified
+   * the native transition. `confirmed` is the only successful terminal
+   * status. The rejected/stale statuses preserve the existing safe boundary. */
+  readonly status: 'pending' | 'confirmed' | 'rejected' | 'stale' | 'failed';
+  readonly requestedAt?: string;
+  readonly confirmedAt?: string;
+  readonly replacementItemId?: string;
+  readonly item?: QueueItem;
+  readonly error?: QueueItemLastError;
+  readonly audit: {
+    readonly recorded: boolean;
+    readonly reason: string;
+  };
+  /** Existing owner command, when the catalog has one. This is guidance only;
+   * callers must still invoke its own authenticated, audited surface. */
+  readonly ownerCommand?: string;
+};
+
+export type QueueItemActionStatus =
+  | 'pending'
+  | 'confirmed'
+  | 'accepted'
+  | 'rejected'
+  | 'stale'
+  | 'failed'
+  | 'not_found'
+  | 'unavailable';
+
+/**
+ * Connector result for a requested transition. `confirmed` means the owner
+ * verified the resulting native state; `pending` means the owner durably
+ * accepted it but has not yet confirmed that state. `accepted` remains as a
+ * compatibility spelling for older connectors and must not be treated as
+ * confirmed by the shared control service.
+ */
+export type QueueItemActionResult = {
+  readonly contractVersion: typeof QUEUE_ITEM_CONTROL_CONTRACT_VERSION;
+  readonly operationId: string;
+  readonly queueId: string;
+  readonly itemId: string;
+  readonly environment: string;
+  readonly action: QueueItemAction;
+  readonly idempotencyKey: string;
+  readonly expectedRevision: string;
+  readonly status: QueueItemActionStatus;
+  readonly item?: QueueItem;
+  readonly replacementItemId?: string;
+  readonly reason?: string;
+  readonly audit: {
+    readonly recorded: boolean;
+    readonly auditId?: string;
+  };
+};
+
+export type QueueItemConnectorContext = {
+  readonly queueId: string;
+  readonly environment: string;
+  readonly now: number;
+  readonly limit: number;
+  readonly cursor?: string;
+  readonly actorId?: string;
+  readonly operationId?: string;
+};
+
+/**
+ * Owner implementation port. The connector owns native reads, state mapping,
+ * lease/CAS checks, and outcome verification. The shared control service only
+ * authenticates, validates, journals idempotency, and returns this result.
+ */
+export type QueueItemConnector = {
+  readonly id: string;
+  readonly queueId: string;
+  readonly source: string;
+  readonly mapNativeState: (nativeState: string) => QueueItemState;
+  readonly list: (context: QueueItemConnectorContext) => Promise<QueueItemList>;
+  readonly get: (
+    context: QueueItemConnectorContext,
+    itemId: string,
+  ) => Promise<QueueItemLookup>;
+  readonly requestAction: (
+    context: QueueItemConnectorContext,
+    itemId: string,
+    request: QueueItemActionRequest,
+  ) => Promise<QueueItemActionOperation>;
+};
+
+/** Conservative shared fallback for an unrecognised native state. */
+export function normalizeQueueItemState(nativeState: string): QueueItemState {
+  const state = nativeState.trim().toLowerCase();
+  if ((QUEUE_ITEM_STATES as readonly string[]).includes(state)) {
+    return state as QueueItemState;
+  }
+  if (
+    state === 'in_flight' ||
+    state === 'in-flight' ||
+    state === 'processing'
+  ) {
+    return 'running';
+  }
+  if (
+    state === 'dead_lettered' ||
+    state === 'dead-lettered' ||
+    state === 'dlq'
+  ) {
+    return 'quarantined';
+  }
+  if (state === 'done' || state === 'succeeded' || state === 'success') {
+    return 'completed';
+  }
+  if (state === 'canceled' || state === 'aborted') return 'cancelled';
+  return 'unknown';
+}
+
+export function unavailableQueueItemList(input: {
+  queueId: string;
+  environment: string;
+  observedAt: number;
+  reason: string;
+  limit?: number;
+  cursor?: string | null;
+}): QueueItemList {
+  return {
+    contractVersion: QUEUE_ITEM_CONTROL_CONTRACT_VERSION,
+    queueId: input.queueId,
+    environment: input.environment,
+    observedAt: input.observedAt,
+    items: [],
+    nextCursor: null,
+    limit: input.limit ?? 50,
+    cursor: input.cursor ?? null,
+    unavailable: { code: 'item_reader_unavailable', reason: input.reason },
+  };
+}
+
+/** A deterministic request identity is safe to show before an owner has a
+ * durable operation store. It is explicitly not represented as an audit. */
+export function queueItemOperationId(input: {
+  queueId: string;
+  itemId: string;
+  action: QueueItemAction;
+  idempotencyKey: string;
+}): string {
+  const value = [
+    input.queueId,
+    input.itemId,
+    input.action,
+    input.idempotencyKey,
+  ]
+    .map((part) => part.trim())
+    .join(':');
+  let hash = 2166136261;
+  for (let index = 0; index < value.length; index += 1) {
+    hash ^= value.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return `queue-item-${(hash >>> 0).toString(36)}`;
+}
+
 /** Every backend—including Absurd—must implement this port at its boundary. */
 export type DlqAdapter = {
   readonly id: string;

package/dist/bundling-sources/shared_libs/observability/queue-item-connectors.ts

@@ -0,0 +1,314 @@
+import {
+  normalizeQueueItemState,
+  queueItemOperationId,
+  unavailableQueueItemList,
+  type QueueItemActionOperation,
+  type QueueItemActionRequest,
+  type QueueItemConnector,
+  type QueueItemConnectorContext,
+  type QueueItemList,
+  type QueueItemLookup,
+  type QueueItemState,
+} from './dlq';
+import { OPERATIONS_QUEUE_CATALOG } from './queue-health';
+
+/** Native source families supported by the shared control-plane port. */
+export type QueueConnectorBackend =
+  | 'convex'
+  | 'workpool'
+  | 'postgres-outbox'
+  | 'absurd'
+  | 'provider';
+
+export type QueueItemConnectorRegistration = {
+  readonly backend: QueueConnectorBackend;
+  readonly connector: QueueItemConnector;
+};
+
+export type QueueItemConnectorRegistryOptions = {
+  /** Registration is intentionally injectable for tests and owner wiring. */
+  readonly registrations?: readonly QueueItemConnectorRegistration[];
+};
+
+export type QueueItemConnectorRegisterOptions = {
+  /** Only an unavailable placeholder may be replaced at startup. */
+  readonly replaceUnavailable?: boolean;
+};
+
+/**
+ * The registry is the routing boundary: the control service looks up a queue
+ * by its stable public id and never switches on a native table or provider.
+ * Duplicate registrations are rejected so a new owner cannot silently shadow
+ * an existing mutation implementation.
+ */
+export class QueueItemConnectorRegistry {
+  private readonly registrations = new Map<
+    string,
+    QueueItemConnectorRegistration
+  >();
+
+  constructor(options: QueueItemConnectorRegistryOptions = {}) {
+    for (const registration of options.registrations ?? []) {
+      this.register(registration);
+    }
+  }
+
+  register(
+    registration: QueueItemConnectorRegistration,
+    options: QueueItemConnectorRegisterOptions = {},
+  ): this {
+    const { backend, connector } = registration;
+    if (!backend) throw new Error('Queue connector backend is required.');
+    if (!connector.queueId.trim())
+      throw new Error('Queue connector queueId is required.');
+    if (!connector.id.trim())
+      throw new Error('Queue connector id is required.');
+    const existing = this.registrations.get(connector.queueId);
+    if (existing && !options.replaceUnavailable) {
+      throw new Error(
+        `A queue connector is already registered for ${connector.queueId}.`,
+      );
+    }
+    if (
+      existing &&
+      options.replaceUnavailable &&
+      !existing.connector.id.endsWith('-unavailable')
+    ) {
+      throw new Error(
+        `Queue connector ${connector.queueId} is already owned by ${existing.connector.id}; replacement is not allowed.`,
+      );
+    }
+    this.registrations.set(connector.queueId, registration);
+    return this;
+  }
+
+  get(queueId: string): QueueItemConnectorRegistration | undefined {
+    return this.registrations.get(queueId);
+  }
+
+  has(queueId: string): boolean {
+    return this.registrations.has(queueId);
+  }
+
+  list(): readonly QueueItemConnectorRegistration[] {
+    return [...this.registrations.values()];
+  }
+}
+
+type UnavailableConnectorOptions = {
+  readonly queueId: string;
+  readonly backend: QueueConnectorBackend;
+  readonly source: string;
+  readonly id?: string;
+  readonly reason?: string;
+};
+
+const unavailableReason = (options: UnavailableConnectorOptions): string =>
+  options.reason ??
+  `The ${options.backend} owner has not registered a bounded item reader and fenced mutation adapter for ${options.queueId}.`;
+
+/**
+ * Safe connector shell used until an owner supplies source-owned reads and
+ * transactions. It is deliberately observable and routable, but cannot
+ * claim an item exists or that a transition was applied.
+ */
+export function createUnavailableQueueItemConnector(
+  options: UnavailableConnectorOptions,
+): QueueItemConnectorRegistration {
+  const reason = unavailableReason(options);
+  const connector: QueueItemConnector = {
+    id: options.id ?? `${options.backend}-${options.queueId}-unavailable`,
+    queueId: options.queueId,
+    source: options.source,
+    mapNativeState: (nativeState: string): QueueItemState =>
+      normalizeQueueItemState(nativeState),
+    async list(context: QueueItemConnectorContext): Promise<QueueItemList> {
+      return unavailableQueueItemList({
+        queueId: options.queueId,
+        environment: context.environment,
+        observedAt: context.now,
+        reason,
+        limit: context.limit,
+        cursor: context.cursor,
+      });
+    },
+    async get(
+      _context: QueueItemConnectorContext,
+      _itemId: string,
+    ): Promise<QueueItemLookup> {
+      return {
+        kind: 'unavailable',
+        unavailable: { code: 'item_reader_unavailable', reason },
+      };
+    },
+    async requestAction(
+      context: QueueItemConnectorContext,
+      itemId: string,
+      request: QueueItemActionRequest,
+    ): Promise<QueueItemActionOperation> {
+      const operationId =
+        context.operationId ??
+        queueItemOperationId({
+          queueId: options.queueId,
+          itemId,
+          action: request.action,
+          idempotencyKey: request.idempotencyKey,
+        });
+      return {
+        contractVersion: 1,
+        operationId,
+        queueId: options.queueId,
+        itemId,
+        environment: context.environment,
+        action: request.action,
+        idempotencyKey: request.idempotencyKey,
+        expectedRevision: request.expectedRevision,
+        status: 'rejected',
+        audit: { recorded: false, reason },
+      };
+    },
+  };
+  return { backend: options.backend, connector };
+}
+
+type BackendConnectorOptions = Omit<
+  UnavailableConnectorOptions,
+  'backend' | 'source'
+> & {
+  readonly source?: string;
+  readonly reason?: string;
+};
+
+export function createConvexQueueItemConnector(
+  options: BackendConnectorOptions,
+): QueueItemConnectorRegistration {
+  return createUnavailableQueueItemConnector({
+    ...options,
+    backend: 'convex',
+    source: options.source ?? 'convex',
+  });
+}
+
+export function createWorkpoolQueueItemConnector(
+  options: BackendConnectorOptions,
+): QueueItemConnectorRegistration {
+  return createUnavailableQueueItemConnector({
+    ...options,
+    backend: 'workpool',
+    source: options.source ?? 'convex-workpool',
+  });
+}
+
+export function createPostgresOutboxQueueItemConnector(
+  options: BackendConnectorOptions,
+): QueueItemConnectorRegistration {
+  return createUnavailableQueueItemConnector({
+    ...options,
+    backend: 'postgres-outbox',
+    source: options.source ?? 'postgres-outbox',
+  });
+}
+
+export function createAbsurdQueueItemConnector(
+  options: BackendConnectorOptions,
+): QueueItemConnectorRegistration {
+  return createUnavailableQueueItemConnector({
+    ...options,
+    backend: 'absurd',
+    source: options.source ?? 'absurd',
+  });
+}
+
+export function createProviderQueueItemConnector(
+  options: BackendConnectorOptions,
+): QueueItemConnectorRegistration {
+  return createUnavailableQueueItemConnector({
+    ...options,
+    backend: 'provider',
+    source: options.source ?? 'provider',
+  });
+}
+
+/** Explicit owner mapping. Unknown queues must fail registration instead of
+ * silently falling back to a connector with potentially wrong semantics. */
+export const QUEUE_CONNECTOR_BACKENDS: Readonly<
+  Record<string, QueueConnectorBackend>
+> = {
+  'play-runtime-tasks': 'absurd',
+  'run-ledger-projector-outbox': 'postgres-outbox',
+  'compute-billing-jobs': 'postgres-outbox',
+  'play-webhook-receipts': 'convex',
+  'workflow-cron-bindings': 'convex',
+  'play-cron-bindings': 'convex',
+  'ingestion-jobs': 'provider',
+  'monitor-fleet-convergence': 'convex',
+  'monitor-deployment-jobs': 'workpool',
+  'monitor-batch-operations': 'workpool',
+  'provider-monitor-reconciliation': 'provider',
+  'signal-radar-inbound-dlq': 'provider',
+  'charge-reconciliation-backlog': 'postgres-outbox',
+  'workspace-fast-settlement': 'convex',
+  'metronome-usage-delivery': 'postgres-outbox',
+  'billing-operations': 'postgres-outbox',
+  'stripe-financial-inbox': 'provider',
+  'async-usage-recovery': 'postgres-outbox',
+  'notification-deliveries': 'workpool',
+  'usage-rollup-fold': 'postgres-outbox',
+  'play-deployment-jobs': 'workpool',
+  'github-runner-wait': 'provider',
+  'sandbox-cleanup-jobs': 'postgres-outbox',
+};
+
+function defaultRegistration(
+  queueId: string,
+  backend: QueueConnectorBackend,
+): QueueItemConnectorRegistration {
+  const options = {
+    queueId,
+    source: `${backend}:unavailable`,
+  };
+  if (backend === 'convex') return createConvexQueueItemConnector(options);
+  if (backend === 'workpool') return createWorkpoolQueueItemConnector(options);
+  if (backend === 'postgres-outbox')
+    return createPostgresOutboxQueueItemConnector(options);
+  if (backend === 'absurd') return createAbsurdQueueItemConnector(options);
+  return createProviderQueueItemConnector(options);
+}
+
+export function createDefaultQueueItemConnectorRegistry(): QueueItemConnectorRegistry {
+  const catalogIds = new Set(OPERATIONS_QUEUE_CATALOG.map((entry) => entry.id));
+  const mappedIds = Object.keys(QUEUE_CONNECTOR_BACKENDS);
+  const missing = [...catalogIds].filter(
+    (queueId) => QUEUE_CONNECTOR_BACKENDS[queueId] === undefined,
+  );
+  const extra = mappedIds.filter((queueId) => !catalogIds.has(queueId));
+  if (missing.length > 0 || extra.length > 0) {
+    throw new Error(
+      `Queue connector mapping drift: missing=${missing.join(',') || 'none'} extra=${extra.join(',') || 'none'}`,
+    );
+  }
+  return new QueueItemConnectorRegistry({
+    registrations: OPERATIONS_QUEUE_CATALOG.map((entry) =>
+      defaultRegistration(entry.id, QUEUE_CONNECTOR_BACKENDS[entry.id]!),
+    ),
+  });
+}
+
+/** Process-local default used by the admin control service. Owner modules can
+ * replace an unavailable registration during startup with
+ * `register(ownerRegistration, { replaceUnavailable: true })`; there is no
+ * fallback that mutates source data. */
+export const defaultQueueItemConnectorRegistry =
+  createDefaultQueueItemConnectorRegistry();
+
+/**
+ * Owner startup hook. A source module should register its connector before
+ * serving admin requests, replacing only the matching unavailable placeholder.
+ */
+export function registerQueueItemConnector(
+  registration: QueueItemConnectorRegistration,
+): void {
+  defaultQueueItemConnectorRegistry.register(registration, {
+    replaceUnavailable: true,
+  });
+}

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -655,7 +655,7 @@
     "id": "convex.provider-monitor-credit-recovery",
     "name": "Provider monitor credit-balance recovery",
     "scheduler": "convex_scheduled_action",
-    "cadence": "runAfter(0) after a non-admitting balance becomes positive",
+    "cadence": "runAfter(0) after a non-admitting balance becomes positive (legacy queued delivery)",
     "timezone": "UTC",
     "source": "packages/convex/scheduledJobsNode.ts",
     "locator": "recoverProviderMonitorsAfterCreditBalanceChange",
@@ -669,6 +669,24 @@
     "failureAlert": "convex_action_wrapper",
     "domainReportRequired": true
   },
+  {
+    "id": "convex.credit-blocked-automation-recovery",
+    "name": "Credit-blocked automation recovery",
+    "scheduler": "convex_scheduled_action",
+    "cadence": "runAfter(0) after any positive balance movement; Monitor recovery only at the non-admitting-to-positive crossing",
+    "timezone": "UTC",
+    "source": "packages/convex/scheduledJobsNode.ts",
+    "locator": "recoverCreditBlockedAutomationsAfterCreditBalanceChange",
+    "owner": "billing",
+    "criticality": "critical",
+    "purpose": "Recover scheduled Plays and provider Monitors blocked by insufficient credits after a positive balance movement.",
+    "successDefinition": "Eligible credit-blocked automations are resumed or a valid no-op is recorded; ineligible resources remain unchanged.",
+    "failureDefinition": "Recovery intent cannot be durably recorded or an eligible automation cannot be rescheduled or reactivated.",
+    "startGraceMs": 120000,
+    "runTimeoutMs": 1800000,
+    "failureAlert": "convex_action_wrapper",
+    "domainReportRequired": true
+  },
   {
     "id": "inngest.workflow-run-watchdog",
     "name": "Workflow run watchdog",

package/dist/bundling-sources/shared_libs/observability/telemetry.ts

@@ -31,6 +31,9 @@ export type TelemetryContext = {
   workflowOrigin?: string | null;
   playName?: string | null;
   machineId?: string | null;
+  gatewayRole?: string | null;
+  /** Stable identity for one in-process gateway handler lifecycle. */
+  gatewayLifecycleId?: string | null;
 };
 
 export type TelemetryLevel = 'debug' | 'info' | 'warn' | 'error';
@@ -182,6 +185,8 @@ const RESERVED_FIELDS = new Set([
   'workflowOrigin',
   'playName',
   'machineId',
+  'gatewayRole',
+  'gatewayLifecycleId',
 ]);
 
 function errorTriage(fields: TelemetryFields | undefined): TelemetryTriage {

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -110,6 +110,7 @@ import {
   type ToolResponseView,
 } from '../plays/tool-response-contract';
 import {
+  AllAdmittedRowsFailedError,
   isProviderExhaustedFailure,
   ProviderExhaustedError,
 } from './run-failure';
@@ -1529,10 +1530,13 @@ type FieldMapRunResult = {
   failedRows: PersistableMapRow[];
   completedRowCount: number;
   failedRowCount: number;
+  failureSamples: Array<{ rowLabel: string; error: string }>;
   previewRows: Record<string, unknown>[];
   retainedRowsComplete: boolean;
 };
 
+const MAP_FAILURE_SAMPLE_LIMIT = 3;
+
 type RuntimeMapRowPersistence = {
   persistRows: (rows: PersistableMapRow[]) => {
     admitted: Promise<void>;
@@ -6804,6 +6808,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       let failedCount = 0;
       let duplicateReuseCount = 0;
       let executedCount = 0;
+      const failureSamples: Array<{ rowLabel: string; error: string }> = [];
+      const failureSampleErrors = new Set<string>();
       const previewRows: Record<string, unknown>[] = [];
       // A bounded dataset may be materialized by authored code immediately
       // after run(). Keep the rows produced by this exact awaited execution so
@@ -7179,6 +7185,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         executedCount += rowsToExecute.length;
         successfulCount += mapResult.completedRowCount;
         failedCount += mapResult.failedRowCount;
+        for (const sample of mapResult.failureSamples) {
+          if (
+            failureSamples.length >= MAP_FAILURE_SAMPLE_LIMIT ||
+            failureSampleErrors.has(sample.error)
+          ) {
+            continue;
+          }
+          failureSampleErrors.add(sample.error);
+          failureSamples.push(sample);
+        }
         processedCount += rowsToExecute.length;
         page.rows.length = 0;
       }
@@ -7216,10 +7232,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           totalRows: totalInputCount,
           at: Date.now(),
         });
-        throw new Error(
-          `ctx.dataset("${normalizedMapNamespace}") failed for every admitted row (${failedCount}/${totalInputCount}). ` +
-            'Failed rows were persisted and will be retried on the next run.',
-        );
+        throw new AllAdmittedRowsFailedError({
+          mapName: normalizedMapNamespace,
+          failedCount,
+          totalInputCount,
+          samples: failureSamples,
+        });
       }
 
       this.setMapFrame({
@@ -8105,6 +8123,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     const failFastOnRowError = runtimeOptions?.onRowError === 'fail';
     const completedRowsToPersist: PersistableMapRow[] = [];
     const failedRowsToPersist: PersistableMapRow[] = [];
+    const failureSamples: Array<{ rowLabel: string; error: string }> = [];
+    const failureSampleErrors = new Set<string>();
     const previewRows: Record<string, unknown>[] = [];
     let completedRowCount = 0;
     let failedRowCount = 0;
@@ -8368,6 +8388,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         failedRows: [],
         completedRowCount: results.length,
         failedRowCount: 0,
+        failureSamples: [],
         previewRows: results.slice(0, 5),
         retainedRowsComplete: true,
       };
@@ -8573,6 +8594,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               error: formattedError,
             });
             failedRowCount += 1;
+            if (
+              failureSamples.length < MAP_FAILURE_SAMPLE_LIMIT &&
+              !failureSampleErrors.has(formattedError)
+            ) {
+              failureSampleErrors.add(formattedError);
+              failureSamples.push({
+                rowLabel: failedMapRowLogLabel(failedRow),
+                error: formattedError,
+              });
+            }
             retainPersistedRow(failedRow, failedRowsToPersist);
             this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
               rowId: idx,
@@ -8985,6 +9016,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       failedRows: failedRowsToPersist,
       completedRowCount,
       failedRowCount,
+      failureSamples,
       previewRows,
       retainedRowsComplete,
     };

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -58,6 +58,93 @@ export const MODAL_PAYLOAD_UPLOAD_TIMEOUT_MESSAGE =
 
 export const WORKSPACE_STORAGE_NOT_READY_CODE = 'WORKSPACE_STORAGE_NOT_READY';
 
+export const ALL_ADMITTED_ROWS_FAILED_CODE = 'ALL_ADMITTED_ROWS_FAILED';
+
+export type AllAdmittedRowsFailedSample = {
+  rowLabel: string;
+  error: string;
+};
+
+const ALL_ADMITTED_ROWS_FAILED_SAMPLE_LIMIT = 3;
+const ALL_ADMITTED_ROWS_FAILED_SAMPLE_TEXT_LIMIT = 500;
+
+function boundedAllAdmittedRowsFailedSample(value: string): string {
+  const normalized = value.trim();
+  if (normalized.length <= ALL_ADMITTED_ROWS_FAILED_SAMPLE_TEXT_LIMIT) {
+    return normalized;
+  }
+  return `${normalized.slice(0, ALL_ADMITTED_ROWS_FAILED_SAMPLE_TEXT_LIMIT - 1)}…`;
+}
+
+function allAdmittedRowsFailedMessage(input: {
+  mapName: string;
+  failedCount: number;
+  totalInputCount: number;
+  samples: readonly AllAdmittedRowsFailedSample[];
+}): string {
+  const representativeFailures = input.samples
+    .slice(0, ALL_ADMITTED_ROWS_FAILED_SAMPLE_LIMIT)
+    .map(
+      (sample) =>
+        `${boundedAllAdmittedRowsFailedSample(sample.rowLabel)}: ${boundedAllAdmittedRowsFailedSample(sample.error)}`,
+    )
+    .join('; ');
+  const omittedCount = Math.max(
+    0,
+    input.failedCount -
+      Math.min(input.samples.length, ALL_ADMITTED_ROWS_FAILED_SAMPLE_LIMIT),
+  );
+  const omittedSuffix =
+    omittedCount > 0 ? `; ${omittedCount} additional failure(s) omitted` : '';
+  return (
+    `ctx.dataset("${input.mapName}") produced no successful rows ` +
+    `(${input.failedCount}/${input.totalInputCount} admitted rows failed). ` +
+    (representativeFailures
+      ? `Representative failures: ${representativeFailures}${omittedSuffix}. `
+      : '') +
+    'Failed rows were persisted for review and may be retried on a later run.'
+  );
+}
+
+export class AllAdmittedRowsFailedError extends Error {
+  readonly code = ALL_ADMITTED_ROWS_FAILED_CODE;
+
+  constructor(input: {
+    mapName: string;
+    failedCount: number;
+    totalInputCount: number;
+    samples?: readonly AllAdmittedRowsFailedSample[];
+  }) {
+    super(
+      `${ALL_ADMITTED_ROWS_FAILED_CODE}: ${allAdmittedRowsFailedMessage({
+        ...input,
+        samples: input.samples ?? [],
+      })}`,
+    );
+    this.name = 'AllAdmittedRowsFailedError';
+  }
+}
+
+export function isAllAdmittedRowsFailedFailure(error: unknown): boolean {
+  if (error instanceof AllAdmittedRowsFailedError) return true;
+  const code =
+    error && typeof error === 'object'
+      ? (error as { code?: unknown }).code
+      : null;
+  const message = error instanceof Error ? error.message : String(error ?? '');
+  return (
+    code === ALL_ADMITTED_ROWS_FAILED_CODE ||
+    new RegExp(`^${ALL_ADMITTED_ROWS_FAILED_CODE}:\\s*`).test(message)
+  );
+}
+
+function allAdmittedRowsFailedPublicMessage(cause: string): string {
+  return cause.replace(
+    new RegExp(`^${ALL_ADMITTED_ROWS_FAILED_CODE}:\\s*`),
+    '',
+  );
+}
+
 // User-facing message. Contains NO SQL, role names, database names, or hosts by
 // construction, so it is safe past redactRuntimeInternalError (which only strips
 // postgres-URI passwords). The raw pg error goes in the Error `cause` for server
@@ -323,6 +410,15 @@ function toErrorText(error: unknown): string {
 export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
   const rawCause = toErrorText(error);
   const cause = boundedFailureText(rawCause);
+  if (isAllAdmittedRowsFailedFailure(error)) {
+    return {
+      code: ALL_ADMITTED_ROWS_FAILED_CODE,
+      phase: 'runtime',
+      message: allAdmittedRowsFailedPublicMessage(cause),
+      retryable: false,
+      cause,
+    };
+  }
   if (
     error instanceof ToolExecutionError &&
     error.origin === 'deepline' &&

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -2043,6 +2043,16 @@ async function prepareAbsurdAttempt(input: {
             exists: projection.exists,
             blocked: projection.blocked,
           });
+          console.info('[runtime.scheduler.projection_deferred]', {
+            runId: input.runId,
+            eventId,
+            phase: 'run_created_projection_gate',
+            status: 'deferred',
+            exists: projection.exists,
+            blocked: projection.blocked,
+            delaySeconds: projection.blocked ? 5 : 1,
+            attemptAllocated: false,
+          });
           throw new DeferTask(
             `Run-created projection is not yet delivered for ${input.runId}.`,
             projection.blocked ? 5 : 1,

package/dist/cli/index.js

@@ -3068,7 +3068,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.120",
+  version: "0.3.122",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/cli/index.mjs

@@ -3063,7 +3063,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.120",
+  version: "0.3.122",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/index.js

@@ -863,7 +863,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.120",
+  version: "0.3.122",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/index.mjs

@@ -768,7 +768,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.120",
+  version: "0.3.122",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/install-integrity.json

@@ -35,6 +35,7 @@
     "dist/bundling-sources/shared_libs/monitors/analytics-install-guides.ts",
     "dist/bundling-sources/shared_libs/monitors/analytics-public.ts",
     "dist/bundling-sources/shared_libs/monitors/billing/charges.ts",
+    "dist/bundling-sources/shared_libs/monitors/billing/credit-recovery.ts",
     "dist/bundling-sources/shared_libs/monitors/billing/policy-builders.ts",
     "dist/bundling-sources/shared_libs/monitors/billing/pricing.ts",
     "dist/bundling-sources/shared_libs/monitors/billing/public-pricing.ts",
@@ -93,6 +94,7 @@
     "dist/bundling-sources/shared_libs/observability/log-metadata.ts",
     "dist/bundling-sources/shared_libs/observability/node-tracing.ts",
     "dist/bundling-sources/shared_libs/observability/queue-health.ts",
+    "dist/bundling-sources/shared_libs/observability/queue-item-connectors.ts",
     "dist/bundling-sources/shared_libs/observability/redaction.ts",
     "dist/bundling-sources/shared_libs/observability/request-context.ts",
     "dist/bundling-sources/shared_libs/observability/scheduled-job-errors.ts",

package/dist/release.d.mts

@@ -149,7 +149,7 @@ type SdkRelease = {
     supportPolicy: SdkSupportPolicy;
 };
 declare const SDK_RELEASE: {
-    readonly version: "0.3.120";
+    readonly version: "0.3.122";
     readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
     readonly packageCapabilities: {
         readonly updatePreferences: 1;

package/dist/release.d.ts

@@ -149,7 +149,7 @@ type SdkRelease = {
     supportPolicy: SdkSupportPolicy;
 };
 declare const SDK_RELEASE: {
-    readonly version: "0.3.120";
+    readonly version: "0.3.122";
     readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
     readonly packageCapabilities: {
         readonly updatePreferences: 1;

package/dist/release.js

@@ -74,7 +74,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.120",
+  version: "0.3.122",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/release.mjs

@@ -48,7 +48,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.120",
+  version: "0.3.122",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.3.120",
+  "version": "0.3.122",
   "description": "GTM data CLI and TypeScript SDK for coding agents",
   "license": "MIT",
   "homepage": "https://code.deepline.com",