deepline 0.3.121 → 0.3.123

package/dist/bundling-sources/shared_libs/observability/dlq.ts

@@ -177,6 +177,345 @@ export type DlqApplyResult = {
   readonly verificationRequired: boolean;
 };
 
+/**
+ * Queue-item control is deliberately a separate port from an inventory. A
+ * count and a bounded index description are not permission to manufacture a
+ * row scan: owners register this reader only when they can return exact item
+ * evidence without widening their data-access boundary.
+ */
+export const QUEUE_ITEM_CONTROL_CONTRACT_VERSION = 1 as const;
+
+/** The common state vocabulary exposed by every queue connector. */
+export const QUEUE_ITEM_STATES = [
+  'ready',
+  'delayed',
+  'running',
+  'held',
+  'quarantined',
+  'archived',
+  'completed',
+  'cancelled',
+  'unknown',
+] as const;
+
+export type QueueItemState = (typeof QUEUE_ITEM_STATES)[number];
+
+export type QueueItemAction =
+  | 'archive'
+  | 'quarantine'
+  | 'retry'
+  | 'hold'
+  | 'resume'
+  | 'cancel'
+  | 'update';
+
+export const QUEUE_ITEM_ACTIONS = [
+  'archive',
+  'quarantine',
+  'retry',
+  'hold',
+  'resume',
+  'cancel',
+  'update',
+] as const satisfies readonly QueueItemAction[];
+
+export type QueueItemActionAvailability = {
+  readonly action: QueueItemAction;
+  readonly allowed: boolean;
+  readonly reason?: string;
+};
+
+export type QueueItemFieldType =
+  | 'string'
+  | 'number'
+  | 'boolean'
+  | 'datetime'
+  | 'json';
+
+export type QueueItemFieldDefinition = {
+  readonly name: string;
+  readonly type: QueueItemFieldType;
+  readonly required: boolean;
+  readonly mutable: boolean;
+  readonly description?: string;
+};
+
+export type QueueItemLegacyAction = Exclude<DlqPathAction, 'inspect'>;
+export type QueueItemActionInput = QueueItemAction | QueueItemLegacyAction;
+
+export type QueueItemLastError = {
+  readonly code: string;
+  readonly message: string;
+};
+
+/** Names used in the API documentation and in connector implementations. */
+export type ActionAvailability = QueueItemActionAvailability;
+export type FieldDefinition = QueueItemFieldDefinition;
+
+/** A normalized item returned by an owner connector. `nativeState` is always
+ * retained so an operator can distinguish a connector mapping from the
+ * backend's actual state. */
+export type QueueItem = {
+  readonly queueId: string;
+  readonly id: string;
+  /** Opaque source revision used for compare-and-set mutation. */
+  readonly revision: string;
+  readonly observedAt: string;
+  readonly state: QueueItemState;
+  readonly nativeState: string;
+  readonly lastError?: QueueItemLastError;
+  readonly actions: readonly ActionAvailability[];
+  readonly editableFields: readonly FieldDefinition[];
+  readonly createdAt?: string;
+  readonly updatedAt?: string;
+  readonly summary?: string;
+};
+
+/** Keep old DLQ verbs working at the boundary while exposing a smaller,
+ * operator-readable control vocabulary. */
+export function normalizeQueueItemAction(
+  action: QueueItemActionInput,
+): QueueItemAction {
+  const aliases: Readonly<Record<QueueItemLegacyAction, QueueItemAction>> = {
+    redrive: 'retry',
+    discard: 'archive',
+    mark_done: 'archive',
+    handoff: 'resume',
+    recover: 'retry',
+    reconcile: 'update',
+    replay: 'retry',
+    retry: 'retry',
+    quarantine: 'quarantine',
+    release: 'cancel',
+    settle: 'resume',
+  };
+  return action in aliases
+    ? aliases[action as QueueItemLegacyAction]
+    : (action as QueueItemAction);
+}
+
+/**
+ * Legacy bounded evidence kept for callers that only consume the original
+ * `id/state/revision` projection. New connectors must return `QueueItem`.
+ */
+export type QueueItemEvidence = {
+  readonly id: string;
+  readonly state: string;
+  readonly revision: string;
+  /** Additive fields understood by newer readers; old producers may omit them. */
+  readonly nativeState?: string;
+  readonly lastError?: QueueItemLastError;
+  readonly actions?: readonly QueueItemActionAvailability[];
+  readonly editableFields?: readonly QueueItemFieldDefinition[];
+  readonly createdAt?: number;
+  readonly updatedAt?: number;
+  readonly summary?: string;
+};
+
+export type QueueItemUnavailable = {
+  readonly code: 'item_reader_unavailable';
+  readonly reason: string;
+};
+
+export type QueueItemList = {
+  readonly contractVersion: typeof QUEUE_ITEM_CONTROL_CONTRACT_VERSION;
+  readonly queueId: string;
+  readonly environment: string;
+  readonly observedAt: number;
+  readonly items: readonly QueueItem[];
+  readonly nextCursor: string | null;
+  readonly limit: number;
+  readonly cursor: string | null;
+  readonly unavailable?: QueueItemUnavailable;
+};
+
+export type QueueItemLookup =
+  | { readonly kind: 'found'; readonly item: QueueItem }
+  | { readonly kind: 'not_found' }
+  | {
+      readonly kind: 'unavailable';
+      readonly unavailable: QueueItemUnavailable;
+    };
+
+export type QueueItemActionRequest = {
+  readonly action: QueueItemAction;
+  readonly sourceAction?: QueueItemActionInput;
+  readonly idempotencyKey: string;
+  readonly expectedRevision: string;
+  readonly reason: string;
+  /** Connector-declared mutable fields for the `update` action only. */
+  readonly fields?: Readonly<Record<string, unknown>>;
+  /** Authenticated platform-admin identity forwarded to the owner connector. */
+  readonly actorId?: string;
+};
+
+export type QueueItemActionOperation = {
+  readonly contractVersion: typeof QUEUE_ITEM_CONTROL_CONTRACT_VERSION;
+  readonly operationId: string;
+  readonly queueId: string;
+  readonly itemId: string;
+  readonly environment: string;
+  readonly action: QueueItemAction;
+  readonly idempotencyKey: string;
+  readonly expectedRevision: string;
+  /** `pending` means the owner accepted the request but has not yet verified
+   * the native transition. `confirmed` is the only successful terminal
+   * status. The rejected/stale statuses preserve the existing safe boundary. */
+  readonly status: 'pending' | 'confirmed' | 'rejected' | 'stale' | 'failed';
+  readonly requestedAt?: string;
+  readonly confirmedAt?: string;
+  readonly replacementItemId?: string;
+  readonly item?: QueueItem;
+  readonly error?: QueueItemLastError;
+  readonly audit: {
+    readonly recorded: boolean;
+    readonly reason: string;
+  };
+  /** Existing owner command, when the catalog has one. This is guidance only;
+   * callers must still invoke its own authenticated, audited surface. */
+  readonly ownerCommand?: string;
+};
+
+export type QueueItemActionStatus =
+  | 'pending'
+  | 'confirmed'
+  | 'accepted'
+  | 'rejected'
+  | 'stale'
+  | 'failed'
+  | 'not_found'
+  | 'unavailable';
+
+/**
+ * Connector result for a requested transition. `confirmed` means the owner
+ * verified the resulting native state; `pending` means the owner durably
+ * accepted it but has not yet confirmed that state. `accepted` remains as a
+ * compatibility spelling for older connectors and must not be treated as
+ * confirmed by the shared control service.
+ */
+export type QueueItemActionResult = {
+  readonly contractVersion: typeof QUEUE_ITEM_CONTROL_CONTRACT_VERSION;
+  readonly operationId: string;
+  readonly queueId: string;
+  readonly itemId: string;
+  readonly environment: string;
+  readonly action: QueueItemAction;
+  readonly idempotencyKey: string;
+  readonly expectedRevision: string;
+  readonly status: QueueItemActionStatus;
+  readonly item?: QueueItem;
+  readonly replacementItemId?: string;
+  readonly reason?: string;
+  readonly audit: {
+    readonly recorded: boolean;
+    readonly auditId?: string;
+  };
+};
+
+export type QueueItemConnectorContext = {
+  readonly queueId: string;
+  readonly environment: string;
+  readonly now: number;
+  readonly limit: number;
+  readonly cursor?: string;
+  readonly actorId?: string;
+  readonly operationId?: string;
+};
+
+/**
+ * Owner implementation port. The connector owns native reads, state mapping,
+ * lease/CAS checks, and outcome verification. The shared control service only
+ * authenticates, validates, journals idempotency, and returns this result.
+ */
+export type QueueItemConnector = {
+  readonly id: string;
+  readonly queueId: string;
+  readonly source: string;
+  readonly mapNativeState: (nativeState: string) => QueueItemState;
+  readonly list: (context: QueueItemConnectorContext) => Promise<QueueItemList>;
+  readonly get: (
+    context: QueueItemConnectorContext,
+    itemId: string,
+  ) => Promise<QueueItemLookup>;
+  readonly requestAction: (
+    context: QueueItemConnectorContext,
+    itemId: string,
+    request: QueueItemActionRequest,
+  ) => Promise<QueueItemActionOperation>;
+};
+
+/** Conservative shared fallback for an unrecognised native state. */
+export function normalizeQueueItemState(nativeState: string): QueueItemState {
+  const state = nativeState.trim().toLowerCase();
+  if ((QUEUE_ITEM_STATES as readonly string[]).includes(state)) {
+    return state as QueueItemState;
+  }
+  if (
+    state === 'in_flight' ||
+    state === 'in-flight' ||
+    state === 'processing'
+  ) {
+    return 'running';
+  }
+  if (
+    state === 'dead_lettered' ||
+    state === 'dead-lettered' ||
+    state === 'dlq'
+  ) {
+    return 'quarantined';
+  }
+  if (state === 'done' || state === 'succeeded' || state === 'success') {
+    return 'completed';
+  }
+  if (state === 'canceled' || state === 'aborted') return 'cancelled';
+  return 'unknown';
+}
+
+export function unavailableQueueItemList(input: {
+  queueId: string;
+  environment: string;
+  observedAt: number;
+  reason: string;
+  limit?: number;
+  cursor?: string | null;
+}): QueueItemList {
+  return {
+    contractVersion: QUEUE_ITEM_CONTROL_CONTRACT_VERSION,
+    queueId: input.queueId,
+    environment: input.environment,
+    observedAt: input.observedAt,
+    items: [],
+    nextCursor: null,
+    limit: input.limit ?? 50,
+    cursor: input.cursor ?? null,
+    unavailable: { code: 'item_reader_unavailable', reason: input.reason },
+  };
+}
+
+/** A deterministic request identity is safe to show before an owner has a
+ * durable operation store. It is explicitly not represented as an audit. */
+export function queueItemOperationId(input: {
+  queueId: string;
+  itemId: string;
+  action: QueueItemAction;
+  idempotencyKey: string;
+}): string {
+  const value = [
+    input.queueId,
+    input.itemId,
+    input.action,
+    input.idempotencyKey,
+  ]
+    .map((part) => part.trim())
+    .join(':');
+  let hash = 2166136261;
+  for (let index = 0; index < value.length; index += 1) {
+    hash ^= value.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return `queue-item-${(hash >>> 0).toString(36)}`;
+}
+
 /** Every backend—including Absurd—must implement this port at its boundary. */
 export type DlqAdapter = {
   readonly id: string;

package/dist/bundling-sources/shared_libs/observability/queue-item-connectors.ts

@@ -0,0 +1,314 @@
+import {
+  normalizeQueueItemState,
+  queueItemOperationId,
+  unavailableQueueItemList,
+  type QueueItemActionOperation,
+  type QueueItemActionRequest,
+  type QueueItemConnector,
+  type QueueItemConnectorContext,
+  type QueueItemList,
+  type QueueItemLookup,
+  type QueueItemState,
+} from './dlq';
+import { OPERATIONS_QUEUE_CATALOG } from './queue-health';
+
+/** Native source families supported by the shared control-plane port. */
+export type QueueConnectorBackend =
+  | 'convex'
+  | 'workpool'
+  | 'postgres-outbox'
+  | 'absurd'
+  | 'provider';
+
+export type QueueItemConnectorRegistration = {
+  readonly backend: QueueConnectorBackend;
+  readonly connector: QueueItemConnector;
+};
+
+export type QueueItemConnectorRegistryOptions = {
+  /** Registration is intentionally injectable for tests and owner wiring. */
+  readonly registrations?: readonly QueueItemConnectorRegistration[];
+};
+
+export type QueueItemConnectorRegisterOptions = {
+  /** Only an unavailable placeholder may be replaced at startup. */
+  readonly replaceUnavailable?: boolean;
+};
+
+/**
+ * The registry is the routing boundary: the control service looks up a queue
+ * by its stable public id and never switches on a native table or provider.
+ * Duplicate registrations are rejected so a new owner cannot silently shadow
+ * an existing mutation implementation.
+ */
+export class QueueItemConnectorRegistry {
+  private readonly registrations = new Map<
+    string,
+    QueueItemConnectorRegistration
+  >();
+
+  constructor(options: QueueItemConnectorRegistryOptions = {}) {
+    for (const registration of options.registrations ?? []) {
+      this.register(registration);
+    }
+  }
+
+  register(
+    registration: QueueItemConnectorRegistration,
+    options: QueueItemConnectorRegisterOptions = {},
+  ): this {
+    const { backend, connector } = registration;
+    if (!backend) throw new Error('Queue connector backend is required.');
+    if (!connector.queueId.trim())
+      throw new Error('Queue connector queueId is required.');
+    if (!connector.id.trim())
+      throw new Error('Queue connector id is required.');
+    const existing = this.registrations.get(connector.queueId);
+    if (existing && !options.replaceUnavailable) {
+      throw new Error(
+        `A queue connector is already registered for ${connector.queueId}.`,
+      );
+    }
+    if (
+      existing &&
+      options.replaceUnavailable &&
+      !existing.connector.id.endsWith('-unavailable')
+    ) {
+      throw new Error(
+        `Queue connector ${connector.queueId} is already owned by ${existing.connector.id}; replacement is not allowed.`,
+      );
+    }
+    this.registrations.set(connector.queueId, registration);
+    return this;
+  }
+
+  get(queueId: string): QueueItemConnectorRegistration | undefined {
+    return this.registrations.get(queueId);
+  }
+
+  has(queueId: string): boolean {
+    return this.registrations.has(queueId);
+  }
+
+  list(): readonly QueueItemConnectorRegistration[] {
+    return [...this.registrations.values()];
+  }
+}
+
+type UnavailableConnectorOptions = {
+  readonly queueId: string;
+  readonly backend: QueueConnectorBackend;
+  readonly source: string;
+  readonly id?: string;
+  readonly reason?: string;
+};
+
+const unavailableReason = (options: UnavailableConnectorOptions): string =>
+  options.reason ??
+  `The ${options.backend} owner has not registered a bounded item reader and fenced mutation adapter for ${options.queueId}.`;
+
+/**
+ * Safe connector shell used until an owner supplies source-owned reads and
+ * transactions. It is deliberately observable and routable, but cannot
+ * claim an item exists or that a transition was applied.
+ */
+export function createUnavailableQueueItemConnector(
+  options: UnavailableConnectorOptions,
+): QueueItemConnectorRegistration {
+  const reason = unavailableReason(options);
+  const connector: QueueItemConnector = {
+    id: options.id ?? `${options.backend}-${options.queueId}-unavailable`,
+    queueId: options.queueId,
+    source: options.source,
+    mapNativeState: (nativeState: string): QueueItemState =>
+      normalizeQueueItemState(nativeState),
+    async list(context: QueueItemConnectorContext): Promise<QueueItemList> {
+      return unavailableQueueItemList({
+        queueId: options.queueId,
+        environment: context.environment,
+        observedAt: context.now,
+        reason,
+        limit: context.limit,
+        cursor: context.cursor,
+      });
+    },
+    async get(
+      _context: QueueItemConnectorContext,
+      _itemId: string,
+    ): Promise<QueueItemLookup> {
+      return {
+        kind: 'unavailable',
+        unavailable: { code: 'item_reader_unavailable', reason },
+      };
+    },
+    async requestAction(
+      context: QueueItemConnectorContext,
+      itemId: string,
+      request: QueueItemActionRequest,
+    ): Promise<QueueItemActionOperation> {
+      const operationId =
+        context.operationId ??
+        queueItemOperationId({
+          queueId: options.queueId,
+          itemId,
+          action: request.action,
+          idempotencyKey: request.idempotencyKey,
+        });
+      return {
+        contractVersion: 1,
+        operationId,
+        queueId: options.queueId,
+        itemId,
+        environment: context.environment,
+        action: request.action,
+        idempotencyKey: request.idempotencyKey,
+        expectedRevision: request.expectedRevision,
+        status: 'rejected',
+        audit: { recorded: false, reason },
+      };
+    },
+  };
+  return { backend: options.backend, connector };
+}
+
+type BackendConnectorOptions = Omit<
+  UnavailableConnectorOptions,
+  'backend' | 'source'
+> & {
+  readonly source?: string;
+  readonly reason?: string;
+};
+
+export function createConvexQueueItemConnector(
+  options: BackendConnectorOptions,
+): QueueItemConnectorRegistration {
+  return createUnavailableQueueItemConnector({
+    ...options,
+    backend: 'convex',
+    source: options.source ?? 'convex',
+  });
+}
+
+export function createWorkpoolQueueItemConnector(
+  options: BackendConnectorOptions,
+): QueueItemConnectorRegistration {
+  return createUnavailableQueueItemConnector({
+    ...options,
+    backend: 'workpool',
+    source: options.source ?? 'convex-workpool',
+  });
+}
+
+export function createPostgresOutboxQueueItemConnector(
+  options: BackendConnectorOptions,
+): QueueItemConnectorRegistration {
+  return createUnavailableQueueItemConnector({
+    ...options,
+    backend: 'postgres-outbox',
+    source: options.source ?? 'postgres-outbox',
+  });
+}
+
+export function createAbsurdQueueItemConnector(
+  options: BackendConnectorOptions,
+): QueueItemConnectorRegistration {
+  return createUnavailableQueueItemConnector({
+    ...options,
+    backend: 'absurd',
+    source: options.source ?? 'absurd',
+  });
+}
+
+export function createProviderQueueItemConnector(
+  options: BackendConnectorOptions,
+): QueueItemConnectorRegistration {
+  return createUnavailableQueueItemConnector({
+    ...options,
+    backend: 'provider',
+    source: options.source ?? 'provider',
+  });
+}
+
+/** Explicit owner mapping. Unknown queues must fail registration instead of
+ * silently falling back to a connector with potentially wrong semantics. */
+export const QUEUE_CONNECTOR_BACKENDS: Readonly<
+  Record<string, QueueConnectorBackend>
+> = {
+  'play-runtime-tasks': 'absurd',
+  'run-ledger-projector-outbox': 'postgres-outbox',
+  'compute-billing-jobs': 'postgres-outbox',
+  'play-webhook-receipts': 'convex',
+  'workflow-cron-bindings': 'convex',
+  'play-cron-bindings': 'convex',
+  'ingestion-jobs': 'provider',
+  'monitor-fleet-convergence': 'convex',
+  'monitor-deployment-jobs': 'workpool',
+  'monitor-batch-operations': 'workpool',
+  'provider-monitor-reconciliation': 'provider',
+  'signal-radar-inbound-dlq': 'provider',
+  'charge-reconciliation-backlog': 'postgres-outbox',
+  'workspace-fast-settlement': 'convex',
+  'metronome-usage-delivery': 'postgres-outbox',
+  'billing-operations': 'postgres-outbox',
+  'stripe-financial-inbox': 'provider',
+  'async-usage-recovery': 'postgres-outbox',
+  'notification-deliveries': 'workpool',
+  'usage-rollup-fold': 'postgres-outbox',
+  'play-deployment-jobs': 'workpool',
+  'github-runner-wait': 'provider',
+  'sandbox-cleanup-jobs': 'postgres-outbox',
+};
+
+function defaultRegistration(
+  queueId: string,
+  backend: QueueConnectorBackend,
+): QueueItemConnectorRegistration {
+  const options = {
+    queueId,
+    source: `${backend}:unavailable`,
+  };
+  if (backend === 'convex') return createConvexQueueItemConnector(options);
+  if (backend === 'workpool') return createWorkpoolQueueItemConnector(options);
+  if (backend === 'postgres-outbox')
+    return createPostgresOutboxQueueItemConnector(options);
+  if (backend === 'absurd') return createAbsurdQueueItemConnector(options);
+  return createProviderQueueItemConnector(options);
+}
+
+export function createDefaultQueueItemConnectorRegistry(): QueueItemConnectorRegistry {
+  const catalogIds = new Set(OPERATIONS_QUEUE_CATALOG.map((entry) => entry.id));
+  const mappedIds = Object.keys(QUEUE_CONNECTOR_BACKENDS);
+  const missing = [...catalogIds].filter(
+    (queueId) => QUEUE_CONNECTOR_BACKENDS[queueId] === undefined,
+  );
+  const extra = mappedIds.filter((queueId) => !catalogIds.has(queueId));
+  if (missing.length > 0 || extra.length > 0) {
+    throw new Error(
+      `Queue connector mapping drift: missing=${missing.join(',') || 'none'} extra=${extra.join(',') || 'none'}`,
+    );
+  }
+  return new QueueItemConnectorRegistry({
+    registrations: OPERATIONS_QUEUE_CATALOG.map((entry) =>
+      defaultRegistration(entry.id, QUEUE_CONNECTOR_BACKENDS[entry.id]!),
+    ),
+  });
+}
+
+/** Process-local default used by the admin control service. Owner modules can
+ * replace an unavailable registration during startup with
+ * `register(ownerRegistration, { replaceUnavailable: true })`; there is no
+ * fallback that mutates source data. */
+export const defaultQueueItemConnectorRegistry =
+  createDefaultQueueItemConnectorRegistry();
+
+/**
+ * Owner startup hook. A source module should register its connector before
+ * serving admin requests, replacing only the matching unavailable placeholder.
+ */
+export function registerQueueItemConnector(
+  registration: QueueItemConnectorRegistration,
+): void {
+  defaultQueueItemConnectorRegistry.register(registration, {
+    replaceUnavailable: true,
+  });
+}

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -655,7 +655,7 @@
     "id": "convex.provider-monitor-credit-recovery",
     "name": "Provider monitor credit-balance recovery",
     "scheduler": "convex_scheduled_action",
-    "cadence": "runAfter(0) after a non-admitting balance becomes positive",
+    "cadence": "runAfter(0) after a non-admitting balance becomes positive (legacy queued delivery)",
     "timezone": "UTC",
     "source": "packages/convex/scheduledJobsNode.ts",
     "locator": "recoverProviderMonitorsAfterCreditBalanceChange",
@@ -669,6 +669,24 @@
     "failureAlert": "convex_action_wrapper",
     "domainReportRequired": true
   },
+  {
+    "id": "convex.credit-blocked-automation-recovery",
+    "name": "Credit-blocked automation recovery",
+    "scheduler": "convex_scheduled_action",
+    "cadence": "runAfter(0) after any positive balance movement; Monitor recovery only at the non-admitting-to-positive crossing",
+    "timezone": "UTC",
+    "source": "packages/convex/scheduledJobsNode.ts",
+    "locator": "recoverCreditBlockedAutomationsAfterCreditBalanceChange",
+    "owner": "billing",
+    "criticality": "critical",
+    "purpose": "Recover scheduled Plays and provider Monitors blocked by insufficient credits after a positive balance movement.",
+    "successDefinition": "Eligible credit-blocked automations are resumed or a valid no-op is recorded; ineligible resources remain unchanged.",
+    "failureDefinition": "Recovery intent cannot be durably recorded or an eligible automation cannot be rescheduled or reactivated.",
+    "startGraceMs": 120000,
+    "runTimeoutMs": 1800000,
+    "failureAlert": "convex_action_wrapper",
+    "domainReportRequired": true
+  },
   {
     "id": "inngest.workflow-run-watchdog",
     "name": "Workflow run watchdog",