@opengeni/worker-bundle 1.0.0-canary.0 → 1.0.0

package/src/activities/session-state.ts

@@ -3,22 +3,30 @@ import {
   applySessionTurnSettlement,
   enqueueSessionWorkflowWake,
   failSessionWorkBeforeAttemptClaim,
+  blockSessionWorkBeforeAttemptClaim,
   requestSessionTurnRecovery,
   recoverSessionDispatch,
+  reconcileSettledSessionAttempt,
   reconcileSessionAttemptQuiescence,
+  reconcileCompletedSandboxSetup,
   peekSessionWork as peekSessionWorkDb,
   settleSessionInputWait as settleSessionInputWaitDb,
   countQueuedTurns,
   getSessionAttemptActivityRef,
   getSessionEvent,
   getSessionTurnForAttempt,
+  getWorkspace,
   expireSessionInteractionIntervention as expireSessionInteractionInterventionDb,
+  expireScheduledRunHumanWait as expireScheduledRunHumanWaitDb,
   expireSessionHumanInputRequest,
   markSessionAttemptQuiesced,
   requireSession,
   settleSessionIdleWithParentOutbox,
+  SANDBOX_SETUP_RECOVERY_LIMIT,
 } from "@opengeni/db";
 import { publishDurableSessionEvents } from "@opengeni/events";
+import { CancelledFailure } from "@temporalio/activity";
+import { currentActivityContext } from "./streaming";
 import { deliverFailedChildTurnToParent, notifyParentOfChildIdle } from "./parent-wake";
 import { recordTurnsQueuedGauge, recordWorkerDeathRecoveryMetrics } from "../observability-metrics";
 import {
@@ -30,6 +38,8 @@ import type {
   ExpireSessionHumanInputInput,
   ExpireSessionHumanInputResult,
   ExpireSessionInteractionInterventionInput,
+  ExpireScheduledRunHumanWaitInput,
+  ExpireScheduledRunHumanWaitResult,
   ExpireSessionInteractionInterventionResult,
   PeekSessionWorkInput,
   FailSessionAttemptInput,
@@ -39,6 +49,8 @@ import type {
   PersistSessionAttemptQuiescenceInput,
   ReconcileSessionAttemptQuiescenceInput,
   ReconcileSessionAttemptQuiescenceResult,
+  ReconcileSettledSessionAttemptInput,
+  ReconcileSettledSessionAttemptResult,
   RecoverDispatchInput,
   RecoverDispatchResult,
   RecoverEscapedMcpTimeoutInput,
@@ -52,17 +64,22 @@ export type SessionStateActivityOverrides = Partial<{
   applySessionTurnSettlement: typeof applySessionTurnSettlement;
   enqueueSessionWorkflowWake: typeof enqueueSessionWorkflowWake;
   failSessionWorkBeforeAttemptClaim: typeof failSessionWorkBeforeAttemptClaim;
+  blockSessionWorkBeforeAttemptClaim: typeof blockSessionWorkBeforeAttemptClaim;
   requestSessionTurnRecovery: typeof requestSessionTurnRecovery;
   recoverSessionDispatch: typeof recoverSessionDispatch;
+  reconcileSettledSessionAttempt: typeof reconcileSettledSessionAttempt;
   reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
+  reconcileCompletedSandboxSetup: typeof reconcileCompletedSandboxSetup;
   peekSessionWork: typeof peekSessionWorkDb;
   settleSessionInputWait: typeof settleSessionInputWaitDb;
   countQueuedTurns: typeof countQueuedTurns;
   getSessionAttemptActivityRef: typeof getSessionAttemptActivityRef;
   getSessionEvent: typeof getSessionEvent;
   getSessionTurnForAttempt: typeof getSessionTurnForAttempt;
+  getWorkspace: typeof getWorkspace;
   expireSessionHumanInputRequest: typeof expireSessionHumanInputRequest;
   expireSessionInteractionIntervention: typeof expireSessionInteractionInterventionDb;
+  expireScheduledRunHumanWait: typeof expireScheduledRunHumanWaitDb;
   requireSession: typeof requireSession;
   settleSessionIdleWithParentOutbox: typeof settleSessionIdleWithParentOutbox;
   markSessionAttemptQuiesced: typeof markSessionAttemptQuiesced;
@@ -91,22 +108,31 @@ export function createSessionStateActivities(
     overrides.enqueueSessionWorkflowWake ?? enqueueSessionWorkflowWake;
   const failSessionWorkBeforeAttemptClaimFn =
     overrides.failSessionWorkBeforeAttemptClaim ?? failSessionWorkBeforeAttemptClaim;
+  const blockSessionWorkBeforeAttemptClaimFn =
+    overrides.blockSessionWorkBeforeAttemptClaim ?? blockSessionWorkBeforeAttemptClaim;
   const requestSessionTurnRecoveryFn =
     overrides.requestSessionTurnRecovery ?? requestSessionTurnRecovery;
   const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
+  const reconcileSettledSessionAttemptFn =
+    overrides.reconcileSettledSessionAttempt ?? reconcileSettledSessionAttempt;
   const reconcileSessionAttemptQuiescenceFn =
     overrides.reconcileSessionAttemptQuiescence ?? reconcileSessionAttemptQuiescence;
   const peekSessionWorkFn = overrides.peekSessionWork ?? peekSessionWorkDb;
+  const reconcileCompletedSandboxSetupFn =
+    overrides.reconcileCompletedSandboxSetup ?? reconcileCompletedSandboxSetup;
   const settleSessionInputWaitFn = overrides.settleSessionInputWait ?? settleSessionInputWaitDb;
   const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns;
   const getSessionAttemptActivityRefFn =
     overrides.getSessionAttemptActivityRef ?? getSessionAttemptActivityRef;
   const getSessionEventFn = overrides.getSessionEvent ?? getSessionEvent;
   const getSessionTurnForAttemptFn = overrides.getSessionTurnForAttempt ?? getSessionTurnForAttempt;
+  const getWorkspaceFn = overrides.getWorkspace ?? getWorkspace;
   const expireSessionHumanInputRequestFn =
     overrides.expireSessionHumanInputRequest ?? expireSessionHumanInputRequest;
   const expireSessionInteractionInterventionFn =
     overrides.expireSessionInteractionIntervention ?? expireSessionInteractionInterventionDb;
+  const expireScheduledRunHumanWaitFn =
+    overrides.expireScheduledRunHumanWait ?? expireScheduledRunHumanWaitDb;
   const requireSessionFn = overrides.requireSession ?? requireSession;
   const settleSessionIdleWithParentOutboxFn =
     overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
@@ -153,6 +179,28 @@ export function createSessionStateActivities(
 
       const preClaimFailureDisposition =
         input.preClaimFailure?.disposition ?? input.preClaimFailureDisposition;
+      if (
+        preClaimFailureDisposition === "blocked" &&
+        input.admissionFence &&
+        input.preClaimFailure?.reason
+      ) {
+        const blocked = await blockSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
+          accountId: input.accountId,
+          sessionId: input.sessionId,
+          workflowId,
+          attemptId: input.attemptId,
+          fence: input.admissionFence,
+          reason: input.preClaimFailure.reason,
+          sqlState: input.preClaimFailure.sqlState ?? null,
+        });
+        await publishDurableSessionEventsFn(
+          bus,
+          input.workspaceId,
+          input.sessionId,
+          blocked.events,
+        );
+        return { action: blocked.action };
+      }
       if (preClaimFailureDisposition === "permanent" && input.trigger) {
         const failed = await failSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
           accountId: input.accountId,
@@ -160,7 +208,14 @@ export function createSessionStateActivities(
           workflowId,
           trigger: input.trigger,
           error: input.error ?? "Agent turn admission failed before attempt claim.",
-          ...(input.preClaimFailure ? { admissionFailure: input.preClaimFailure } : {}),
+          ...(input.preClaimFailure?.disposition === "permanent"
+            ? {
+                admissionFailure: {
+                  disposition: "permanent" as const,
+                  code: input.preClaimFailure.code,
+                },
+              }
+            : {}),
         });
         if (failed.action === "terminal") return { action: "terminal" };
         if (failed.action === "stale") return { action: "stale" };
@@ -198,6 +253,9 @@ export function createSessionStateActivities(
       turn.triggerEventId === postClaimRecovery.triggerEventId &&
       turn.executionGeneration === postClaimRecovery.executionGeneration,
     );
+    if (postClaimRecovery?.sandboxSetupRecoveryExhausted && !postClaimIdentityMatches) {
+      return { action: "stale" };
+    }
     const providerRecoveryCount = postClaimIdentityMatches
       ? postClaimRecovery?.providerRecoveryCount
       : undefined;
@@ -206,6 +264,19 @@ export function createSessionStateActivities(
       : undefined;
     const hasProviderRecoveryCount = providerRecoveryCount !== undefined;
     const hasProviderFailureCode = providerFailureCode !== undefined;
+    const setupOutcomeUnknown =
+      postClaimIdentityMatches && postClaimRecovery?.sandboxSetupOutcomeUnknown === true;
+    const setupRecoveryExhausted =
+      postClaimIdentityMatches && postClaimRecovery?.sandboxSetupRecoveryExhausted === true;
+    if (
+      (setupOutcomeUnknown && setupRecoveryExhausted) ||
+      ((setupOutcomeUnknown || setupRecoveryExhausted) &&
+        (hasProviderRecoveryCount || hasProviderFailureCode)) ||
+      (setupRecoveryExhausted &&
+        turn.metadata?.providerRecoveryCount !== SANDBOX_SETUP_RECOVERY_LIMIT)
+    ) {
+      return { action: "stale" };
+    }
     if (hasProviderRecoveryCount !== hasProviderFailureCode) {
       return { action: "stale" };
     }
@@ -234,11 +305,30 @@ export function createSessionStateActivities(
         turnId: turn.id,
         triggerEventId: turn.triggerEventId,
         attemptId: input.attemptId,
-        reason: providerFailureCode ?? "claimed_attempt_database_failure",
+        reason: setupOutcomeUnknown
+          ? "sandbox_command_start_outcome_unknown"
+          : setupRecoveryExhausted
+            ? "sandbox_command_start_recovery_exhausted"
+            : (providerFailureCode ?? "claimed_attempt_database_failure"),
+        ...(setupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {}),
+        ...(setupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {}),
         ...(providerRecoveryCount !== undefined ? { providerRecoveryCount } : {}),
         detail: {
-          code: providerFailureCode ?? recoveredClaimCode,
-          retryable: true,
+          code: setupOutcomeUnknown
+            ? "sandbox_command_start_outcome_unknown"
+            : setupRecoveryExhausted
+              ? "sandbox_command_start_recovery_exhausted"
+              : (providerFailureCode ?? recoveredClaimCode),
+          retryable: !setupOutcomeUnknown && !setupRecoveryExhausted,
+          ...(setupOutcomeUnknown ? { setupOutcome: "unknown", replay: "blocked" } : {}),
+          ...(setupRecoveryExhausted
+            ? {
+                setupOutcome: "not_started",
+                replay: "blocked",
+                recoveryExhausted: true,
+                providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT,
+              }
+            : {}),
           ...(providerRecoveryCount !== undefined
             ? {
                 databaseFailureCode: recoveredClaimCode,
@@ -390,6 +480,63 @@ export function createSessionStateActivities(
     return { action: result.action };
   }
 
+  /** A new authenticated inspection, never trust the earlier observer hint.
+   * The DB seam independently revalidates the entire stored/current identity,
+   * control and both writer predicates under canonical locks before closing. */
+  async function reconcileSettledSessionAttemptActivity(
+    input: ReconcileSettledSessionAttemptInput,
+  ): Promise<ReconcileSettledSessionAttemptResult> {
+    const { db, bus, inspectSessionAttemptActivity, settings, observability, wakeSessionWorkflow } =
+      await services();
+    const ref = await getSessionAttemptActivityRefFn(db, {
+      ...input,
+      temporalWorkflowId: input.workflowId,
+    });
+    if (!ref || ref.workflowRunId !== input.workflowRunId || ref.activityId !== input.activityId)
+      return { action: "stale" };
+    if (!inspectSessionAttemptActivity) return { action: "pending" };
+    try {
+      if ((await inspectSessionAttemptActivity(ref)) !== "settled") return { action: "pending" };
+    } catch (error) {
+      if (error instanceof CancelledFailure) throw error;
+      if (currentActivityContext()?.cancellationSignal.aborted)
+        throw new CancelledFailure("Settled-owner reconciliation cancelled");
+      return { action: "pending" };
+    }
+    if (currentActivityContext()?.cancellationSignal.aborted)
+      throw new CancelledFailure("Settled-owner reconciliation cancelled");
+    const result = await reconcileSettledSessionAttemptFn(db, {
+      accountId: input.accountId,
+      workspaceId: input.workspaceId,
+      sessionId: input.sessionId,
+      turnId: input.turnId,
+      attemptId: input.attemptId,
+      executionGeneration: input.executionGeneration,
+      temporalWorkflowId: ref.workflowId,
+      temporalWorkflowRunId: ref.workflowRunId,
+      temporalActivityId: ref.activityId,
+      activitySettled: true,
+      maxRedispatches: WORKER_DEATH_MAX_REDISPATCHES,
+    });
+    // Atomic Postgres recovery/quiescence/wake truth is already sufficient if
+    // live fanout fails. It must not be replaced by an observer-only wake.
+    if (result.events.length > 0)
+      await publishDurableSessionEventsFn(
+        bus,
+        input.workspaceId,
+        input.sessionId,
+        result.events,
+      ).catch(() => undefined);
+    if (result.action === "exceeded")
+      await deliverFailedChildTurnToParentFn(
+        { db, bus, settings, observability, wakeSessionWorkflow },
+        input.workspaceId,
+        input.sessionId,
+        result.turnId,
+      );
+    return { action: result.action };
+  }
+
   /**
    * Recover the same current inference when its worker dies without completing
    * a graceful checkpoint (heartbeat timeout, SIGKILL, OOM, or node loss).
@@ -488,8 +635,86 @@ export function createSessionStateActivities(
   }
 
   async function peekSessionWork(input: PeekSessionWorkInput) {
-    const { db, observability } = await services();
-    const peek = await peekSessionWorkFn(db, input.workspaceId, input.sessionId);
+    const { db, bus, observability, inspectSessionAttemptActivity } = await services();
+    // Already scheduled activities retain their old input across workflow upgrades.
+    // Resolve its workspace scope exactly as the legacy DB path did, then use the
+    // observer path so absent rows and a still-owned attempt are observations.
+    const observerAccountId =
+      input.observerAccountId ?? (await getWorkspaceFn(db, input.workspaceId))?.accountId;
+    if (!observerAccountId) return { kind: "unavailable" as const };
+    let peek = await peekSessionWorkFn(
+      db,
+      input.workspaceId,
+      input.sessionId,
+      input.includeAdmissionFence,
+      observerAccountId,
+    );
+    if (peek.kind === "unavailable") return peek;
+    if (peek.kind === "admission-blocked" && peek.reason === "sandbox_setup_outcome_unknown") {
+      const ref = peek.ref;
+      if (ref && "turnId" in ref && "attemptId" in ref) {
+        const settled = await reconcileCompletedSandboxSetupFn(db, {
+          accountId: observerAccountId,
+          workspaceId: input.workspaceId,
+          sessionId: input.sessionId,
+          turnId: ref.turnId,
+          attemptId: ref.attemptId,
+        });
+        if (settled.events.length > 0)
+          await publishDurableSessionEventsFn(
+            bus,
+            input.workspaceId,
+            input.sessionId,
+            settled.events,
+          );
+        if (settled.reconciled)
+          peek = await peekSessionWorkFn(
+            db,
+            input.workspaceId,
+            input.sessionId,
+            input.includeAdmissionFence,
+            observerAccountId,
+          );
+      }
+    }
+    if (peek.kind === "attempt-owned") {
+      // Observation never revokes a writer or recovers a live owner. In
+      // particular, a settled Temporal activity is not physical-writer proof.
+      let ownerActivityState: "pending" | "settled" | "unknown" = "unknown";
+      if (inspectSessionAttemptActivity) {
+        try {
+          ownerActivityState = await inspectSessionAttemptActivity(peek.activityRef);
+        } catch (error) {
+          if (error instanceof CancelledFailure) throw error;
+          if (currentActivityContext()?.cancellationSignal.aborted)
+            throw new CancelledFailure("Control observation cancelled");
+          // This optional metadata observation grants no recovery authority.
+          // An unavailable inspector must not pin the control activity in
+          // retries and prevent a fresh Pause/owner/visibility observation.
+          // Database reads below remain outside this catch and retry normally.
+        }
+      }
+      if (currentActivityContext()?.cancellationSignal.aborted)
+        throw new CancelledFailure("Control observation cancelled");
+      const current = await peekSessionWorkFn(
+        db,
+        input.workspaceId,
+        input.sessionId,
+        input.includeAdmissionFence,
+        observerAccountId,
+      );
+      if (
+        current.kind !== "attempt-owned" ||
+        current.turnId !== peek.turnId ||
+        current.attemptId !== peek.attemptId ||
+        current.executionGeneration !== peek.executionGeneration ||
+        current.activityRef.workflowId !== peek.activityRef.workflowId ||
+        current.activityRef.workflowRunId !== peek.activityRef.workflowRunId ||
+        current.activityRef.activityId !== peek.activityRef.activityId
+      )
+        return current;
+      return { ...current, ownerActivityState };
+    }
     await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
     return peek;
   }
@@ -530,6 +755,18 @@ export function createSessionStateActivities(
     return { action: result.action };
   }
 
+  /** A scheduled run's approval timeout answers for its unanswered human wait. */
+  async function expireScheduledRunHumanWait(
+    input: ExpireScheduledRunHumanWaitInput,
+  ): Promise<ExpireScheduledRunHumanWaitResult> {
+    const { db, bus } = await services();
+    const result = await expireScheduledRunHumanWaitFn(db, input);
+    if (result.events.length > 0) {
+      await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
+    }
+    return { action: result.action };
+  }
+
   async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
     const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
     const settled = await settleSessionIdleWithParentOutboxFn(
@@ -560,12 +797,14 @@ export function createSessionStateActivities(
     settleSessionInterruptions,
     persistSessionAttemptQuiescence,
     reconcileSessionAttemptQuiescence: reconcileSessionAttemptQuiescenceActivity,
+    reconcileSettledSessionAttempt: reconcileSettledSessionAttemptActivity,
     recoverDispatch,
     recoverEscapedMcpTimeout,
     peekSessionWork,
     settleSessionInputWait,
     expireSessionHumanInput,
     expireSessionInteractionIntervention,
+    expireScheduledRunHumanWait,
     markSessionIdle,
   };
 }

package/src/activities/site-auth-maintenance.ts

@@ -174,6 +174,9 @@ async function dispatchMaintenanceSession(
     accountId: maintenance.accountId,
     workspaceId: maintenance.workspaceId,
     initialMessage: maintenancePrompt(maintenance),
+    // OpenGeni's own periodic check: no human or product surface entered it.
+    surface: "system",
+    metrics: service.observability,
     resources: [],
     tools: [{ kind: "mcp", id: "opengeni" }],
     toolPolicy: { mode: "explicit", inheritedFromSessionId: null },

package/src/activities/types.ts

@@ -7,7 +7,11 @@ import type {
   ScheduledTaskTriggerType,
   TurnInitiator,
 } from "@opengeni/contracts";
-import type { Database, SessionWorkflowWakeDeliveryResult } from "@opengeni/db";
+import type {
+  Database,
+  SessionWorkflowWakeDeliveryResult,
+  SessionAdmissionFence,
+} from "@opengeni/db";
 import type { DocumentServices } from "@opengeni/documents";
 import type { EventBus } from "@opengeni/events";
 import type { Observability } from "@opengeni/observability";
@@ -140,6 +144,11 @@ export type ControlActivityServices = SharedActivityServices;
 
 /** Turn workers own the model loop and never construct document parsers. */
 export type TurnActivityServices = SharedActivityServices & {
+  /** Host-owned containment: stop polling and checkpoint peer turns before exit.
+   * Bare activity embedders must provide their own worker lifecycle edge. */
+  requestWorkerDrain: () => void;
+  /** Deterministic test clock for the cleanup-only monitor. */
+  turnFinalizationTimeoutMs?: number | undefined;
   runtime: OpenGeniRuntime;
   /** Provider-free test/profiling seam; production injects the real runtime summarizer. */
   summarizeContextForCompaction: typeof import("@opengeni/runtime").summarizeForCompaction;
@@ -153,7 +162,7 @@ export type ActivityServices = ControlActivityServices &
 
 export type CodexCapacityWaitRef = {
   /** Absent only for Temporal histories written before provider-tagged waits. */
-  provider?: "codex" | "xai";
+  provider?: "codex" | "xai" | "claude";
   waiterId: string;
   generation: number;
   nextCheckAt: string;
@@ -175,7 +184,7 @@ export type ReconcileCodexCapacityWaitInput = {
   generation: number;
   cause: "timer" | "signal" | "queue" | "recovery";
   /** Absent only for Codex waits and pre-provider-tagged workflow histories. */
-  provider?: "codex" | "xai";
+  provider?: "codex" | "xai" | "claude";
 };
 
 export type ReconcileCodexCapacityWaitResult =
@@ -235,6 +244,22 @@ export type ReconcileSessionAttemptQuiescenceResult = {
   action: "quiesced" | "pending" | "stale";
 };
 
+export type ReconcileSettledSessionAttemptInput = {
+  accountId: string;
+  workspaceId: string;
+  sessionId: string;
+  turnId: string;
+  attemptId: string;
+  executionGeneration: number;
+  workflowId: string;
+  workflowRunId: string;
+  activityId: string;
+};
+
+export type ReconcileSettledSessionAttemptResult = {
+  action: "recovering" | "exceeded" | "pending" | "stale";
+};
+
 export type FailSessionAttemptInput = {
   accountId: string;
   workspaceId: string;
@@ -252,6 +277,7 @@ export type FailSessionAttemptInput = {
    * committed claim discovered by the control lane can recover the exact
    * attempt instead of terminally failing it. */
   preClaimFailure?: PreClaimFailureDetail;
+  admissionFence?: SessionAdmissionFence;
   /** Added in v4. A claimed attempt that lost operational database access
    * before turn-start completion carries its exact immutable turn identity so
    * the DB-only control lane can recover it instead of terminally failing it. */
@@ -263,17 +289,24 @@ export type FailSessionAttemptInput = {
 };
 
 export type FailSessionAttemptResult =
+  | { action: "blocked" }
   | { action: "failed" }
   | { action: "recovering" }
   | { action: "unclaimed" }
   | { action: "terminal" }
   | { action: "stale" };
 
-export type PreClaimFailureDisposition = "retryable" | "permanent";
+export type PreClaimFailureDisposition = "retryable" | "permanent" | "blocked";
 
 export type PreClaimFailureDetail = {
   disposition: PreClaimFailureDisposition;
   code: "db_deadlock" | "db_serialization_failure" | "db_failure" | "claim_invariant";
+  sqlState?: string | null;
+  reason?:
+    | "database_claim_rejected"
+    | "initiator_membership_required"
+    | "personal_resource_grant_required";
+  retryPolicy?: "explicit_recheck";
 };
 
 export const PRE_CLAIM_FAILURE_TYPE = "OpenGeniPreClaimFailure";
@@ -290,6 +323,10 @@ export type PostClaimDatabaseRecoveryDetail = {
   providerRecoveryCount?: number;
   /** Safe classified provider cause paired with providerRecoveryCount. */
   providerFailureCode?: string;
+  /** Persist the no-replay setup marker, not a new provider/setup attempt. */
+  sandboxSetupOutcomeUnknown?: true;
+  /** Preserve the exhausted budget for a command proven not dispatched. */
+  sandboxSetupRecoveryExhausted?: true;
 };
 
 export const POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE = "OpenGeniPostClaimDatabaseRecovery";
@@ -344,6 +381,9 @@ export type RecoverEscapedMcpTimeoutResult = {
 export type PeekSessionWorkInput = {
   workspaceId: string;
   sessionId: string;
+  includeAdmissionFence?: boolean;
+  /** Versioned workflow observer opt-in, never a caller authorization grant. */
+  observerAccountId?: string;
 };
 
 export type SettleSessionInputWaitInput = {
@@ -380,6 +420,18 @@ export type ExpireSessionInteractionInterventionResult = {
   action: "expired" | "stale" | "not_found";
 };
 
+export type ExpireScheduledRunHumanWaitInput = {
+  accountId: string;
+  workspaceId: string;
+  sessionId: string;
+  turnId: string;
+  runId: string;
+};
+
+export type ExpireScheduledRunHumanWaitResult = {
+  action: "expired" | "stale" | "not_found";
+};
+
 export type MarkSessionIdleInput = {
   workspaceId: string;
   sessionId: string;
@@ -409,6 +461,7 @@ export type DispatchScheduledTaskRunInput = {
       triggerType: Extract<ScheduledTaskTriggerType, "scheduled">;
       agentRunUsageIdempotencyKey?: never;
       initiator?: never;
+      credentialRestriction?: never;
     }
   | {
       triggerType: Extract<
@@ -418,6 +471,8 @@ export type DispatchScheduledTaskRunInput = {
       agentRunUsageIdempotencyKey: string;
       /** Exact identity used by the API-side charge for this same trigger. */
       initiator: TurnInitiator;
+      /** Trusted API-side caller ceiling for this occurrence, not the schedule. */
+      credentialRestriction?: "developer_setup";
     }
 );
 
@@ -425,8 +480,13 @@ export type DispatchScheduledTaskRunResult =
   | { action: "deleted" }
   | {
       action: "blocked";
+      runId?: string;
+      diagnostic?: import("@opengeni/contracts").ConnectionAccountSelectionDiagnostic;
+      /** Present when the occurrence was refused as a visible run receipt. */
+      refusal?: import("@opengeni/contracts").ScheduledTaskAdmissionRefusal;
       reason:
         | "insufficient_credits"
+        | "allowance_exhausted"
         | "monthly_model_cost_limit"
         | "monthly_agent_run_limit"
         | "malformed_manual_trigger"
@@ -434,8 +494,15 @@ export type DispatchScheduledTaskRunResult =
         | "scheduled_authority_exhausted"
         | "scheduled_run_terminal"
         | "scheduled_execution_unrepresentable"
+        | "connection_account_unavailable"
+        | "scheduled_authority_unavailable"
+        | "machine_target_unavailable"
+        | "machine_enrollment_inactive"
+        | "variable_set_unavailable"
+        | "rig_version_unavailable"
         | "knowledge_source_paused"
         | "legacy_source_schedule_requires_migration"
+        | "atlassian_native_retired"
         | "incident_preflight_metadata_missing"
         | "incident_responder_under_capable"
         | "incident_data_source_unsuitable";
@@ -552,5 +619,5 @@ export type RunAgentTurnResult =
   | ClaimedRunAgentTurnResult
   | {
       status: "unclaimed";
-      reason: "gate-closed" | "no-work" | "stale-approval" | "control-pending";
+      reason: "gate-closed" | "no-work" | "stale-approval" | "control-pending" | "dispatch-expired";
     };

package/src/activities/workflow-wake.ts

@@ -6,6 +6,7 @@ import {
   type Database,
 } from "@opengeni/db";
 import { publishDurableWorkspaceControlEvent } from "@opengeni/events";
+import { recordWorkflowWakeReconciliation } from "@opengeni/observability";
 import type { ControlActivityServices } from "./types";
 import {
   reconcileAutomaticSessionTitleFanout,
@@ -115,23 +116,7 @@ export function createWorkflowWakeActivities(services: () => Promise<ControlActi
         pendingAdmissionBlockers,
         exhaustedBatchLimit,
       };
-      if (claimed > 0) {
-        service.observability.info("session workflow wake delivery reconciled", {
-          claimed,
-          signaled,
-          delivered,
-          pendingAdmission,
-          unconfirmed,
-          failed,
-          exhaustedBatchLimit,
-          ...Object.fromEntries(
-            Object.entries(pendingAdmissionBlockers).map(([blocker, count]) => [
-              `pendingAdmissionBlockers.${blocker}`,
-              count,
-            ]),
-          ),
-        });
-      }
+      recordWorkflowWakeReconciliation(service.observability, result);
       return result;
     },
   };

package/src/activities/workspace-capture.ts

@@ -947,13 +947,24 @@ async function runCapture(
   }
 
   // ── 10. observe completion ────────────────────────────────────────────────
-  const durationMs = Date.now() - startedAt;
+  recordWorkspaceCaptureCompleted(observability, Date.now() - startedAt);
+}
+
+/** Count one committed logical capture and observe its end-to-end duration. */
+export function recordWorkspaceCaptureCompleted(
+  observability: Pick<Observability, "incrementCounter" | "observeHistogram">,
+  durationMs: number,
+): void {
   observability.incrementCounter({
     name: "opengeni_workspace_capture_total",
     labels: { result: "ok" },
   });
+  // Distinct from the physical `opengeni_workspace_capture_duration_seconds`
+  // {backend,outcome} histogram the runtime hook records in this same
+  // registry: one name cannot carry two label sets.
   observability.observeHistogram({
-    name: "opengeni_workspace_capture_duration_seconds",
+    name: "opengeni_workspace_capture_revision_duration_seconds",
+    help: "Turn-end logical workspace revision capture duration for committed revisions.",
     value: durationMs / 1000,
   });
 }