@opengeni/worker-bundle 1.0.2 → 1.2.0

package/src/activities/agent-turn/model-usage.ts

@@ -1,3 +1,4 @@
+import { canonicalizeConfiguredModelId } from "@opengeni/config";
 import {
   applyCreditDebitUpToBalance,
   recordUsageEvent,
@@ -55,6 +56,27 @@ export function modelUsageSourceKey(input: {
   return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
 }
 
+/** Legacy aggregate usage may only debit a policy shared by its completed calls. */
+export function aggregateCreditPolicyRevision(input: {
+  responseRevisions: ReadonlySet<number | undefined>;
+  lastAdmittedRevision: number | undefined;
+  chargesOpenGeniCredits: boolean;
+  totalTokens: number | null;
+}): number | undefined {
+  if (
+    input.chargesOpenGeniCredits &&
+    input.responseRevisions.size > 1 &&
+    (input.totalTokens ?? 0) > 0
+  ) {
+    throw new Error("Aggregate model usage spans different credit policy revisions");
+  }
+  // Bind the completed response, even if later preparation changed admission.
+  // Older runtimes without response callbacks retain their admitted snapshot.
+  return input.responseRevisions.size === 1
+    ? input.responseRevisions.values().next().value
+    : input.lastAdmittedRevision;
+}
+
 export function providerContextTokens(
   usage:
     | {
@@ -209,6 +231,7 @@ export async function processModelResponseTerminalEvent(input: {
   externallyBilled: boolean;
   chargesOpenGeniCredits?: boolean;
   countsTowardTokenCap?: boolean;
+  creditPolicyRevision?: number | undefined;
   servingCredentialId: string | null;
   priorSessionCredentialId: string | null;
   emittedSourceKeys: Set<string>;
@@ -231,6 +254,15 @@ export async function processModelResponseTerminalEvent(input: {
   if (!terminal) {
     return { status: "not_response" };
   }
+  // Some providers mirror a terminal response before normalized usage arrives.
+  // Claiming that empty raw mirror would discard the SDK's billable response.
+  if (
+    !terminal.usage &&
+    input.event.type === "raw_model_stream_event" &&
+    input.event.data.type !== "response_done"
+  ) {
+    return { status: "not_response" };
+  }
 
   const responseOrdinal = input.state.responseCount + 1;
   const sourceKey = modelUsageSourceKey({
@@ -268,6 +300,7 @@ export async function processModelResponseTerminalEvent(input: {
         turnId: input.turnId,
         turnAttemptId: input.turnAttemptId,
         model: input.model,
+        creditPolicyRevision: input.creditPolicyRevision,
         externallyBilled: input.externallyBilled,
         ...(input.chargesOpenGeniCredits !== undefined
           ? { chargesOpenGeniCredits: input.chargesOpenGeniCredits }
@@ -374,6 +407,7 @@ export async function processCompactionModelUsageEvent(input: {
   externallyBilled: boolean;
   chargesOpenGeniCredits?: boolean;
   countsTowardTokenCap?: boolean;
+  creditPolicyRevision?: number | undefined;
   servingCredentialId: string | null;
   priorSessionCredentialId: string | null;
   emittedSourceKeys: Set<string>;
@@ -416,6 +450,7 @@ export async function processCompactionModelUsageEvent(input: {
         turnId: input.turnId,
         turnAttemptId: input.turnAttemptId,
         model: input.model,
+        creditPolicyRevision: input.creditPolicyRevision,
         externallyBilled: input.externallyBilled,
         ...(input.chargesOpenGeniCredits !== undefined
           ? { chargesOpenGeniCredits: input.chargesOpenGeniCredits }
@@ -651,6 +686,7 @@ export async function recordModelUsageAndDebitCredits(
     externallyBilled: boolean;
     chargesOpenGeniCredits?: boolean;
     countsTowardTokenCap?: boolean;
+    creditPolicyRevision?: number | undefined;
     gatewayBilling?: ModelResponseUsage["gatewayBilling"];
     usage?: ModelUsageInput | ModelCallUsageInput | null;
     normalizedUsage?: ModelCallUsageNormalization;
@@ -842,6 +878,8 @@ export async function recordModelUsageAndDebitCredits(
       workspaceId: input.workspaceId,
       type: "model_usage_debit",
       requestedAmountMicros: costMicros,
+      modelId: canonicalizeConfiguredModelId(settings, input.model),
+      creditPolicyRevision: input.creditPolicyRevision,
       sourceType: "model_response",
       sourceId: `${input.turnId}:${input.sourceKey}`,
       idempotencyKey: `credit:model_usage_debit:${input.turnId}:${input.sourceKey}`,

package/src/activities/agent-turn/provider-recovery-metrics.ts

@@ -0,0 +1,64 @@
+import type { Observability } from "@opengeni/observability";
+
+export type ProviderRecoveryObservation = {
+  startedAt: number;
+  cause: "rate_limited" | "unavailable" | "connectivity";
+};
+
+export function providerRecoveryCause(code: unknown): ProviderRecoveryObservation["cause"] | null {
+  if (code === "provider_rate_limited") return "rate_limited";
+  if (code === "provider_unavailable") return "unavailable";
+  if (code === "upstream_connectivity_unavailable") return "connectivity";
+  return null;
+}
+
+export function readProviderRecoveryObservation(
+  metadata: Record<string, unknown>,
+): ProviderRecoveryObservation | undefined {
+  const cause = providerRecoveryCause(metadata.providerRecoveryReason);
+  const startedAt =
+    typeof metadata.providerRecoveryStartedAt === "string"
+      ? Date.parse(metadata.providerRecoveryStartedAt)
+      : Number.NaN;
+  return cause && Number.isFinite(startedAt) ? { cause, startedAt } : undefined;
+}
+
+/** Operational observations only; never retry authority, billing or diagnostics. */
+export function recordProviderRecoveryOutcome(
+  observability: Observability,
+  input: {
+    route?: { provider: string; model: string } | undefined;
+    cause: ProviderRecoveryObservation["cause"];
+    outcome: "scheduled" | "recovered" | "exhausted";
+    delayMs?: number;
+    elapsedMs?: number;
+  },
+): void {
+  if (!input.route) return;
+  const labels = { ...input.route, cause: input.cause, outcome: input.outcome };
+  try {
+    observability.incrementCounter({
+      name: "opengeni_model_recovery_total",
+      help: "Observed model recovery decisions and successful resumptions.",
+      labels,
+    });
+    if (input.delayMs !== undefined) {
+      observability.observeHistogram({
+        name: "opengeni_model_recovery_delay_seconds",
+        help: "Scheduled model recovery delay, including provider hints and jitter.",
+        labels: { ...input.route, cause: input.cause },
+        value: Math.max(0, input.delayMs) / 1_000,
+      });
+    }
+    if (input.elapsedMs !== undefined) {
+      observability.observeHistogram({
+        name: "opengeni_model_recovery_duration_seconds",
+        help: "Elapsed recovery episode including backoff, preparation and model requests.",
+        labels,
+        value: Math.max(0, input.elapsedMs) / 1_000,
+      });
+    }
+  } catch {
+    // Metrics cannot interrupt settlement or model progress.
+  }
+}

package/src/activities/agent-turn/run.ts

@@ -1090,6 +1090,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
               ),
             );
           const compactionPrep = await prepareCompaction({
+            entitlements,
             input,
             settings: capabilitySettings,
             db,

package/src/activities/agent-turn/stream-attempt.ts

@@ -18,6 +18,7 @@ import {
   updateSessionTitleWithEvent,
 } from "@opengeni/db";
 import { publishDurableSessionEvents } from "@opengeni/events";
+import { recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
 import {
   AssistantMessagePhaseTracker,
   normalizeModelCallUsage,
@@ -121,6 +122,7 @@ import {
   completedToolCallFromSdkEvent,
 } from "./history";
 import {
+  aggregateCreditPolicyRevision,
   modelUsageSourceKey,
   recordCompletedModelCallBeforeOwnershipFences,
   TurnEventPublisher,
@@ -383,6 +385,8 @@ export async function runTurnStreamAttempt(
   );
   let parallelSessionTitle: ReturnType<typeof startParallelSessionTitleGeneration> | null = null;
   let parallelSessionTitleFinished = false;
+  let creditPolicyRevision: number | undefined;
+  let titleCreditPolicyRevision: number | undefined;
   const finishParallelSessionTitle = async (): Promise<void> => {
     if (parallelSessionTitleFinished) return;
     parallelSessionTitleFinished = true;
@@ -392,6 +396,7 @@ export async function runTurnStreamAttempt(
     if (generated.usage) {
       await processSessionTitleModelUsageEvent({
         usage: generated.usage,
+        creditPolicyRevision: titleCreditPolicyRevision,
         state: sessionTitleUsageState,
         dispatchId: modelUsageDispatchId,
         settings,
@@ -594,7 +599,8 @@ export async function runTurnStreamAttempt(
   let finalReplyNudged = false;
   const revalidateModelCallAdmission = async () => {
     await historySink.reconcileConversationTruth({ requireDurable: true });
-    await ensureRunAllowedBetweenModelCalls({
+    creditPolicyRevision = await ensureRunAllowedBetweenModelCalls({
+      modelId: resolvedModel?.configured.id ?? turn.model,
       settings,
       db,
       accountId: input.accountId,
@@ -624,6 +630,7 @@ export async function runTurnStreamAttempt(
       signal: runtimeCancellationSignal,
       admit: revalidateModelCallAdmission,
     });
+    const responseCreditPolicyRevisions = new Set<number | undefined>();
     const responseCountBeforeStream = modelResponseState.responseCount;
     eventing.batcher = null;
     // The SDK emits every processed call item for one model response before
@@ -841,7 +848,10 @@ export async function runTurnStreamAttempt(
         attempt.modelRequestStarted = true;
         return await runtime.runStream(agent, runInput!, eventing.modelRunSettings, {
           beforeModelRequest: modelCallAdmission.beforeModelRequest,
-          onModelResponse: modelCallAdmission.onModelResponse,
+          onModelResponse: (event) => {
+            responseCreditPolicyRevisions.add(creditPolicyRevision);
+            return modelCallAdmission.onModelResponse(event);
+          },
           signal: runtimeCancellationSignal,
           sandboxEnvironment,
           onModelVisibleContext: async (snapshot) => {
@@ -1067,6 +1077,7 @@ export async function runTurnStreamAttempt(
           ? await media.retainNativeGeneratedImage(generatedImage)
           : null;
         const responseResult = await processModelResponseTerminalEvent({
+          creditPolicyRevision,
           event: next.value,
           state: modelResponseState,
           dispatchId: modelUsageDispatchId,
@@ -1132,6 +1143,15 @@ export async function runTurnStreamAttempt(
             ]);
             attempt.providerRecoveryCount = 0;
           }
+          if (attempt.providerRecoveryObservation) {
+            recordProviderRecoveryOutcome(observability, {
+              route: attempt.modelMetricRoute,
+              cause: attempt.providerRecoveryObservation.cause,
+              outcome: "recovered",
+              elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt,
+            });
+            attempt.providerRecoveryObservation = undefined;
+          }
           const rawStreamHistory = (eventing.stream.state as { history?: unknown[] }).history;
           if (Array.isArray(rawStreamHistory)) {
             // The completed image item is normally retained from its own
@@ -1148,21 +1168,9 @@ export async function runTurnStreamAttempt(
           await historySink.reconcileConversationTruth();
           turnLifecycleMetricsFor(observability).progress({ attemptId: input.attemptId });
           modelCheckpointMemoryCollector.schedule(observability);
-          await ensureRunAllowedBetweenModelCalls({
-            settings,
-            db,
-            accountId: input.accountId,
-            workspaceId: input.workspaceId,
-            isExternallyBilledTurn: billingState.isExternallyBilledTurn,
-            entitlements,
-            chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
-            countsTowardTokenCap: billingState.countsTowardTokenCap,
-            initiatingHumanSubjectId: turn.initiatingHumanSubjectId,
-            serializedRunState: () =>
-              media.compactMediaRunState(String(eventing.stream!.state.toString())),
-          });
         }
-        // Release only after both the debit and frozen-human admission finish.
+        // Release after settlement. The producer checks fresh admission before
+        // its next request; a final response keeps the revision it ran under.
         modelCallAdmission.settle(next.value);
         const durableSdkEvent = generatedImageReceipt
           ? compactGeneratedImageSdkEvent(next.value, generatedImageReceipt)
@@ -1551,6 +1559,12 @@ export async function runTurnStreamAttempt(
     if (!streamSawPerResponseUsage) {
       const aggregateUsage = eventing.stream.state.usage;
       const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
+      const aggregatePolicyRevision = aggregateCreditPolicyRevision({
+        responseRevisions: responseCreditPolicyRevisions,
+        lastAdmittedRevision: creditPolicyRevision,
+        chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
+        totalTokens: normalizedAggregateUsage.totalTokens,
+      });
       const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
       const aggregateSourceKey = modelUsageSourceKey({
         responseId: null,
@@ -1573,6 +1587,7 @@ export async function runTurnStreamAttempt(
           leaseLostMessage: "Provider credential lease expired during the active turn",
           recordUsage: async () => {
             const billing = await recordModelUsageAndDebitCredits(settings, db, {
+              creditPolicyRevision: aggregatePolicyRevision,
               accountId: input.accountId,
               workspaceId: input.workspaceId,
               sessionId: input.sessionId,
@@ -1965,6 +1980,7 @@ export async function runTurnStreamAttempt(
       turnExecutionPolicy.providerId,
       turnExecutionPolicy.latencyMode,
     );
+    titleCreditPolicyRevision = creditPolicyRevision;
     parallelSessionTitle = startParallelSessionTitleGeneration({
       signal: runtimeCancellationSignal,
       generate: async (signal) =>

package/src/activities/agent-turn/tool-environment.ts

@@ -82,6 +82,7 @@ import { createTurnMediaArtifacts } from "./media-artifacts";
 import { SandboxChannelAService } from "@opengeni/runtime/sandbox";
 import { sandboxRunAs } from "@opengeni/runtime";
 import {
+  bundledSkillSelectionForAgentConfig,
   DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
   resolveAgentToolFamilies,
   type ResourceRef,
@@ -596,7 +597,11 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
     session.firstPartyMcpPermissions,
     linkedAuthority,
   );
-  const toolFamilies = resolveAgentToolFamilies(session.agent);
+  // Background-command tools need compute: the effective route of this turn,
+  // a managed sandbox or an attached Connected Machine, not the durable home.
+  const toolFamilies = resolveAgentToolFamilies(session.agent, {
+    sandboxAttached: (activeSandboxBackend ?? groupBoxBackend) !== "none",
+  });
   const selectedFirstPartyMcpTools = toolFamilies.firstPartyTools(
     allowedFirstPartyMcpToolsForSession(runSettings, session.firstPartyMcpTools),
   );
@@ -669,7 +674,9 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
     }),
   ]);
   const bundledSkills = loadConfiguredBundledSkills({
-    bundledSkillIds: session.bundledSkillIds,
+    // Rows that could not freeze the "none" default at create (scheduled
+    // generated sessions, pre-existing rows) get the same rule here.
+    bundledSkillIds: bundledSkillSelectionForAgentConfig(session.bundledSkillIds, session.agent),
     firstPartyTools: selectedFirstPartyMcpTools,
     videoGenerationEnabled:
       skillConfiguration.defaultModelId !== null && skillConfiguration.enabledModelIds.length > 0,

package/src/activities/agent-turn/turn-context.ts

@@ -68,6 +68,10 @@ export type AttemptIdentityState = {
   triggerEventId: string | undefined;
   executionGeneration: number;
   providerRecoveryCount: number;
+  providerRecoveryObservation?:
+    | import("./provider-recovery-metrics").ProviderRecoveryObservation
+    | undefined;
+  modelMetricRoute?: { provider: string; model: string };
   claudeAuthRecovery?: { credentialId: string; credentialVersion: number } | undefined;
   modelRequestStarted: boolean;
   redispatchesAtDispatch: number;

package/src/activities/goals.ts

@@ -1,27 +1,16 @@
 import {
   allowedFirstPartyMcpToolsForSession,
-  configuredStaticUsageLimits,
-  isModelAvailableForNewSelection,
-  policyProviderIdForModel,
-  resolveModelProvider,
   resolveTurnExecutionPolicyV1,
-  withCodexCatalogProvider,
-  withXaiSubscriptionCatalogProvider,
-  WORKSPACE_GATEWAY_MODEL_ID_PREFIX,
-  WORKSPACE_OPENROUTER_MODEL_ID_PREFIX,
   type Settings,
 } from "@opengeni/config";
 import {
-  evaluateWorkspaceModelPolicy,
   mergeToolRefs,
   readTurnExecutionPolicyV1,
   type SessionGoal,
   type ToolRef,
 } from "@opengeni/contracts";
-import { isCodexBilledModel } from "@opengeni/codex";
 import {
   enqueueSessionWorkflowWakeIfRunnable,
-  getWorkspaceModelPolicy,
   getSessionGoal,
   getSessionTurn,
   materializeGoalContinuation,
@@ -34,10 +23,10 @@ import type {
 } from "./types";
 import {
   modelFundingForAdmission,
-  resolveCatalogSettings,
-  resolveWorkspaceCatalogSettings,
+  goalRunBudgetBlocked,
+  resolveGoalModelAdmission,
 } from "@opengeni/core";
-import { agentRunAdmissionDenial } from "./agent-run-admission";
+export { goalContinuationModelDecision, goalRunBudgetBlocked } from "@opengeni/core";
 import { turnCredentialRestriction } from "./agent-turn/credential-restriction";
 
 export function createGoalActivities(services: () => Promise<ControlActivityServices>) {
@@ -80,42 +69,18 @@ export function createGoalActivities(services: () => Promise<ControlActivityServ
     if (session.status === "failed" || session.status === "cancelled") {
       return { action: "none" };
     }
-    let settings = (await resolveCatalogSettings(db, catalogSourceSettings)).settings;
-    const inheritedContinuationModel = session.model;
-    let continuationModel = inheritedContinuationModel;
+    const modelDecision = await resolveGoalModelAdmission(db, catalogSourceSettings, {
+      accountId: input.accountId,
+      workspaceId: input.workspaceId,
+      model: session.model,
+      codexCompactionMode: session.codexCompactionMode,
+      latencyMode: session.latencyMode,
+    });
+    const settings = modelDecision.settings;
+    const continuationModel = modelDecision.model;
     const continuationReasoningEffort = session.reasoningEffort;
     const continuationLatencyMode = session.latencyMode;
-    const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
-    if (
-      inheritedContinuationModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) ||
-      inheritedContinuationModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) ||
-      session.model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) ||
-      session.model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX)
-    ) {
-      settings = (
-        await resolveWorkspaceCatalogSettings(db, catalogSourceSettings, {
-          accountId: input.accountId,
-          workspaceId: input.workspaceId,
-          retainedProductModelIds: [inheritedContinuationModel, session.model],
-        })
-      ).settings;
-    }
-    const modelDecision = goalContinuationModelDecision({
-      settings,
-      workspaceModelPolicy,
-      inheritedModel: inheritedContinuationModel,
-    });
-    continuationModel = modelDecision.model;
-    let modelPolicyBlocked = modelDecision.blocked;
-    // remote_v2 sessions may only continue on Codex models — refuse synthesis
-    // that would leave the portable/non-Codex path (and mixed history shapes).
-    if (
-      !modelPolicyBlocked &&
-      session.codexCompactionMode === "remote_v2" &&
-      !isCodexBilledModel(continuationModel)
-    ) {
-      modelPolicyBlocked = `session is locked to Codex remote compaction v2; model "${continuationModel}" is not a Codex subscription model`;
-    }
+    const modelPolicyBlocked = modelDecision.blocked;
     const turnExecutionPolicy = modelPolicyBlocked
       ? undefined
       : resolveTurnExecutionPolicyV1(settings, {
@@ -183,11 +148,12 @@ export function createGoalActivities(services: () => Promise<ControlActivityServ
                 initiatingHumanSubjectId: causalTurn?.initiatingHumanSubjectId ?? null,
               },
             );
+        const pausedReason = modelPolicyBlocked
+          ? modelDecision.pausedReason
+          : budgetBlocked?.pausedReason;
         return {
           budgetBlocked: modelPolicyBlocked ?? budgetBlocked?.message ?? null,
-          budgetPausedReason: modelPolicyBlocked
-            ? "limits"
-            : (budgetBlocked?.pausedReason ?? "limits"),
+          ...(pausedReason ? { budgetPausedReason: pausedReason } : {}),
         };
       },
       policy: continuationPolicy,
@@ -220,47 +186,6 @@ export function createGoalActivities(services: () => Promise<ControlActivityServ
   };
 }
 
-export function goalContinuationModelDecision(input: {
-  settings: Settings;
-  workspaceModelPolicy: Awaited<ReturnType<typeof getWorkspaceModelPolicy>>;
-  inheritedModel: string;
-}): { model: string; blocked: string | null } {
-  const catalogSettings = input.settings.supergrokSubscriptionEnabled
-    ? withXaiSubscriptionCatalogProvider(
-        input.settings.codexSubscriptionEnabled
-          ? withCodexCatalogProvider(input.settings)
-          : input.settings,
-      )
-    : input.settings.codexSubscriptionEnabled
-      ? withCodexCatalogProvider(input.settings)
-      : input.settings;
-  const policyBlocks = (modelId: string): boolean =>
-    input.workspaceModelPolicy !== null &&
-    !evaluateWorkspaceModelPolicy(input.workspaceModelPolicy, {
-      providerId: policyProviderIdForModel(catalogSettings, modelId),
-      modelId,
-    }).allowed;
-  if (!resolveModelProvider(catalogSettings, input.inheritedModel)) {
-    return {
-      model: input.inheritedModel,
-      blocked: `model "${input.inheritedModel}" is no longer in the deployment or workspace catalog; choose an available model before resuming the goal`,
-    };
-  }
-  if (!isModelAvailableForNewSelection(catalogSettings, input.inheritedModel)) {
-    return {
-      model: input.inheritedModel,
-      blocked: `model "${input.inheritedModel}" is retired from new selection; choose an available model before resuming the goal`,
-    };
-  }
-  if (!policyBlocks(input.inheritedModel)) {
-    return { model: input.inheritedModel, blocked: null };
-  }
-  return {
-    model: input.inheritedModel,
-    blocked: `workspace model policy blocks model "${input.inheritedModel}"; pick an allowed model or change the workspace model policy`,
-  };
-}
-
 export function goalContinuationFundedWithoutCredits(
   settings: Settings,
   model: string,
@@ -364,24 +289,3 @@ export function withFirstPartyTools(settings: Settings, tools: ToolRef[]): ToolR
   }
   return mergeToolRefs(tools, [{ kind: "mcp", id: "opengeni" }]);
 }
-
-/**
- * Goals share scheduled admission and pause visibly without synthesizing work.
- */
-export async function goalRunBudgetBlocked(
-  services: Parameters<typeof agentRunAdmissionDenial>[0],
-  input: Omit<Parameters<typeof agentRunAdmissionDenial>[1], "requestedAgentRuns">,
-): Promise<{ pausedReason: "limits" | "allowance"; message: string } | null> {
-  const denial = await agentRunAdmissionDenial(services, { ...input, requestedAgentRuns: 1 });
-  if (denial === null) return null;
-  if (denial === "allowance_exhausted") {
-    return { pausedReason: "allowance", message: "Opengeni usage allowance exhausted" };
-  }
-  const limits = configuredStaticUsageLimits(services.settings);
-  const messages = {
-    insufficient_credits: "insufficient Opengeni credits",
-    monthly_model_cost_limit: `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`,
-    monthly_agent_run_limit: `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`,
-  };
-  return { pausedReason: "limits", message: messages[denial] };
-}

package/src/activities/knowledge-indexing.ts

@@ -10,7 +10,7 @@ import {
   creditDebitAttributionMetadata,
   deferKnowledgeIndexJob,
   freezeKnowledgeIndexBillingMode,
-  getBillingBalance,
+  getSpendableCreditBalance,
   guardPaidKnowledgeIndexPublication,
   knowledgeIndexBillingActivationTime,
   readKnowledgeIndexSource,
@@ -211,7 +211,7 @@ export function createKnowledgeIndexingActivities(
                 return;
               }
               if (paid && current.nextIndex === 0) {
-                const balance = await getBillingBalance(lockedDb, claim.accountId);
+                const balance = await getSpendableCreditBalance(lockedDb, claim.accountId);
                 if (balance.balanceMicros <= 0) {
                   await waitKnowledgeIndexForFunding(lockedDb, claim);
                   result.deferred++;

package/src/activities/scheduled-tasks.ts

@@ -98,6 +98,7 @@ import {
   allowedFirstPartyMcpToolsForSession,
   resolveFirstPartyMcpToolPolicy,
   resolveTurnExecutionPolicyV1,
+  TurnExecutionPolicyModelUnavailableError,
 } from "@opengeni/config";
 import { Context } from "@temporalio/activity";
 import { createHash } from "node:crypto";
@@ -1082,8 +1083,9 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
             }
           }
         : undefined;
-      const turnExecutionPolicy = scheduledTaskRunExecutionPolicy(
-        resolveTurnExecutionPolicyV1(settings, {
+      let acceptedTurnExecutionPolicy: ReturnType<typeof resolveTurnExecutionPolicyV1>;
+      try {
+        acceptedTurnExecutionPolicy = resolveTurnExecutionPolicyV1(settings, {
           modelId: acceptedModel,
           requestedModelId:
             generatedTarget && task.agentConfig.model ? task.agentConfig.model : null,
@@ -1100,7 +1102,20 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
             : "session",
           latencyMode: acceptedLatencyMode,
           latencyModeSource: generatedTarget ? "deployment" : "session",
-        }),
+        });
+      } catch (error) {
+        // A retired or removed model refuses every occurrence until the task
+        // (or its target session) names an available model: record that as a
+        // visible terminal run instead of exhausting activity retries.
+        if (!(error instanceof TurnExecutionPolicyModelUnavailableError)) throw error;
+        return await refuseAdmission(
+          "scheduled_model_unavailable",
+          false,
+          `${error.message}: ${acceptedModel}`,
+        );
+      }
+      const turnExecutionPolicy = scheduledTaskRunExecutionPolicy(
+        acceptedTurnExecutionPolicy,
         input,
         creatorPolicy?.credentialRestriction,
       );

package/src/activities/types.ts

@@ -500,6 +500,7 @@ export type DispatchScheduledTaskRunResult =
         | "machine_enrollment_inactive"
         | "variable_set_unavailable"
         | "rig_version_unavailable"
+        | "scheduled_model_unavailable"
         | "knowledge_source_paused"
         | "legacy_source_schedule_requires_migration"
         | "atlassian_native_retired"

package/src/index.ts

@@ -1214,8 +1214,6 @@ export async function startWorker() {
     rlsStrategy: settings.rlsStrategy,
     expectedRole: settings.runtimeDatabaseRole,
     targetSchema: settings.dbSchema.trim() || "public",
-    organizationTenancyCanonicalActivationEnabled:
-      settings.organizationTenancyCanonicalActivationEnabled,
   } as const;
   const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
   let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;

package/src/observability-metrics.ts

@@ -1308,6 +1308,7 @@ export function recordSandboxDeadlineRotationsRequested(
 export const SANDBOX_COMMAND_CONTAINMENT_OUTCOMES = [
   "idle_enrolled",
   "deadline_enrolled",
+  "quiescence_enrolled",
   "resumed_enrolled",
   "not_eligible",
   "inspection_failed",