@opengeni/worker-bundle 1.0.0-canary.1 → 1.0.0

package/src/activities/agent-turn/model-usage.ts

@@ -16,7 +16,9 @@ import {
 import {
   calculateGatewayReportedCostBreakdown,
   calculateGatewayReportedProviderCostMicros,
+  calculateModelListUsageCostSnapshot,
   calculateModelUsageCostBreakdown,
+  configuredModelListPricingSchedules,
   configuredModelPricingSchedules,
   resolveModelProvider,
   responseSatisfiesLatencyMode,
@@ -39,6 +41,7 @@ import {
   type ModelContextContributionSummary,
   type SessionEvent,
 } from "@opengeni/contracts";
+import type { InsightsUsageClassMicros } from "@opengeni/contracts/insights-usage";
 import { safeErrorDiagnostic } from "./errors";
 
 export function modelUsageSourceKey(input: {
@@ -293,6 +296,7 @@ export async function processModelResponseTerminalEvent(input: {
         usage: responseUsage,
         normalizedUsage,
         ...(billing ? { billingPath: billing.billingPath } : {}),
+        ...(billing ? { billingSnapshot: billing } : {}),
         ...(billing?.upstreamProvider ? { upstreamProvider: billing.upstreamProvider } : {}),
         servingAccountHash: accountContext.servingAccountHash,
         accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
@@ -439,6 +443,7 @@ export async function processCompactionModelUsageEvent(input: {
         usage: input.usage,
         normalizedUsage,
         ...(billing ? { billingPath: billing.billingPath } : {}),
+        ...(billing ? { billingSnapshot: billing } : {}),
         ...(billing?.upstreamProvider ? { upstreamProvider: billing.upstreamProvider } : {}),
         servingAccountHash: accountContext.servingAccountHash,
         accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
@@ -492,6 +497,16 @@ export async function emitModelCallUsage(input: {
   normalizedUsage?: ModelCallUsageNormalization;
   /** Accepted billing authority persisted for Insights repair after a soft fact-write failure. */
   billingPath?: ModelUsageBillingRecord["billingPath"];
+  /** Accepted comparisons are frozen with the event, not repriced during fact repair. */
+  billingSnapshot?: Pick<
+    ModelUsageBillingRecord,
+    | "pricedCostMicros"
+    | "estimatedProviderCostMicros"
+    | "equivalentCreditCostMicros"
+    | "pricingSource"
+    | "listByClassMicros"
+    | "listByClassApprox"
+  >;
   /** Validated Gateway endpoint provider persisted for exact Insights repair. */
   upstreamProvider?: string;
   // Prompt-cache research dimensions (log-only; NEVER on a metric label or a
@@ -527,6 +542,16 @@ export async function emitModelCallUsage(input: {
           sourceKey: input.sourceKey,
           ...(input.billingPath ? { billingPath: input.billingPath } : {}),
           ...(input.upstreamProvider ? { upstreamProvider: input.upstreamProvider } : {}),
+          ...(input.billingSnapshot
+            ? {
+                pricedCostMicros: input.billingSnapshot.pricedCostMicros,
+                estimatedProviderCostMicros: input.billingSnapshot.estimatedProviderCostMicros,
+                equivalentCreditCostMicros: input.billingSnapshot.equivalentCreditCostMicros,
+                pricingSource: input.billingSnapshot.pricingSource,
+                listByClassMicros: input.billingSnapshot.listByClassMicros ?? null,
+                listByClassApprox: input.billingSnapshot.listByClassApprox ?? false,
+              }
+            : {}),
           ...telemetry,
         },
       },
@@ -605,6 +630,9 @@ export type ModelUsageBillingRecord = {
   /** Hypothetical OpenGeni credit price at the captured rate; never a debit. */
   equivalentCreditCostMicros: number | null;
   pricingSource: "configured_list_price" | "gateway_reported" | null;
+  /** Forward-only provider list class snapshot; older facts/events stay unknown. */
+  listByClassMicros?: InsightsUsageClassMicros | null;
+  listByClassApprox?: boolean;
   normalizedUsage: ModelCallUsageNormalization;
   upstreamProvider?: string;
 };
@@ -665,7 +693,7 @@ export async function recordModelUsageAndDebitCredits(
     }
     if (unpinnedWorkspaceGatewayModel && chargesOpenGeniCredits) {
       throw new Error(
-        `Workspace Gateway custom model ${input.model} cannot charge OpenGeni credits without pinned pricing`,
+        `Workspace Gateway custom model ${input.model} cannot charge Opengeni credits without pinned pricing`,
       );
     }
   }
@@ -697,10 +725,30 @@ export async function recordModelUsageAndDebitCredits(
   const hasCompleteCoreTokenTelemetry =
     normalizedUsage.telemetry.inputTokens !== null &&
     normalizedUsage.telemetry.outputTokens !== null;
+  // Comparison rates are deliberately separate from debit authority. The
+  // current usage frame does not establish geography/service-tier provenance,
+  // so forward class splits stay unknown even when a total estimate is priced.
+  const listPricingSchedules = configuredModelListPricingSchedules(settings);
+  const configuredListPricingModel = listPricingSchedules[input.model]
+    ? input.model
+    : input.model.startsWith("codex/") && listPricingSchedules[input.model.slice("codex/".length)]
+      ? input.model.slice("codex/".length)
+      : null;
+  const listSnapshot =
+    !gatewayBilling && hasCompleteCoreTokenTelemetry && configuredListPricingModel
+      ? calculateModelListUsageCostSnapshot(settings, configuredListPricingModel, sanitizedUsage, {
+          latencyMode: input.latencyMode ?? "standard",
+          priceContextKnown: false,
+        })
+      : null;
+  const listClasses = {
+    listByClassMicros: listSnapshot?.listByClassMicros ?? null,
+    listByClassApprox: listSnapshot?.listByClassApprox ?? false,
+  };
   const estimatedProviderCostMicros = gatewayBilling
     ? (pricingBreakdown?.providerCostMicros ?? null)
     : hasCompleteCoreTokenTelemetry
-      ? (pricingBreakdown?.providerCostMicros ?? null)
+      ? (listSnapshot?.providerCostMicros ?? pricingBreakdown?.providerCostMicros ?? null)
       : null;
   const equivalentCreditCostMicros =
     pricingBreakdown && !unpinnedWorkspaceGatewayModel
@@ -753,6 +801,7 @@ export async function recordModelUsageAndDebitCredits(
       estimatedProviderCostMicros,
       equivalentCreditCostMicros,
       pricingSource,
+      ...listClasses,
       normalizedUsage,
       ...(gatewayBilling ? { upstreamProvider: gatewayBilling.finalProvider } : {}),
     };
@@ -765,6 +814,7 @@ export async function recordModelUsageAndDebitCredits(
       estimatedProviderCostMicros,
       equivalentCreditCostMicros,
       pricingSource,
+      ...listClasses,
       normalizedUsage,
       ...(gatewayBilling ? { upstreamProvider: gatewayBilling.finalProvider } : {}),
     };
@@ -819,6 +869,7 @@ export async function recordModelUsageAndDebitCredits(
     estimatedProviderCostMicros,
     equivalentCreditCostMicros,
     pricingSource,
+    ...listClasses,
     normalizedUsage,
     ...(gatewayBilling ? { upstreamProvider: gatewayBilling.finalProvider } : {}),
   };
@@ -857,6 +908,8 @@ export async function recordAuthoritativeModelCallFact(input: {
       estimatedProviderCostMicros: input.billing.estimatedProviderCostMicros,
       equivalentCreditCostMicros: input.billing.equivalentCreditCostMicros,
       pricingSource: input.billing.pricingSource,
+      listByClassMicros: input.billing.listByClassMicros ?? null,
+      listByClassApprox: input.billing.listByClassApprox ?? false,
       inputTokens: telemetry.inputTokens,
       outputTokens: telemetry.outputTokens,
       cachedTokens: telemetry.cachedTokens,
@@ -883,7 +936,9 @@ export function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization
       ? { outputTokens: normalized.telemetry.outputTokens }
       : {}),
     ...(normalized.totalTokens !== null ? { totalTokens: normalized.totalTokens } : {}),
-    ...(normalized.telemetry.cachedTokens !== null || normalized.telemetry.cacheWriteTokens !== null
+    ...(normalized.telemetry.cachedTokens !== null ||
+    normalized.telemetry.cacheWriteTokens !== null ||
+    normalized.cacheWriteTokensByTtl !== undefined
       ? {
           inputTokensDetails: {
             ...(normalized.telemetry.cachedTokens === null
@@ -892,6 +947,12 @@ export function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization
             ...(normalized.telemetry.cacheWriteTokens === null
               ? {}
               : { cache_write_tokens: normalized.telemetry.cacheWriteTokens }),
+            ...(normalized.cacheWriteTokensByTtl?.fiveMinute == null
+              ? {}
+              : { cache_write_tokens_5m: normalized.cacheWriteTokensByTtl.fiveMinute }),
+            ...(normalized.cacheWriteTokensByTtl?.oneHour == null
+              ? {}
+              : { cache_write_tokens_1h: normalized.cacheWriteTokensByTtl.oneHour }),
           },
         }
       : {}),

package/src/activities/agent-turn/model-wire-dispatch.ts

@@ -0,0 +1,27 @@
+import type { ModelTransportDispatchClock } from "@opengeni/runtime";
+
+type DispatchIdentity = { provider: string; dispatchId: string };
+
+/** One immutable physical fetch-entry observation per worker attempt. Generic
+ * fallback events already carry that attempt's dispatch/execution identity. */
+export class InitialModelWireDispatchClock {
+  private initial: (DispatchIdentity & { initialWireDispatchedAt: string }) | null = null;
+
+  record(identity: DispatchIdentity, clock: ModelTransportDispatchClock): void {
+    if (this.initial !== null) return;
+    const timestamp = new Date(clock.dispatchedAtUnixMs);
+    if (!Number.isFinite(timestamp.getTime())) return;
+    this.initial = { ...identity, initialWireDispatchedAt: timestamp.toISOString() };
+  }
+
+  payload(identity: DispatchIdentity): { initialWireDispatchedAt?: string } {
+    if (
+      this.initial === null ||
+      this.initial.provider !== identity.provider ||
+      this.initial.dispatchId !== identity.dispatchId
+    ) {
+      return {};
+    }
+    return { initialWireDispatchedAt: this.initial.initialWireDispatchedAt };
+  }
+}

package/src/activities/agent-turn/optional-repositories.ts

@@ -0,0 +1,224 @@
+import type { Settings } from "@opengeni/config";
+import { resourceMountPath, type ResourceRef } from "@opengeni/contracts";
+import { prReviewRegistrationIdFromCredentialBinding } from "@opengeni/core";
+import { areGitHubRepositoriesAllowedForWorkspace, type Database } from "@opengeni/db";
+import {
+  findInaccessibleGitHubAppInstallationRepositories,
+  githubAppMissingSettings,
+} from "@opengeni/github";
+import { gitHubTokenAuthorizationSelections } from "../environment";
+
+/**
+ * Turn-time access check for automatically attached (`optional: true`) GitHub
+ * App repositories.
+ *
+ * A Slack-started session attaches the caller's recently used repositories as
+ * optional resources. The per-turn workspace-allowlist recheck and the
+ * installation-token mint both cover every GitHub repository of the session in
+ * one request, so one optional repository that loses access after the session
+ * started (an admin removes it from the allowlist, or it is deleted or removed
+ * from the App on GitHub) would fail every later turn. Before those checks, this
+ * drops each optional repository that is no longer allowlisted or no longer
+ * reachable through its installation, for this turn only, and reports it the
+ * same way a skipped optional clone is reported. It only ever removes
+ * repositories: a remaining repository still passes the unchanged strict
+ * allowlist recheck and mint, and an explicitly attached repository is never
+ * dropped, so it keeps failing the turn when its access is gone.
+ */
+
+export const OPTIONAL_REPOSITORY_ACCESS_OPERATION = "optional-repository-access";
+
+type OptionalRepositoryCandidate = {
+  resource: Extract<ResourceRef, { kind: "repository" }>;
+  installationId: number;
+  repositoryId: number;
+};
+
+export type OptionalRepositoryDropReason = "not_allowlisted" | "inaccessible" | "unverified";
+
+export type DropUnavailableOptionalRepositoriesInput = {
+  db: Database;
+  settings: Settings;
+  workspaceId: string;
+  activeSandboxBackend: Settings["sandboxBackend"] | undefined;
+  /**
+   * True when a host `gitCredentials` port mints the Git tokens: the host may
+   * use another GitHub App, so this deployment's App is not asked.
+   */
+  hostMintsGitCredentials: boolean;
+  turnResources: readonly ResourceRef[];
+  runtimeResources: readonly ResourceRef[];
+  /** Publishes one attempt-fenced session event batch. */
+  publish: (
+    events: Array<{
+      type: "sandbox.operation.completed";
+      payload: {
+        name: typeof OPTIONAL_REPOSITORY_ACCESS_OPERATION;
+        repositoryCount: number;
+        skippedOptionalRepositories: string[];
+      };
+    }>,
+  ) => Promise<void>;
+  /** Operational log line; counts only, never repository names. */
+  warn: (message: string, fields: Record<string, string>) => void;
+  /** Test seam; defaults to the workspace allowlist under RLS. */
+  isAllowlisted?: (installationId: number, repositoryId: number) => Promise<boolean>;
+  /** Test seam; defaults to a live GitHub App token-mint probe. */
+  findInaccessible?: (installationId: number, repositoryIds: number[]) => Promise<number[]>;
+};
+
+function optionalGitHubAppRepositoryCandidate(
+  resource: ResourceRef,
+): OptionalRepositoryCandidate | null {
+  if (resource.kind !== "repository" || resource.optional !== true) return null;
+  let selections: ReturnType<typeof gitHubTokenAuthorizationSelections>;
+  try {
+    // The exact selection the allowlist recheck and the mint derive, so the
+    // decision here covers precisely what those strict steps would check.
+    selections = gitHubTokenAuthorizationSelections([resource]);
+  } catch {
+    return null;
+  }
+  const [selection] = selections;
+  if (!selection || selections.length !== 1 || selection.repositoryIds.length !== 1) return null;
+  // PR-review bindings are authorized by their own registration, not the
+  // workspace allowlist; they are never automatically attached.
+  if (prReviewRegistrationIdFromCredentialBinding(selection.credentialBindingId)) return null;
+  return {
+    resource,
+    installationId: selection.installationId,
+    repositoryId: selection.repositoryIds[0]!,
+  };
+}
+
+function repositoryIdentity(resource: ResourceRef): string | null {
+  return resource.kind === "repository"
+    ? `${resource.uri}\u0000${resourceMountPath(resource)}`
+    : null;
+}
+
+export async function dropUnavailableOptionalRepositories(
+  input: DropUnavailableOptionalRepositoriesInput,
+): Promise<{
+  turnResources: ResourceRef[];
+  runtimeResources: ResourceRef[];
+  /**
+   * The same decision for another view of this turn's resources (the GitHub
+   * REST tool surface is built from the unbound session resources).
+   */
+  retainsResource: (resource: ResourceRef) => boolean;
+}> {
+  const passthrough = {
+    turnResources: [...input.turnResources],
+    runtimeResources: [...input.runtimeResources],
+    retainsResource: () => true,
+  };
+  // A Connected Machine receives no platform Git credential and runs neither
+  // the allowlist recheck nor the mint, so nothing can fail there.
+  if (input.activeSandboxBackend === "selfhosted") return passthrough;
+  const candidates = input.turnResources.flatMap(
+    (resource) => optionalGitHubAppRepositoryCandidate(resource) ?? [],
+  );
+  if (candidates.length === 0) return passthrough;
+
+  const isAllowlisted =
+    input.isAllowlisted ??
+    ((installationId: number, repositoryId: number) =>
+      areGitHubRepositoriesAllowedForWorkspace(input.db, input.workspaceId, installationId, [
+        repositoryId,
+      ]));
+  const dropped = new Map<OptionalRepositoryCandidate, OptionalRepositoryDropReason>();
+  const allowlisted: OptionalRepositoryCandidate[] = [];
+  const admitted = await Promise.all(
+    candidates.map(async (candidate) => {
+      try {
+        return (await isAllowlisted(candidate.installationId, candidate.repositoryId))
+          ? ("allowlisted" as const)
+          : ("not_allowlisted" as const);
+      } catch {
+        // The check itself failed: an optional repository must not fail the
+        // turn, so it sits this turn out like an unverifiable GitHub answer.
+        return "unverified" as const;
+      }
+    }),
+  );
+  candidates.forEach((candidate, index) => {
+    const outcome = admitted[index]!;
+    if (outcome === "allowlisted") allowlisted.push(candidate);
+    else dropped.set(candidate, outcome);
+  });
+
+  const findInaccessible =
+    input.findInaccessible ??
+    (!input.hostMintsGitCredentials && githubAppMissingSettings(input.settings).length === 0
+      ? (installationId: number, repositoryIds: number[]) =>
+          findInaccessibleGitHubAppInstallationRepositories(input.settings, {
+            installationId,
+            repositoryIds,
+          })
+      : null);
+  if (findInaccessible && allowlisted.length > 0) {
+    const byInstallation = new Map<number, OptionalRepositoryCandidate[]>();
+    for (const candidate of allowlisted) {
+      const group = byInstallation.get(candidate.installationId) ?? [];
+      group.push(candidate);
+      byInstallation.set(candidate.installationId, group);
+    }
+    await Promise.all(
+      [...byInstallation].map(async ([installationId, group]) => {
+        try {
+          const inaccessible = new Set(
+            await findInaccessible(
+              installationId,
+              group.map((candidate) => candidate.repositoryId),
+            ),
+          );
+          for (const candidate of group) {
+            if (inaccessible.has(candidate.repositoryId)) dropped.set(candidate, "inaccessible");
+          }
+        } catch {
+          // GitHub could not answer. An optional repository must never fail
+          // the turn, and keeping it would put it into a mint GitHub may then
+          // refuse, so it sits this turn out and is checked again next turn.
+          for (const candidate of group) dropped.set(candidate, "unverified");
+        }
+      }),
+    );
+  }
+  if (dropped.size === 0) return passthrough;
+
+  const droppedIdentities = new Set(
+    [...dropped.keys()].flatMap((candidate) => repositoryIdentity(candidate.resource) ?? []),
+  );
+  const keep = (resource: ResourceRef): boolean => {
+    if (resource.kind !== "repository" || resource.optional !== true) return true;
+    const identity = repositoryIdentity(resource);
+    return identity === null || !droppedIdentities.has(identity);
+  };
+  const reasons = [...dropped.values()];
+  input.warn("optional repository resources were dropped for this turn", {
+    droppedCount: String(dropped.size),
+    notAllowlistedCount: String(reasons.filter((reason) => reason === "not_allowlisted").length),
+    inaccessibleCount: String(reasons.filter((reason) => reason === "inaccessible").length),
+    unverifiedCount: String(reasons.filter((reason) => reason === "unverified").length),
+    origin: "worker",
+  });
+  await input.publish([
+    {
+      type: "sandbox.operation.completed",
+      payload: {
+        name: OPTIONAL_REPOSITORY_ACCESS_OPERATION,
+        repositoryCount: input.turnResources.filter((resource) => resource.kind === "repository")
+          .length,
+        skippedOptionalRepositories: [
+          ...new Set([...dropped.keys()].map((candidate) => resourceMountPath(candidate.resource))),
+        ].sort(),
+      },
+    },
+  ]);
+  return {
+    turnResources: input.turnResources.filter(keep),
+    runtimeResources: input.runtimeResources.filter(keep),
+    retainsResource: keep,
+  };
+}

package/src/activities/agent-turn/provider-dispatch-barrier.ts

@@ -1,4 +1,14 @@
 import type { TurnHistorySink } from "./history-sink";
+import type { Settings } from "@opengeni/config";
+
+export class SandboxCapabilitiesChangedError extends Error {
+  readonly code = "native_capabilities_changed_this_attempt";
+
+  constructor() {
+    super("The attached sandbox requires rebuilding this attempt's native capabilities.");
+    this.name = "SandboxCapabilitiesChangedError";
+  }
+}
 
 /**
  * Persist the SDK's complete prior model/tool history before another provider
@@ -8,6 +18,18 @@ import type { TurnHistorySink } from "./history-sink";
  */
 export async function checkpointHistoryBeforeProviderDispatch(
   historySink: Pick<TurnHistorySink, "reconcileConversationTruth">,
+  route?: {
+    effectiveSandboxBackend: Settings["sandboxBackend"];
+    routingEnabled: boolean;
+    readActiveSandbox: () => Promise<{ activeSandboxId: string | null } | null>;
+  },
 ): Promise<void> {
   await historySink.reconcileConversationTruth({ requireDurable: true });
+  // A plain Agent cannot acquire SandboxAgent capabilities in place. Preserve
+  // its completed tool batch before recovering the same logical turn; the next
+  // attempt resolves and authorizes the route normally.
+  if (route?.effectiveSandboxBackend === "none" && route.routingEnabled) {
+    const active = await route.readActiveSandbox();
+    if (active?.activeSandboxId) throw new SandboxCapabilitiesChangedError();
+  }
 }

package/src/activities/agent-turn/quiescence.ts

@@ -91,10 +91,9 @@ export const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
 export const TURN_QUIESCENCE_WATCHDOG_MS = 5 * 60_000;
 export const TURN_FINALIZER_STEP_TIMEOUT_MS = 30_000;
 
-/** A fenced activity must never heartbeat forever while its physical writer
- * drain is stuck. The last-resort worker exit stops every in-process writer;
- * Kubernetes replaces the pod and the workflow's heartbeat-lease reconciler
- * then commits the exact quiescence receipt. */
+/** A fenced activity must never heartbeat forever while physical cleanup is
+ * stuck. The callback requests host-owned containment, which drains peer turns
+ * before applying its final exit policy. A timeout is never quiescence proof. */
 export function armTurnQuiescenceWatchdog(input: {
   enabled: boolean;
   timeoutMs?: number;
@@ -198,13 +197,16 @@ export async function drainAttemptOwnedSandboxWriters(input: {
   gitCredentialRenewals: readonly Pick<GitCredentialRenewalController, "stop">[];
   codemodeTokenRenewal: Pick<CodemodeTokenRenewalController, "stop"> | null;
   runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
+  onStage?: (stage: "tool_writers" | "credential_renewals") => void;
 }): Promise<void> {
+  input.onStage?.("tool_writers");
   if (input.toolCancellationFence) {
     input.toolCancellationFence.cancel(
       input.cancellationReason ?? new Error("TURN_ATTEMPT_FENCED"),
     );
     await input.toolCancellationFence.waitForQuiescence();
   }
+  input.onStage?.("credential_renewals");
   await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
   await input.codemodeTokenRenewal?.stop();
   await input.runCredentialRenewal?.stop();
@@ -432,13 +434,14 @@ export async function runMandatoryHistoryPersistenceStep<T>(
 }
 
 export type OpStreamFinalizer = {
-  finalizeOpStreamOps(): Promise<void>;
+  finalizeOpStreamOps(toolCallIds?: readonly string[]): Promise<void>;
 };
 
-/** Release retained Connected Machine output only after the turn is durable. */
+/** Release only durably recorded tool results, or the complete durable turn. */
 export async function finalizeDurableTurnOpStreams(
   sessions: readonly unknown[],
   fallback: OpStreamFinalizer | null,
+  toolCallIds?: readonly string[],
 ): Promise<void> {
   const candidates = new Set<OpStreamFinalizer>();
   for (const session of sessions) {
@@ -452,7 +455,7 @@ export async function finalizeDurableTurnOpStreams(
   }
   for (const candidate of candidates) {
     try {
-      await candidate.finalizeOpStreamOps();
+      await candidate.finalizeOpStreamOps(toolCallIds);
     } catch {
       // The runner's retention TTL owns the fallback.
     }

package/src/activities/agent-turn/recovery-warning.ts

@@ -0,0 +1,16 @@
+import { getSandboxRecoveryDiscontinuity, type Database } from "@opengeni/db";
+
+/** v2 also recognizes system-selected checkpoint fallback receipts, and v3
+ * the empty-workspace continuation receipts. A worker below the version a
+ * session's receipts require cannot claim it (migrations 0526 and 0548). */
+export const FILESYSTEM_DISCONTINUITY_PROTOCOL = 3 as const;
+
+export async function recoveryAwareSessionInstructions(
+  db: Database,
+  workspaceId: string,
+  session: { id: string; instructions?: string | null },
+  readDiscontinuity: typeof getSandboxRecoveryDiscontinuity = getSandboxRecoveryDiscontinuity,
+): Promise<string> {
+  const filesystemDiscontinuity = await readDiscontinuity(db, workspaceId, session.id);
+  return [session.instructions, filesystemDiscontinuity].filter(Boolean).join("\n\n");
+}

package/src/activities/agent-turn/refresh-credentials.ts

@@ -0,0 +1,44 @@
+import type { AttemptToolDefinition } from "@opengeni/codemode";
+
+export const REFRESH_CREDENTIALS_TOOL_NAME = "refresh_credentials";
+
+export const REFRESH_CREDENTIALS_TOOL_DESCRIPTION =
+  "Fetch fresh host-managed credentials for this sandbox now. Use it when a command fails because a token or credential file provided by the workspace has expired or been revoked. The environment variables and credential files that the host manages are replaced for the next command; commands that are already running keep what they started with.";
+
+export function createRefreshCredentialsAttemptToolDefinition(input: {
+  refresh: () => Promise<"completed" | "auth_needed" | "error" | "not_provisioned">;
+}): AttemptToolDefinition {
+  return {
+    identity: { serverId: "opengeni", toolName: REFRESH_CREDENTIALS_TOOL_NAME },
+    modelName: REFRESH_CREDENTIALS_TOOL_NAME,
+    codemodePath: ["opengeni", REFRESH_CREDENTIALS_TOOL_NAME],
+    title: "Refresh credentials",
+    description: REFRESH_CREDENTIALS_TOOL_DESCRIPTION,
+    inputSchema: { type: "object", properties: {}, additionalProperties: false },
+    annotations: {
+      title: "Refresh credentials",
+      readOnlyHint: false,
+      destructiveHint: false,
+      idempotentHint: true,
+      openWorldHint: true,
+    },
+    source: "opengeni",
+    approval: "none",
+    execute: async (args) => {
+      if (Object.keys(args).length !== 0) {
+        throw new Error(`${REFRESH_CREDENTIALS_TOOL_NAME} accepts no arguments`);
+      }
+      const outcome = await input.refresh();
+      const text = {
+        completed: "Credentials refreshed. Retry the command that failed.",
+        auth_needed:
+          "The host returned credentials that need user attention (reconnect or grant access). Continue with what is available and tell the user which service needs reconnecting.",
+        error:
+          "The credential provider could not be reached. The previous credentials are still in place and a retry is scheduled.",
+        not_provisioned:
+          "Credentials are provisioned when the sandbox first starts; run a command and they will be fetched then.",
+      }[outcome];
+      return { isError: outcome === "error", content: [{ type: "text", text }] };
+    },
+  };
+}