@opengeni/worker-bundle 1.0.1 → 2.0.0

package/src/sandbox-snapshot-diagnostics.ts

@@ -0,0 +1,112 @@
+import { sandboxLeaseTelemetryKey, type Observability } from "@opengeni/observability";
+
+type SafeSnapshotError = {
+  errorClass: "SnapshotOperationError";
+  errorCode: "snapshot_operation_failed";
+  status?: number;
+  origin: "sandbox-resume" | "worker-lifecycle";
+  causeName?: string;
+  integrityCode?: string;
+  providerErrorName?: string;
+  providerGrpcCode?: number;
+  providerHttpStatus?: number;
+  providerRetryable?: boolean;
+};
+
+export function safeSnapshotError(
+  error: unknown,
+  origin: SafeSnapshotError["origin"] = "sandbox-resume",
+): SafeSnapshotError {
+  const fields: SafeSnapshotError = {
+    errorClass: "SnapshotOperationError",
+    errorCode: "snapshot_operation_failed",
+    origin,
+  };
+  try {
+    if (error && typeof error === "object") {
+      const candidate = error as {
+        name?: unknown;
+        code?: unknown;
+        status?: unknown;
+        statusCode?: unknown;
+      };
+      if (
+        typeof candidate.name === "string" &&
+        /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)
+      ) {
+        fields.causeName = candidate.name;
+      }
+      if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
+        fields.integrityCode = candidate.code;
+      }
+      const status = Number(candidate.status ?? candidate.statusCode);
+      if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
+    }
+  } catch {
+    // Diagnostics must never replace the exact internal snapshot failure.
+  }
+  // The SDK wraps provider classification in details, not Error.cause. Never
+  // project free-form messages, causes, request IDs or payloads. Read fields
+  // independently so a hostile getter cannot hide other safe classifications.
+  const read = (value: unknown, key: string): unknown => {
+    try {
+      return value && typeof value === "object" ? Reflect.get(value, key) : undefined;
+    } catch {
+      return undefined;
+    }
+  };
+  const details = read(error, "details");
+  const providerName = read(details, "errorName");
+  if (
+    typeof providerName === "string" &&
+    [
+      "ClientError",
+      "TimeoutError",
+      "ConnectionError",
+      "AuthError",
+      "NotFoundError",
+      "InvalidError",
+      "RemoteError",
+      "AbortError",
+    ].includes(providerName)
+  )
+    fields.providerErrorName = providerName;
+  const grpcCode = read(details, "errorCode");
+  if (
+    providerName === "ClientError" &&
+    typeof grpcCode === "number" &&
+    Number.isInteger(grpcCode) &&
+    grpcCode >= 0 &&
+    grpcCode <= 16
+  )
+    fields.providerGrpcCode = grpcCode;
+  for (const key of ["status", "httpStatus", "responseStatus"]) {
+    const status = read(details, key);
+    if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) {
+      fields.providerHttpStatus = status;
+      break;
+    }
+  }
+  const retryable = read(error, "retryable");
+  if (typeof retryable === "boolean") fields.providerRetryable = retryable;
+  return fields;
+}
+
+export function warnDrainSnapshotFailure(
+  observability: Pick<Observability, "warn">,
+  message: string,
+  error: unknown,
+  lease: { workspaceId?: string; sandboxGroupId: string; leaseEpoch: number },
+): void {
+  try {
+    observability.warn(message, {
+      ...safeSnapshotError(error, "worker-lifecycle"),
+      ...(lease.workspaceId
+        ? { sandboxLeaseKey: sandboxLeaseTelemetryKey(lease.workspaceId, lease.sandboxGroupId) }
+        : {}),
+      leaseEpoch: lease.leaseEpoch,
+    });
+  } catch {
+    // Logging cannot replace the provider failure or prevent claim release.
+  }
+}

package/src/warm-capture-holder.ts

@@ -0,0 +1,15 @@
+import type { LeaseSnapshot } from "@opengeni/db";
+
+/** Physical capture retention is not permission to heartbeat or execute a closed turn. */
+export function retainsHolderForWarmCapture(
+  lease: Pick<LeaseSnapshot, "liveness" | "leaseEpoch" | "archiveCapture"> | null,
+  expectedEpoch: number,
+  now = Date.now(),
+): boolean {
+  return Boolean(
+    lease?.liveness === "warm" &&
+    lease.leaseEpoch === expectedEpoch &&
+    lease.archiveCapture &&
+    lease.archiveCapture.deadlineAt.getTime() > now,
+  );
+}

package/src/workflows/session.ts

@@ -229,7 +229,15 @@ export function preClaimFailureDetail(error: unknown): PreClaimFailureDetail | u
   ) {
     return undefined;
   }
-  const detail = cause.details?.[0] as { disposition?: unknown; code?: unknown } | undefined;
+  const detail = cause.details?.[0] as
+    | {
+        disposition?: unknown;
+        code?: unknown;
+        sqlState?: unknown;
+        reason?: unknown;
+        retryPolicy?: unknown;
+      }
+    | undefined;
   if (
     detail?.code !== "db_deadlock" &&
     detail?.code !== "db_serialization_failure" &&
@@ -239,6 +247,27 @@ export function preClaimFailureDetail(error: unknown): PreClaimFailureDetail | u
     return undefined;
   }
   const disposition = detail.disposition;
+  if (disposition === "blocked") {
+    if (
+      detail.code !== "db_failure" ||
+      detail.retryPolicy !== "explicit_recheck" ||
+      (detail.reason !== "database_claim_rejected" &&
+        detail.reason !== "initiator_membership_required" &&
+        detail.reason !== "personal_resource_grant_required") ||
+      !(
+        detail.sqlState === null ||
+        (typeof detail.sqlState === "string" && /^[0-9A-Z]{5}$/.test(detail.sqlState))
+      )
+    )
+      return undefined;
+    return {
+      disposition,
+      code: detail.code,
+      reason: detail.reason,
+      sqlState: detail.sqlState,
+      retryPolicy: "explicit_recheck",
+    };
+  }
   if (disposition !== "retryable" && disposition !== "permanent") return undefined;
   if (
     (detail.code === "db_deadlock" || detail.code === "db_serialization_failure") &&
@@ -351,6 +380,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
   const preserveQuiescenceWake = patched("session-quiescence-reconciliation-wake-v1");
   const staleControlSignalIsOnlyWakeHint = patched("session-control-stale-wake-v1");
   const unclaimedAttemptRecovery = patched("session-unclaimed-attempt-recovery-v1");
+  let durableAdmissionBlocking = false;
   // PR #2208 changed a typed-cancelled result from a plain re-peek into a
   // recoverDispatch activity. Version that new command so histories which
   // already recorded the legacy re-peek remain deterministic on replay.
@@ -561,10 +591,29 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
     const closeSignalVersion = signalVersion;
     const closeNonControlSignalVersion = nonControlSignalVersion;
     const workflowId = workflowInfo().workflowId;
+    // Re-evaluate at the changed command, not only workflow entry. A replay
+    // without the marker keeps its old shape; the next live admission cycle
+    // can activate this fix without waiting for continueAsNew.
+    durableAdmissionBlocking = patched("session-durable-admission-block-v1");
+    const safeObservation = patched("session-safe-control-observation-v1");
     const peek = await activity.peekSessionWork({
       workspaceId: input.workspaceId,
       sessionId: input.sessionId,
+      ...(durableAdmissionBlocking ? { includeAdmissionFence: true } : {}),
+      ...(safeObservation ? { observerAccountId: input.accountId } : {}),
     });
+    if (peek.kind === "unavailable" || peek.kind === "attempt-owned") {
+      // No terminal/idle projection and no successor dispatch. Restoration
+      // need not produce a wake, so retain the observer with a bounded timer
+      // instead of closing and stranding durable work. Signals interrupt the
+      // wait; continue-as-new above bounds history, never business execution.
+      await condition(() => signalVersion !== closeSignalVersion, "30s");
+      continue;
+    }
+    if (peek.kind === "admission-blocked") {
+      if (signalVersion !== closeSignalVersion || pendingQuiescenceProofs.size > 0) continue;
+      return;
+    }
     if (peek.kind === "interruption-pending") {
       const settlement = await activity.settleSessionInterruptions({
         accountId: input.accountId,
@@ -630,6 +679,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
       const finalPeek = await activity.peekSessionWork({
         workspaceId: input.workspaceId,
         sessionId: input.sessionId,
+        ...(safeObservation ? { observerAccountId: input.accountId } : {}),
       });
       if (
         finalPeek.kind !== "sandbox-lifecycle-wait" ||
@@ -711,6 +761,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
       const finalPeek = await activity.peekSessionWork({
         workspaceId: input.workspaceId,
         sessionId: input.sessionId,
+        ...(safeObservation ? { observerAccountId: input.accountId } : {}),
       });
       if (
         finalPeek.kind !== "input-wait" ||
@@ -763,6 +814,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
       const finalPeek = await activity.peekSessionWork({
         workspaceId: input.workspaceId,
         sessionId: input.sessionId,
+        ...(safeObservation ? { observerAccountId: input.accountId } : {}),
       });
       if (finalPeek.kind !== "idle") continue;
       await activity.markSessionIdle({
@@ -779,7 +831,15 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
       peek.kind === "approval-pending"
         ? ({ kind: "approval", triggerEventId: peek.triggerEventId } as const)
         : ({ kind: "next" } as const);
-    if (!(await runTurn(input.accountId, input.workspaceId, input.sessionId, trigger))) {
+    if (
+      !(await runTurn(
+        input.accountId,
+        input.workspaceId,
+        input.sessionId,
+        trigger,
+        "admissionFence" in peek ? peek.admissionFence : undefined,
+      ))
+    ) {
       if (signalVersion !== closeSignalVersion) continue;
       return;
     }
@@ -790,6 +850,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
     workspaceId: string,
     sessionId: string,
     trigger: activities.RunAgentTurnInput["trigger"],
+    admissionFence?: activities.FailSessionAttemptInput["admissionFence"],
   ): Promise<boolean> {
     const capacityWaitEntryBaseline = { wakeups, capacityWakeups };
     // Capture every admission-relevant signal before activity dispatch. A
@@ -1072,6 +1133,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
                   ? { preClaimFailureDisposition: admissionFailureDisposition }
                   : {}),
                 ...(admissionFailure ? { preClaimFailure: admissionFailure } : {}),
+                ...(durableAdmissionBlocking && admissionFence ? { admissionFence } : {}),
                 ...(postClaimDatabaseRecovery ? { postClaimDatabaseRecovery } : {}),
                 trigger,
                 error: workflowFailureMessage(outcome.error),

package/dist/activities/agent-turn/knowledge-sources.d.ts

@@ -1,20 +0,0 @@
-import { type FirstPartyMcpToolName, type Permission, type ResourceRef, type KnowledgeFilePreparationResult } from "@opengeni/contracts";
-import { type Settings } from "@opengeni/config";
-import { createFirstPartyAttemptClient } from "@opengeni/runtime";
-export type KnowledgeSourcePreparationOutcome = KnowledgeFilePreparationResult | {
-    status: "failed";
-    fileId: string;
-    message: string;
-};
-/** Accepted chat/schedule resources use ordinary agent authority and policy. */
-export declare function prepareTurnKnowledgeSources(input: {
-    settings: Settings;
-    scope: Parameters<typeof createFirstPartyAttemptClient>[0]["scope"];
-    resources: readonly ResourceRef[];
-    selectedTools?: readonly FirstPartyMcpToolName[] | undefined;
-    permissions?: readonly Permission[] | null | undefined;
-    learningMode: "automatic" | "review_first" | "off";
-    signal?: AbortSignal | undefined;
-    onOutcome: (outcome: KnowledgeSourcePreparationOutcome) => Promise<void>;
-    prepare?: (fileId: string) => Promise<KnowledgeFilePreparationResult>;
-}): Promise<string | undefined>;

package/dist/activities/packs.d.ts

@@ -1,55 +0,0 @@
-import { CapabilityPack } from "@opengeni/contracts";
-import { type Database } from "@opengeni/db";
-import { packInstallationUsesLegacyRuntime } from "@opengeni/core";
-import { type InstalledSkillActivation, type PackSkillActivation } from "@opengeni/runtime";
-export { resolveRigProviderImageSelection, rigProviderImageSourceImage, settingsWithPackSandboxImage, settingsWithRigImage, settingsWithRigProviderImage, } from "./sandbox-images.js";
-export { packInstallationUsesLegacyRuntime };
-/**
- * Legacy pack-scoped runtime compatibility. V2 Pack installations own ordinary
- * Skill components and select an explicit Rig; they contribute nothing here.
- */
-export type WorkspacePackRuntime = {
-    sandboxImage: string | null;
-    sandboxProviderImages: CapabilityPack["sandboxProviderImages"] | null;
-    skillActivations: PackSkillActivation[];
-};
-export type WorkspaceInstalledSkillRuntime = {
-    activations: InstalledSkillActivation[];
-};
-/**
- * Resolves only pre-V2 active Pack installations. A frozen manifest/digest is
- * the protocol marker for V2: inline Skills were migrated into the ordinary
- * Skill ledger and sandboxImage was resolved to selectedRigId during install,
- * so reading either field directly here would duplicate ownership and silently
- * override session compute.
- */
-export declare function resolveWorkspacePackRuntime(db: Database, workspaceId: string): Promise<WorkspacePackRuntime>;
-/**
- * Resolves every active immutable Skill through the normalized Plugin/Skill-
- * Facet ledger. The data layer has already filtered ineffective owner edges;
- * this boundary independently verifies exact files before runtime activation.
- */
-export declare function resolveWorkspaceInstalledSkillRuntime(db: Database, workspaceId: string): Promise<WorkspaceInstalledSkillRuntime>;
-/**
- * Pure pre-V2 compatibility rule for enabled Pack manifests. At most one
- * legacy Pack may declare a sandbox image, and legacy Skill names must be
- * unique. V2 rows never reach this function through runtime resolution.
- */
-export declare function workspacePackRuntimeFromPacks(packs: CapabilityPack[]): WorkspacePackRuntime;
-/**
- * Resolves the per-workspace agent persona override (the white-label surface).
- * Returns the workspace's stored template when set, else null to mean "use the
- * deployment default" (settings.agentInstructionsTemplate). The runtime always
- * injects the non-bypassable CORE regardless, so a null here keeps the
- * byte-identical default and a non-null value only restyles the persona.
- *
- * This is the workspace tier of the session > workspace > deployment-default
- * resolution; per-session overrides do not exist in this slice, so the worker
- * resolves workspace > default and passes the result as instructionsTemplate.
- */
-export declare function resolveWorkspaceAgentInstructions(db: Database, workspaceId: string): Promise<string | null>;
-/**
- * Applies a pack-declared sandbox image to run settings. With no pack image
- * the settings pass through untouched, so deployments without packs keep the
- * global OPENGENI_DOCKER_IMAGE / OPENGENI_MODAL_IMAGE_REF behavior exactly.
- */