@opengeni/worker-bundle 2.2.0 → 2.2.1-canary.36927100590001

package/src/observability-metrics.ts

@@ -2,12 +2,18 @@ import { createHash } from "node:crypto";
 import { errorCodeToJSON } from "@opengeni/agent-proto";
 import {
   BundledSkillId,
+  CREDIT_GRANT_CLASSES,
   SandboxBackend,
   type SessionEventType,
   type SkillReadKind,
   type SkillUseSource,
 } from "@opengeni/contracts";
-import type { SessionEventAppendPhaseObservation } from "@opengeni/db";
+import {
+  ACTIVE_USER_WINDOWS,
+  type ActiveUserWindow,
+  type CreditGrantTotals,
+  type SessionEventAppendPhaseObservation,
+} from "@opengeni/db";
 import {
   natsSubscriptionTerminationCounter,
   type EventBusOptions,
@@ -1122,6 +1128,54 @@ export function recordCreditBalanceGauges(
   creditBalanceGaugeAccounts.set(observability, current);
 }
 
+/**
+ * Distinct managed people with authenticated browser activity in each fixed
+ * window (migration 0565 presence). Every control worker publishes the same
+ * global value, so dashboards must aggregate with `max()` across pods, never
+ * `sum()`. API keys, services and embedded hosts are never counted.
+ */
+export function recordActiveUserGauges(
+  observability: Observability,
+  counts: Record<ActiveUserWindow, number>,
+): void {
+  for (const window of ACTIVE_USER_WINDOWS) {
+    observability.setGauge({
+      name: "opengeni_active_users",
+      help: "Distinct managed users with authenticated activity within the window (global; use max across pods).",
+      labels: { window },
+      value: counts[window],
+    });
+  }
+}
+
+/**
+ * Positive credit grants observed by the ledger trigger since migration 0565,
+ * by closed class. These are cumulative database totals published as gauges by
+ * every control worker: aggregate with `max()` across pods, then `increase()`.
+ * Counting in the database covers the grants no application process writes:
+ * the verified-signup trial grant (a setup trigger) and operator grants.
+ */
+export function recordCreditGrantGauges(
+  observability: Observability,
+  totals: CreditGrantTotals,
+): void {
+  for (const grantClass of CREDIT_GRANT_CLASSES) {
+    const total = totals[grantClass];
+    observability.setGauge({
+      name: "opengeni_credit_grants_total",
+      help: "Positive credit grants observed in the ledger since migration 0565, by grant class (global cumulative; use max across pods).",
+      labels: { grant_class: grantClass },
+      value: total.count,
+    });
+    observability.setGauge({
+      name: "opengeni_credit_granted_micros_total",
+      help: "Credit micros granted in the ledger since migration 0565, by grant class (global cumulative; use max across pods).",
+      labels: { grant_class: grantClass },
+      value: total.micros,
+    });
+  }
+}
+
 /**
  * The deployment-level runtime switch for the one-time verified signup trial
  * credit (migration 0521): 1 while it allows grants, 0 when an operator has

package/src/sandbox-resume.ts

@@ -210,6 +210,9 @@ export type ResumeBoxIds = {
    * never conflicts).
    */
   image?: string;
+  /** Deployment/workspace image pins apply only to new creates; an existing
+   * group keeps its image. Omission preserves explicit image matching (B3). */
+  imagePolicy?: "require_match" | "new_creates_only";
   /**
    * RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
    * acquireLease, which stamps it on the cold-create and conflicts on a live box
@@ -445,9 +448,10 @@ export async function waitForSandboxExecReadiness(
   established: EstablishedSandboxSession,
   timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
   identity: { sandboxGroupId?: string | null } = {},
+  signal?: AbortSignal,
 ): Promise<void> {
   try {
-    await verifySandboxExecReadiness(established, timeoutMs);
+    await verifySandboxExecReadiness(established, timeoutMs, signal);
   } catch (error) {
     if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
       throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
@@ -1347,11 +1351,12 @@ async function resumeBoxForTurnOnce(
     },
     os,
     // IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
-    // conflicts on a live box already running a different image. A
+    // conflicts on a live box already running a different required image. A
     // SandboxImageConflictError propagates while another holder is active; a
     // solo change requests a capture-and-drain rotation and this attempt retries
     // after the cold successor can safely stamp the new image.
     ...(ids.image ? { image: ids.image } : {}),
+    ...(ids.imagePolicy ? { imagePolicy: ids.imagePolicy } : {}),
     // RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
     // + conflicts on a live box under a different rig. A SandboxRigConflictError
     // propagates while another holder is active; a solo change uses the same
@@ -1917,9 +1922,14 @@ async function resumeBoxForTurnOnce(
           sandboxGroupId: ids.sandboxGroupId,
         });
       } else {
-        await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
-          sandboxGroupId: ids.sandboxGroupId,
-        });
+        await waitForSandboxExecReadiness(
+          established,
+          MODAL_EXEC_READINESS_TIMEOUT_MS,
+          {
+            sandboxGroupId: ids.sandboxGroupId,
+          },
+          cancellationSignal,
+        );
       }
       await maybeRenewProviderExpiration(true);
       throwIfReleasedOrCancelled();
@@ -2117,9 +2127,14 @@ async function resumeBoxForTurnOnce(
       if (services.verifyAttachedSandboxReadiness) {
         await services.verifyAttachedSandboxReadiness(established);
       } else {
-        await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
-          sandboxGroupId: ids.sandboxGroupId,
-        });
+        await waitForSandboxExecReadiness(
+          established,
+          MODAL_EXEC_READINESS_TIMEOUT_MS,
+          {
+            sandboxGroupId: ids.sandboxGroupId,
+          },
+          cancellationSignal,
+        );
       }
       providerRenewalTarget = {
         backend: ids.backend,

package/src/sandbox-routing.ts

@@ -496,7 +496,7 @@ function afterPersistableHomeMutation(
       op: string;
       backend: ResolvedActiveBackend;
       admission: unknown;
-      outcome: "resolved" | "rejected";
+      outcome: "resolved" | "rejected" | "outcome_unknown";
       result?: unknown;
       retainedProcess?: RoutingRetainedProcess;
     }) => Promise<void | RoutingMutationSettlementResult>)
@@ -523,7 +523,7 @@ function afterPersistableHomeMutation(
     ) {
       throw new Error("Persistable home mutation settlement lacked its bound admission");
     }
-    if (outcome === "resolved" && retainedProcess) {
+    if ((outcome === "resolved" || outcome === "outcome_unknown") && retainedProcess) {
       try {
         await retainWorkspaceProviderCommand(services.db, {
           accountId: fence.accountId,
@@ -582,6 +582,8 @@ function afterPersistableHomeMutation(
       }
       return;
     }
+    if (outcome === "outcome_unknown")
+      throw new Error("Outcome-unknown command settlement requires its exact retained invocation");
     await verifyWorkspaceMutationSettlement(services.db, {
       accountId: fence.accountId,
       workspaceId: ids.workspaceId,

package/src/workflows/browser-deadline-checkpoint.ts

@@ -0,0 +1,50 @@
+import {
+  ParentClosePolicy,
+  WorkflowIdReusePolicy,
+  log,
+  proxyActivities,
+  startChild,
+  workflowInfo,
+} from "@temporalio/workflow";
+import type { BrowserDeadlineCheckpointTarget } from "@opengeni/db";
+import type * as activities from "../activities";
+
+const inventory = proxyActivities<Pick<typeof activities, "listDueBrowserCheckpoints">>({
+  startToCloseTimeout: "5 minutes",
+  heartbeatTimeout: "20 seconds",
+});
+const checkpoint = proxyActivities<Pick<typeof activities, "checkpointBrowserBeforeDeadline">>({
+  startToCloseTimeout: "25 minutes",
+  heartbeatTimeout: "20 seconds",
+  retry: { initialInterval: "1 second", maximumInterval: "30 seconds" },
+});
+
+export async function browserDeadlineCheckpointWorkflow(target: BrowserDeadlineCheckpointTarget) {
+  return await checkpoint.checkpointBrowserBeforeDeadline(target);
+}
+
+/** Runs from the existing lease-sweep dispatch. Inventory never performs provider
+ * I/O; each exact browser has an independent durable capture/cleanup child. */
+export async function browserDeadlineCheckpointSweepWorkflow(): Promise<void> {
+  const targets = await inventory.listDueBrowserCheckpoints();
+  await Promise.all(
+    targets.map(async (target) => {
+      try {
+        await startChild(browserDeadlineCheckpointWorkflow, {
+          workflowId: `browser-deadline:${target.browserSessionId}:${target.leaseEpoch}:${target.controllerGeneration}`,
+          taskQueue: workflowInfo().taskQueue,
+          workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
+          parentClosePolicy: ParentClosePolicy.ABANDON,
+          args: [target],
+        });
+      } catch (error) {
+        if (error instanceof Error && error.name === "WorkflowExecutionAlreadyStartedError") return;
+        // Another sweep retries start; it cannot retire the browser's authority.
+        log.warn("browser deadline checkpoint child start deferred", {
+          browserSessionId: target.browserSessionId,
+          error: error instanceof Error ? error.name : "unknown",
+        });
+      }
+    }),
+  );
+}

package/src/workflows/scheduled-tasks.ts

@@ -7,6 +7,7 @@ import {
   WorkflowIdReusePolicy,
 } from "@temporalio/workflow";
 import type { TurnInitiator } from "@opengeni/contracts";
+import type { DispatchScheduledTaskRunInput } from "../activities/types";
 import { scheduledTaskActivity } from "./activities";
 import {
   knowledgeSourceSyncWake,
@@ -26,14 +27,38 @@ export type ScheduledTaskFireWorkflowInput = ScheduledTaskFireWorkflowBase &
         triggerType: "scheduled";
         agentRunUsageIdempotencyKey?: never;
         initiator?: never;
+        credentialRestriction?: never;
       }
     | {
         triggerType: "manual" | "initial" | "provider_event" | "retry" | "repair";
         agentRunUsageIdempotencyKey: string;
         initiator: TurnInitiator;
+        credentialRestriction?: "developer_setup";
       }
   );
 
+export function scheduledTaskDispatchInput(
+  input: ScheduledTaskFireWorkflowInput,
+  producerKey: string,
+): DispatchScheduledTaskRunInput {
+  const base = {
+    workspaceId: input.workspaceId,
+    taskId: input.taskId,
+    producerKey,
+  };
+  return input.triggerType !== "scheduled"
+    ? {
+        ...base,
+        triggerType: input.triggerType,
+        agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey,
+        initiator: input.initiator,
+        ...(input.credentialRestriction === "developer_setup"
+          ? { credentialRestriction: input.credentialRestriction }
+          : {}),
+      }
+    : { ...base, triggerType: "scheduled" };
+}
+
 export async function scheduledTaskFireWorkflow(
   input: ScheduledTaskFireWorkflowInput,
 ): Promise<void> {
@@ -47,20 +72,8 @@ export async function scheduledTaskFireWorkflow(
   ) {
     return;
   }
-  const base = {
-    workspaceId: input.workspaceId,
-    taskId: input.taskId,
-    producerKey: workflowInfo().workflowId,
-  };
   const result = await scheduledTaskActivity.dispatchScheduledTaskRun(
-    input.triggerType !== "scheduled"
-      ? {
-          ...base,
-          triggerType: input.triggerType,
-          agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey,
-          initiator: input.initiator,
-        }
-      : { ...base, triggerType: "scheduled" },
+    scheduledTaskDispatchInput(input, workflowInfo().workflowId),
   );
   if (result.action !== "knowledge_source_sync") return;
 

package/src/workflows.ts

@@ -28,6 +28,10 @@ export {
   sandboxReaperWorkflowV2,
 } from "./workflows/sandbox-reaper";
 export { fileUploadReaperWorkflow } from "./workflows/file-upload-reaper";
+export {
+  browserDeadlineCheckpointWorkflow,
+  browserDeadlineCheckpointSweepWorkflow,
+} from "./workflows/browser-deadline-checkpoint";
 export { siteAuthMaintenanceWorkflow } from "./workflows/site-auth-maintenance";
 export {
   videoGenerationWorkflow,