@opengeni/worker-bundle 2.2.0 → 2.2.1-canary.36919050519001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/activities/agent-run-admission.d.ts +3 -1
  2. package/dist/activities/agent-turn/admission.d.ts +24 -3
  3. package/dist/activities/agent-turn/agent-build.d.ts +1 -0
  4. package/dist/activities/agent-turn/claude-usage-observer.d.ts +13 -1
  5. package/dist/activities/agent-turn/compaction-prep.d.ts +20 -0
  6. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  7. package/dist/activities/agent-turn/governance-model.d.ts +6 -0
  8. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  9. package/dist/activities/agent-turn/run-credentials.d.ts +2 -0
  10. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  11. package/dist/activities/agent-turn/session-title.d.ts +3 -1
  12. package/dist/activities/agent-turn/stream-attempt.d.ts +1 -0
  13. package/dist/activities/agent-turn/turn-context.d.ts +1 -0
  14. package/dist/activities/automations.d.ts +3 -1
  15. package/dist/activities/browser-deadline-checkpoint.d.ts +29 -0
  16. package/dist/activities/goals.d.ts +8 -0
  17. package/dist/activities/run-credentials.d.ts +4 -2
  18. package/dist/activities/types.d.ts +1 -1
  19. package/dist/{activities-control-MHNA3CYV.js → activities-control-55ECL47G.js} +403 -116
  20. package/dist/activities-control-55ECL47G.js.map +1 -0
  21. package/dist/activities-control.d.ts +30 -0
  22. package/dist/{activities-turn-FQ5OO2WW.js → activities-turn-T7Y3BWYO.js} +678 -246
  23. package/dist/activities-turn-T7Y3BWYO.js.map +1 -0
  24. package/dist/activities.d.ts +30 -0
  25. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  26. package/dist/{chunk-RZAAGRFQ.js → chunk-OWNLATJL.js} +189 -80
  27. package/dist/chunk-OWNLATJL.js.map +1 -0
  28. package/dist/index.d.ts +15 -0
  29. package/dist/index.js +45 -23
  30. package/dist/index.js.map +1 -1
  31. package/dist/sandbox-resume.d.ts +4 -1
  32. package/dist/workflow-bundle.js +72 -10
  33. package/dist/workflows/browser-deadline-checkpoint.d.ts +9 -0
  34. package/dist/workflows.d.ts +1 -0
  35. package/package.json +21 -20
  36. package/src/activities/agent-run-admission.ts +17 -1
  37. package/src/activities/agent-turn/admission.ts +64 -1
  38. package/src/activities/agent-turn/agent-build.ts +24 -3
  39. package/src/activities/agent-turn/claim.ts +56 -14
  40. package/src/activities/agent-turn/claude-usage-observer.ts +33 -1
  41. package/src/activities/agent-turn/compaction-prep.ts +67 -16
  42. package/src/activities/agent-turn/errors.ts +55 -0
  43. package/src/activities/agent-turn/failure-settlement.ts +26 -6
  44. package/src/activities/agent-turn/final-reply.ts +46 -0
  45. package/src/activities/agent-turn/governance-model.ts +12 -0
  46. package/src/activities/agent-turn/model-call-admission.ts +94 -0
  47. package/src/activities/agent-turn/run-credentials.ts +15 -3
  48. package/src/activities/agent-turn/run.ts +43 -5
  49. package/src/activities/agent-turn/sandbox-establish.ts +9 -5
  50. package/src/activities/agent-turn/session-title.ts +8 -2
  51. package/src/activities/agent-turn/stream-attempt.ts +144 -38
  52. package/src/activities/agent-turn/tool-environment.ts +51 -39
  53. package/src/activities/agent-turn/turn-context.ts +1 -0
  54. package/src/activities/automations.ts +55 -3
  55. package/src/activities/browser-deadline-checkpoint.ts +279 -0
  56. package/src/activities/capabilities.ts +52 -14
  57. package/src/activities/goals.ts +67 -95
  58. package/src/activities/knowledge-indexing.ts +29 -0
  59. package/src/activities/open-suffix-resume.ts +2 -1
  60. package/src/activities/run-credentials.ts +22 -16
  61. package/src/activities/scheduled-tasks.ts +83 -13
  62. package/src/activities/types.ts +1 -0
  63. package/src/activities/workspace-credential-provider.ts +12 -0
  64. package/src/activities-control.ts +2 -0
  65. package/src/activities.ts +3 -0
  66. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  67. package/src/index.ts +53 -25
  68. package/src/sandbox-resume.ts +23 -8
  69. package/src/sandbox-routing.ts +4 -2
  70. package/src/workflows/browser-deadline-checkpoint.ts +50 -0
  71. package/src/workflows.ts +4 -0
  72. package/dist/activities-control-MHNA3CYV.js.map +0 -1
  73. package/dist/activities-turn-FQ5OO2WW.js.map +0 -1
  74. package/dist/chunk-RZAAGRFQ.js.map +0 -1
@@ -2,6 +2,7 @@ import type { Agent } from "@openai/agents";
2
2
  import {
3
3
  OPEN_SUFFIX_RUN_STATE_BLOB,
4
4
  approvalIdentifier,
5
+ withPublicApprovalFields,
5
6
  type HumanInputResponse,
6
7
  type SessionEvent,
7
8
  } from "@opengeni/contracts";
@@ -380,7 +381,7 @@ export async function settleOpenSuffixResumeIfNeeded(input: {
380
381
  ? [
381
382
  {
382
383
  type: "session.requiresAction" as const,
383
- payload: { approvals: requiresActionApprovals },
384
+ payload: { approvals: requiresActionApprovals.map(withPublicApprovalFields) },
384
385
  },
385
386
  ]
386
387
  : []),
@@ -5,6 +5,7 @@ import type {
5
5
  RunCredentialsRequest,
6
6
  Session,
7
7
  SessionTurn,
8
+ ToolRef,
8
9
  } from "@opengeni/contracts";
9
10
  import type { Settings } from "@opengeni/config";
10
11
  import { getSessionRootId, type Database } from "@opengeni/db";
@@ -30,6 +31,8 @@ export type RunCredentialResolutionContext = {
30
31
  initiatingHumanSubjectId?: string | null;
31
32
  /** Installed in-process API routes must never enter a product callback. */
32
33
  localMcpServerIds?: readonly string[];
34
+ /** Exact execution selection, resolved once at the turn policy boundary. */
35
+ effectiveTools: readonly ToolRef[];
33
36
  };
34
37
 
35
38
  export type BoundRunCredentialResolver = {
@@ -42,7 +45,7 @@ export type BoundRunCredentialResolver = {
42
45
  export function buildRunCredentialsRequest(
43
46
  input: Omit<
44
47
  RunCredentialResolutionContext,
45
- "db" | "connectionCredentials" | "settings" | "initiatingHumanSubjectId"
48
+ "db" | "connectionCredentials" | "settings" | "initiatingHumanSubjectId" | "effectiveTools"
46
49
  > & {
47
50
  rootSessionId: string;
48
51
  purpose: "provision" | "renewal";
@@ -125,7 +128,7 @@ export async function bindRunCredentialResolver(
125
128
  mcpServers: selectedSessionRemoteMcpTargets(
126
129
  input.settings,
127
130
  input.session.mcpServers ?? [],
128
- input.turn.tools ?? [],
131
+ input.effectiveTools,
129
132
  (input.localMcpServerIds ?? []).map((id) => ({ id })),
130
133
  ),
131
134
  },
@@ -146,22 +149,25 @@ export async function bindRunCredentialResolver(
146
149
  workspaceId: input.workspaceId,
147
150
  sessionId: input.session.id,
148
151
  };
152
+ // A renewable resolver belongs to one admitted turn, even if its caller
153
+ // subsequently replaces or mutates an in-memory session/turn projection.
154
+ const frozenRequest = structuredClone(
155
+ buildRunCredentialsRequest({
156
+ accountId: input.accountId,
157
+ workspaceId: input.workspaceId,
158
+ session: input.session,
159
+ turn: input.turn,
160
+ attemptId: input.attemptId,
161
+ effectiveSandboxBackend: input.effectiveSandboxBackend,
162
+ variableSet: input.variableSet,
163
+ rootSessionId,
164
+ purpose: "provision",
165
+ forceRefresh: false,
166
+ }),
167
+ );
149
168
  return {
150
169
  resolve: async ({ purpose, forceRefresh }) => {
151
- const resolution = await resolver(
152
- buildRunCredentialsRequest({
153
- accountId: input.accountId,
154
- workspaceId: input.workspaceId,
155
- session: input.session,
156
- turn: input.turn,
157
- attemptId: input.attemptId,
158
- effectiveSandboxBackend: input.effectiveSandboxBackend,
159
- variableSet: input.variableSet,
160
- rootSessionId,
161
- purpose,
162
- forceRefresh,
163
- }),
164
- );
170
+ const resolution = await resolver({ ...frozenRequest, purpose, forceRefresh });
165
171
  return normalizeRunCredentialsResolution(resolution, scope);
166
172
  },
167
173
  };
@@ -15,6 +15,8 @@ import {
15
15
  TurnExecutionPolicyV1,
16
16
  type ScheduledTask,
17
17
  type ScheduledTaskRun,
18
+ AgentConfigError,
19
+ type ResolvedAgentConfig,
18
20
  } from "@opengeni/contracts";
19
21
  import {
20
22
  openGeniSlackBotMetadata,
@@ -33,6 +35,9 @@ import {
33
35
  settingsWithEnabledCapabilityMcpServers,
34
36
  settingsWithSessionMcpServerMetadata,
35
37
  swapActiveSandbox,
38
+ agentConfigMayResolve,
39
+ applySessionAgentConfigWriteThrough,
40
+ resolveSessionAgentConfigForCreate,
36
41
  } from "@opengeni/core";
37
42
  import {
38
43
  appendSessionEvents,
@@ -62,6 +67,7 @@ import {
62
67
  getNestedAgentDepthDeploymentPolicy,
63
68
  getSessionByCreateIdempotencyKey,
64
69
  getSessionCreationExecutionPolicy,
70
+ metadataWithAgentConfigCreateIdentity,
65
71
  getVariableSet,
66
72
  isCodexBilledModel,
67
73
  initializeSessionStartAtomically,
@@ -501,7 +507,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
501
507
  settings.openaiReasoningEffort;
502
508
  let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
503
509
  let sandboxOs: "linux" | "macos" | "windows" = "linux";
504
- const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
510
+ let taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
505
511
  // A task created by a live agent attempt froze its creator's effective
506
512
  // first-party selection, permission set, and session access policy;
507
513
  // its generated sessions inherit that boundary under today's
@@ -510,13 +516,55 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
510
516
  const creatorPolicy = await getScheduledTaskCreatorPolicy(db, task.workspaceId, task.id);
511
517
  // A person-chosen Slack channel adds only the two bot posting tools;
512
518
  // they post nowhere else, whatever the creator's own selection was.
513
- const firstPartyMcpTools = withScheduledSlackBotPostingTools(
519
+ let firstPartyMcpTools = withScheduledSlackBotPostingTools(
514
520
  creatorPolicy?.firstPartyMcpTools
515
521
  ? allowedFirstPartyMcpToolsForSession(settings, creatorPolicy.firstPartyMcpTools)
516
522
  : resolveFirstPartyMcpToolPolicy(settings).default,
517
523
  task.agentConfig,
518
524
  resolveFirstPartyMcpToolPolicy(settings).allowed,
519
525
  );
526
+ // Agent configuration for generated sessions (null = legacy). The
527
+ // scheduled baseline above (creator policy or deployment default) is
528
+ // what "all" reproduces; capabilities only narrow it. Frozen into the
529
+ // accepted execution so retries and recovery never resolve again.
530
+ let resolvedAgentConfig: ResolvedAgentConfig | null = null;
531
+ let resolvedAgentInstructions: string | undefined;
532
+ try {
533
+ const agentResolution = resolveSessionAgentConfigForCreate({
534
+ settings,
535
+ creator: "scheduled",
536
+ request: task.agentConfig.agent,
537
+ instructions: undefined,
538
+ workspaceSettings: agentConfigMayResolve(settings, task.agentConfig.agent)
539
+ ? (await requireWorkspace(db, task.workspaceId)).settings
540
+ : {},
541
+ parent: null,
542
+ goal: task.agentConfig.goal !== undefined,
543
+ });
544
+ resolvedAgentConfig = agentResolution.config;
545
+ resolvedAgentInstructions = agentResolution.instructions;
546
+ const written = applySessionAgentConfigWriteThrough({
547
+ config: resolvedAgentConfig,
548
+ firstPartyMcpTools,
549
+ tools: taskTools,
550
+ toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
551
+ productServerIds: task.agentConfig.tools.map((tool) => tool.id),
552
+ });
553
+ firstPartyMcpTools = written.firstPartyMcpTools;
554
+ taskTools = written.tools;
555
+ } catch (error) {
556
+ const status =
557
+ error instanceof Error ? (error as Error & { status?: unknown }).status : undefined;
558
+ if (!(error instanceof Error) || status !== 422) throw error;
559
+ const notEnabled =
560
+ error.cause instanceof AgentConfigError &&
561
+ error.cause.code === "agent_config_not_enabled";
562
+ return await refuseAdmission(
563
+ "scheduled_authority_unavailable",
564
+ notEnabled,
565
+ `scheduled agent configuration is not admissible: ${error.message}`,
566
+ );
567
+ }
520
568
  const firstPartyMcpPermissions = creatorPolicy?.firstPartyMcpPermissions
521
569
  ? [...creatorPolicy.firstPartyMcpPermissions]
522
570
  : [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS];
@@ -917,6 +965,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
917
965
  workspaceId: task.workspaceId,
918
966
  model: targetSessionExecution?.model ?? model,
919
967
  requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1,
968
+ initiatingHumanSubjectId: causalHumanSubjectId,
920
969
  },
921
970
  );
922
971
  if (admissionDenial) {
@@ -994,6 +1043,12 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
994
1043
  resolvedTools: taskTools,
995
1044
  resolvedFirstPartyMcpTools: firstPartyMcpTools,
996
1045
  resolvedFirstPartyMcpPermissions: firstPartyMcpPermissions,
1046
+ ...(resolvedAgentConfig
1047
+ ? {
1048
+ resolvedAgentConfig,
1049
+ ...(resolvedAgentInstructions !== undefined ? { resolvedAgentInstructions } : {}),
1050
+ }
1051
+ : {}),
997
1052
  resolvedVariableSet: acceptedVariableSet
998
1053
  ? { id: acceptedVariableSet.id, generation: acceptedVariableSet.generation }
999
1054
  : null,
@@ -1201,6 +1256,10 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1201
1256
  tools: taskTools,
1202
1257
  firstPartyMcpTools,
1203
1258
  firstPartyMcpPermissions,
1259
+ ...(resolvedAgentConfig ? { agentConfig: resolvedAgentConfig } : {}),
1260
+ ...(resolvedAgentInstructions !== undefined
1261
+ ? { instructions: resolvedAgentInstructions }
1262
+ : {}),
1204
1263
  ...creatorSessionPolicy,
1205
1264
  ...(task.agentConfig.knowledgeSource?.destination.kind === "personal"
1206
1265
  ? {
@@ -2242,6 +2301,12 @@ async function recoverBoundScheduledTaskDispatch(input: {
2242
2301
  tools: input.acceptedExecution.resolvedTools,
2243
2302
  firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
2244
2303
  firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
2304
+ ...(input.acceptedExecution.resolvedAgentConfig
2305
+ ? { agentConfig: input.acceptedExecution.resolvedAgentConfig }
2306
+ : {}),
2307
+ ...(input.acceptedExecution.resolvedAgentInstructions !== undefined
2308
+ ? { instructions: input.acceptedExecution.resolvedAgentInstructions }
2309
+ : {}),
2245
2310
  ...scheduledCreatorSessionPolicyInput(recoveredCreatorPolicy?.sessionPolicy ?? null),
2246
2311
  ...(task.agentConfig.knowledgeSource?.destination.kind === "personal"
2247
2312
  ? {
@@ -2379,20 +2444,23 @@ async function recoverBoundScheduledTaskDispatch(input: {
2379
2444
  );
2380
2445
  const expectedTaskMetadata = { ...task.agentConfig.metadata };
2381
2446
  delete expectedTaskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
2382
- const expectedMetadata = {
2383
- ...expectedTaskMetadata,
2384
- model: input.acceptedExecution.resolvedModel,
2385
- reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
2386
- scheduledTaskId: task.id,
2387
- scheduledTaskRunMode: task.runMode,
2388
- ...(task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {}),
2389
- scheduledTaskRunId: canonicalGeneratedRunId,
2390
- ...(frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}),
2391
- };
2447
+ const expectedMetadata = metadataWithAgentConfigCreateIdentity(
2448
+ {
2449
+ ...expectedTaskMetadata,
2450
+ model: input.acceptedExecution.resolvedModel,
2451
+ reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
2452
+ scheduledTaskId: task.id,
2453
+ scheduledTaskRunMode: task.runMode,
2454
+ ...(task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {}),
2455
+ scheduledTaskRunId: canonicalGeneratedRunId,
2456
+ ...(frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}),
2457
+ },
2458
+ input.acceptedExecution.resolvedAgentConfig,
2459
+ );
2392
2460
  if (
2393
2461
  session.createIdempotencyKey !== expectedCreateKey ||
2394
2462
  session.initialMessage !== task.agentConfig.prompt ||
2395
- session.instructions !== null ||
2463
+ session.instructions !== (input.acceptedExecution.resolvedAgentInstructions ?? null) ||
2396
2464
  session.policyRole !== null ||
2397
2465
  stableJson(session.skills) !== "[]" ||
2398
2466
  stableJson(session.toolPolicy) !==
@@ -2434,6 +2502,8 @@ async function recoverBoundScheduledTaskDispatch(input: {
2434
2502
  stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) ||
2435
2503
  stableJson(session.firstPartyMcpPermissions) !==
2436
2504
  stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) ||
2505
+ stableJson(session.agent) !==
2506
+ stableJson(input.acceptedExecution.resolvedAgentConfig ?? null) ||
2437
2507
  session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) ||
2438
2508
  (session.variableSetId ?? null) !==
2439
2509
  (input.acceptedExecution.resolvedVariableSet?.id ?? null) ||
@@ -458,6 +458,7 @@ export type DispatchScheduledTaskRunResult =
458
458
  refusal?: import("@opengeni/contracts").ScheduledTaskAdmissionRefusal;
459
459
  reason:
460
460
  | "insufficient_credits"
461
+ | "allowance_exhausted"
461
462
  | "monthly_model_cost_limit"
462
463
  | "monthly_agent_run_limit"
463
464
  | "malformed_manual_trigger"
@@ -78,6 +78,8 @@ export function credentialProviderRequestBody(
78
78
  mcpServers: [],
79
79
  },
80
80
  ): CredentialProviderRequest {
81
+ const via = input.initiatorContext.via;
82
+ const latestHop = Array.isArray(via) ? via.at(-1) : null;
81
83
  return {
82
84
  type: "credentials.request",
83
85
  lane: selection.lane,
@@ -92,6 +94,16 @@ export function credentialProviderRequestBody(
92
94
  turnId: input.turnId,
93
95
  attemptId: input.attemptId,
94
96
  initiator: { kind: input.initiator.kind, subjectId: input.initiator.subjectId },
97
+ initiatorContext: {
98
+ kind:
99
+ latestHop?.kind === "agent"
100
+ ? "agent"
101
+ : input.initiator.kind === "subject"
102
+ ? "human"
103
+ : "service",
104
+ initiator: input.initiator,
105
+ context: input.initiatorContext,
106
+ },
95
107
  initiatingHumanSubjectId,
96
108
  initiatingHuman,
97
109
  sandboxBackend: input.effectiveSandboxBackend,
@@ -2,6 +2,7 @@ import { createKnowledgeIndexingActivities } from "./activities/knowledge-indexi
2
2
  import { createSharedActivityServices } from "./activity-services";
3
3
  import { createCodexCapacityActivities } from "./activities/codex-capacity";
4
4
  import { createBrowserStateArtifactMaintenanceActivities } from "./activities/browser-state-artifact-reaper";
5
+ import { createBrowserDeadlineCheckpointActivities } from "./activities/browser-deadline-checkpoint";
5
6
  import { createAutomationActivities } from "./activities/automations";
6
7
  import { createDocumentActivities } from "./activities/documents";
7
8
  import { createFileUploadReaperActivities } from "./activities/file-upload-reaper";
@@ -39,6 +40,7 @@ export function createControlActivitiesFromServices(
39
40
  ...createCodexCapacityActivities(services),
40
41
  ...createRigVerificationActivities(services),
41
42
  ...createBrowserStateArtifactMaintenanceActivities(services),
43
+ ...createBrowserDeadlineCheckpointActivities(services),
42
44
  ...createFileUploadReaperActivities(services),
43
45
  ...createRetainedScreenshotMaintenanceActivities(services),
44
46
  ...createWorkflowWakeActivities(services),
package/src/activities.ts CHANGED
@@ -117,6 +117,9 @@ export const maybeContinueGoal = defaultControlActivities.maybeContinueGoal;
117
117
  export const getCodexCapacityWait = defaultControlActivities.getCodexCapacityWait;
118
118
  export const reconcileCodexCapacityWait = defaultControlActivities.reconcileCodexCapacityWait;
119
119
  export const prepareSandboxLeaseSweep = defaultControlActivities.prepareSandboxLeaseSweep;
120
+ export const listDueBrowserCheckpoints = defaultControlActivities.listDueBrowserCheckpoints;
121
+ export const checkpointBrowserBeforeDeadline =
122
+ defaultControlActivities.checkpointBrowserBeforeDeadline;
120
123
  export const drainSandboxLease = defaultControlActivities.drainSandboxLease;
121
124
  export const maintainSandboxLeaseSweep = defaultControlActivities.maintainSandboxLeaseSweep;
122
125
  export const reapSandboxLeases = defaultControlActivities.reapSandboxLeases;
@@ -0,0 +1,5 @@
1
+ export const BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID = "opengeni-browser-deadline-checkpoints-v1";
2
+ const QUEUE_SUFFIX = "-browser-checkpoint-v1";
3
+ export function browserDeadlineCheckpointTaskQueue(base: string): string {
4
+ return base.endsWith(QUEUE_SUFFIX) ? base : `${base}${QUEUE_SUFFIX}`;
5
+ }
package/src/index.ts CHANGED
@@ -99,6 +99,10 @@ import {
99
99
  SANDBOX_REAPER_V2_WORKFLOW_ID,
100
100
  sandboxLifecycleTaskQueue,
101
101
  } from "./sandbox-reaper-contract";
102
+ import {
103
+ BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
104
+ browserDeadlineCheckpointTaskQueue,
105
+ } from "./browser-deadline-checkpoint-contract";
102
106
 
103
107
  export {
104
108
  createHostExportPump,
@@ -323,28 +327,39 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
323
327
  return { worker, connection };
324
328
  }
325
329
 
330
+ const ownedWorkers = [worker];
326
331
  try {
327
- const sandboxLifecycleWorker = await Worker.create({
328
- ...sharedWorkerOptions,
329
- ...workflowDefinition,
330
- taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
331
- reuseV8Context: true,
332
- workflowThreadPoolSize: 1,
333
- maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
334
- maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
335
- maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
336
- });
332
+ // The new activity type has its own queue. Retain the V1 poller so old
333
+ // drain histories finish; old binaries never receive checkpoint work.
334
+ for (const taskQueue of [
335
+ sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
336
+ browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
337
+ ]) {
338
+ const lifecycleWorker = await Worker.create({
339
+ ...sharedWorkerOptions,
340
+ ...workflowDefinition,
341
+ taskQueue,
342
+ reuseV8Context: true,
343
+ workflowThreadPoolSize: 1,
344
+ maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
345
+ maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
346
+ maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
347
+ });
348
+ ownedWorkers.push(lifecycleWorker);
349
+ }
337
350
  return {
338
- worker: combineWorkerRunTargets([worker, sandboxLifecycleWorker]),
351
+ worker: combineWorkerRunTargets(ownedWorkers),
339
352
  connection,
340
353
  };
341
354
  } catch (error) {
342
- try {
343
- worker.shutdown();
344
- } catch {
345
- // Preserve the lifecycle worker construction failure. The base poller
346
- // still received its shutdown request; a secondary synchronous
347
- // shutdown error must not replace the startup root cause.
355
+ for (const ownedWorker of ownedWorkers) {
356
+ try {
357
+ ownedWorker.shutdown();
358
+ } catch {
359
+ // Preserve the lifecycle worker construction failure. The base poller
360
+ // still received its shutdown request; a secondary synchronous
361
+ // shutdown error must not replace the startup root cause.
362
+ }
348
363
  }
349
364
  throw error;
350
365
  }
@@ -478,20 +493,33 @@ export async function createWorkerWorkflowSignaler(
478
493
  return normalizeTurnTaskQueueStats(response.stats);
479
494
  },
480
495
  startSandboxReaperWorkflow: async () => {
481
- try {
482
- await temporal.workflow.start("sandboxReaperWorkflowV2", {
496
+ // Same backend tick, no additional schedule. A bounded independent
497
+ // inventory cannot prevent the existing per-box drain from starting.
498
+ const starts = await Promise.allSettled([
499
+ temporal.workflow.start("sandboxReaperWorkflowV2", {
483
500
  taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
484
501
  workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
485
502
  workflowIdReusePolicy: "ALLOW_DUPLICATE",
486
503
  args: [],
487
- });
488
- return "started";
489
- } catch (error) {
490
- if (error instanceof WorkflowExecutionAlreadyStartedError) {
491
- return "already_running";
504
+ }),
505
+ temporal.workflow.start("browserDeadlineCheckpointSweepWorkflow", {
506
+ taskQueue: browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
507
+ workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
508
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
509
+ args: [],
510
+ }),
511
+ ]);
512
+ for (const start of starts) {
513
+ if (
514
+ start.status === "rejected" &&
515
+ !(start.reason instanceof WorkflowExecutionAlreadyStartedError)
516
+ ) {
517
+ // The other start already settled. Retry the router without losing
518
+ // ordinary draining or silently omitting the checkpoint sweep.
519
+ throw start.reason;
492
520
  }
493
- throw error;
494
521
  }
522
+ return starts[0].status === "fulfilled" ? "started" : "already_running";
495
523
  },
496
524
  startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
497
525
  try {
@@ -210,6 +210,9 @@ export type ResumeBoxIds = {
210
210
  * never conflicts).
211
211
  */
212
212
  image?: string;
213
+ /** Deployment/workspace image pins apply only to new creates; an existing
214
+ * group keeps its image. Omission preserves explicit image matching (B3). */
215
+ imagePolicy?: "require_match" | "new_creates_only";
213
216
  /**
214
217
  * RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
215
218
  * acquireLease, which stamps it on the cold-create and conflicts on a live box
@@ -445,9 +448,10 @@ export async function waitForSandboxExecReadiness(
445
448
  established: EstablishedSandboxSession,
446
449
  timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
447
450
  identity: { sandboxGroupId?: string | null } = {},
451
+ signal?: AbortSignal,
448
452
  ): Promise<void> {
449
453
  try {
450
- await verifySandboxExecReadiness(established, timeoutMs);
454
+ await verifySandboxExecReadiness(established, timeoutMs, signal);
451
455
  } catch (error) {
452
456
  if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
453
457
  throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
@@ -1347,11 +1351,12 @@ async function resumeBoxForTurnOnce(
1347
1351
  },
1348
1352
  os,
1349
1353
  // IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
1350
- // conflicts on a live box already running a different image. A
1354
+ // conflicts on a live box already running a different required image. A
1351
1355
  // SandboxImageConflictError propagates while another holder is active; a
1352
1356
  // solo change requests a capture-and-drain rotation and this attempt retries
1353
1357
  // after the cold successor can safely stamp the new image.
1354
1358
  ...(ids.image ? { image: ids.image } : {}),
1359
+ ...(ids.imagePolicy ? { imagePolicy: ids.imagePolicy } : {}),
1355
1360
  // RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
1356
1361
  // + conflicts on a live box under a different rig. A SandboxRigConflictError
1357
1362
  // propagates while another holder is active; a solo change uses the same
@@ -1917,9 +1922,14 @@ async function resumeBoxForTurnOnce(
1917
1922
  sandboxGroupId: ids.sandboxGroupId,
1918
1923
  });
1919
1924
  } else {
1920
- await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
1921
- sandboxGroupId: ids.sandboxGroupId,
1922
- });
1925
+ await waitForSandboxExecReadiness(
1926
+ established,
1927
+ MODAL_EXEC_READINESS_TIMEOUT_MS,
1928
+ {
1929
+ sandboxGroupId: ids.sandboxGroupId,
1930
+ },
1931
+ cancellationSignal,
1932
+ );
1923
1933
  }
1924
1934
  await maybeRenewProviderExpiration(true);
1925
1935
  throwIfReleasedOrCancelled();
@@ -2117,9 +2127,14 @@ async function resumeBoxForTurnOnce(
2117
2127
  if (services.verifyAttachedSandboxReadiness) {
2118
2128
  await services.verifyAttachedSandboxReadiness(established);
2119
2129
  } else {
2120
- await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
2121
- sandboxGroupId: ids.sandboxGroupId,
2122
- });
2130
+ await waitForSandboxExecReadiness(
2131
+ established,
2132
+ MODAL_EXEC_READINESS_TIMEOUT_MS,
2133
+ {
2134
+ sandboxGroupId: ids.sandboxGroupId,
2135
+ },
2136
+ cancellationSignal,
2137
+ );
2123
2138
  }
2124
2139
  providerRenewalTarget = {
2125
2140
  backend: ids.backend,
@@ -496,7 +496,7 @@ function afterPersistableHomeMutation(
496
496
  op: string;
497
497
  backend: ResolvedActiveBackend;
498
498
  admission: unknown;
499
- outcome: "resolved" | "rejected";
499
+ outcome: "resolved" | "rejected" | "outcome_unknown";
500
500
  result?: unknown;
501
501
  retainedProcess?: RoutingRetainedProcess;
502
502
  }) => Promise<void | RoutingMutationSettlementResult>)
@@ -523,7 +523,7 @@ function afterPersistableHomeMutation(
523
523
  ) {
524
524
  throw new Error("Persistable home mutation settlement lacked its bound admission");
525
525
  }
526
- if (outcome === "resolved" && retainedProcess) {
526
+ if ((outcome === "resolved" || outcome === "outcome_unknown") && retainedProcess) {
527
527
  try {
528
528
  await retainWorkspaceProviderCommand(services.db, {
529
529
  accountId: fence.accountId,
@@ -582,6 +582,8 @@ function afterPersistableHomeMutation(
582
582
  }
583
583
  return;
584
584
  }
585
+ if (outcome === "outcome_unknown")
586
+ throw new Error("Outcome-unknown command settlement requires its exact retained invocation");
585
587
  await verifyWorkspaceMutationSettlement(services.db, {
586
588
  accountId: fence.accountId,
587
589
  workspaceId: ids.workspaceId,
@@ -0,0 +1,50 @@
1
+ import {
2
+ ParentClosePolicy,
3
+ WorkflowIdReusePolicy,
4
+ log,
5
+ proxyActivities,
6
+ startChild,
7
+ workflowInfo,
8
+ } from "@temporalio/workflow";
9
+ import type { BrowserDeadlineCheckpointTarget } from "@opengeni/db";
10
+ import type * as activities from "../activities";
11
+
12
+ const inventory = proxyActivities<Pick<typeof activities, "listDueBrowserCheckpoints">>({
13
+ startToCloseTimeout: "5 minutes",
14
+ heartbeatTimeout: "20 seconds",
15
+ });
16
+ const checkpoint = proxyActivities<Pick<typeof activities, "checkpointBrowserBeforeDeadline">>({
17
+ startToCloseTimeout: "25 minutes",
18
+ heartbeatTimeout: "20 seconds",
19
+ retry: { initialInterval: "1 second", maximumInterval: "30 seconds" },
20
+ });
21
+
22
+ export async function browserDeadlineCheckpointWorkflow(target: BrowserDeadlineCheckpointTarget) {
23
+ return await checkpoint.checkpointBrowserBeforeDeadline(target);
24
+ }
25
+
26
+ /** Runs from the existing lease-sweep dispatch. Inventory never performs provider
27
+ * I/O; each exact browser has an independent durable capture/cleanup child. */
28
+ export async function browserDeadlineCheckpointSweepWorkflow(): Promise<void> {
29
+ const targets = await inventory.listDueBrowserCheckpoints();
30
+ await Promise.all(
31
+ targets.map(async (target) => {
32
+ try {
33
+ await startChild(browserDeadlineCheckpointWorkflow, {
34
+ workflowId: `browser-deadline:${target.browserSessionId}:${target.leaseEpoch}:${target.controllerGeneration}`,
35
+ taskQueue: workflowInfo().taskQueue,
36
+ workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
37
+ parentClosePolicy: ParentClosePolicy.ABANDON,
38
+ args: [target],
39
+ });
40
+ } catch (error) {
41
+ if (error instanceof Error && error.name === "WorkflowExecutionAlreadyStartedError") return;
42
+ // Another sweep retries start; it cannot retire the browser's authority.
43
+ log.warn("browser deadline checkpoint child start deferred", {
44
+ browserSessionId: target.browserSessionId,
45
+ error: error instanceof Error ? error.name : "unknown",
46
+ });
47
+ }
48
+ }),
49
+ );
50
+ }
package/src/workflows.ts CHANGED
@@ -28,6 +28,10 @@ export {
28
28
  sandboxReaperWorkflowV2,
29
29
  } from "./workflows/sandbox-reaper";
30
30
  export { fileUploadReaperWorkflow } from "./workflows/file-upload-reaper";
31
+ export {
32
+ browserDeadlineCheckpointWorkflow,
33
+ browserDeadlineCheckpointSweepWorkflow,
34
+ } from "./workflows/browser-deadline-checkpoint";
31
35
  export { siteAuthMaintenanceWorkflow } from "./workflows/site-auth-maintenance";
32
36
  export {
33
37
  videoGenerationWorkflow,