@opengeni/worker-bundle 0.7.5 → 0.10.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/chunk-BJUW5LCM.js +325 -0
  2. package/dist/chunk-BJUW5LCM.js.map +1 -0
  3. package/dist/host-export.d.ts +70 -0
  4. package/dist/host-export.js +7 -0
  5. package/dist/host-export.js.map +1 -0
  6. package/dist/index.d.ts +98 -6
  7. package/dist/index.js +4097 -1271
  8. package/dist/index.js.map +1 -1
  9. package/dist/workflow-bundle.js +12360 -0
  10. package/package.json +18 -12
  11. package/src/activities/agent-turn.ts +2674 -828
  12. package/src/activities/capabilities.ts +11 -2
  13. package/src/activities/codex-capacity.ts +6 -11
  14. package/src/activities/codex-rotation.ts +75 -86
  15. package/src/activities/context-compaction.ts +104 -24
  16. package/src/activities/environment.ts +503 -97
  17. package/src/activities/expiring-material-renewal.ts +143 -0
  18. package/src/activities/goals.ts +4 -0
  19. package/src/activities/mcp-credentials.ts +38 -0
  20. package/src/activities/packs.ts +115 -2
  21. package/src/activities/rig-verification.ts +897 -244
  22. package/src/activities/run-credential-renewal.ts +70 -0
  23. package/src/activities/run-credentials.ts +132 -0
  24. package/src/activities/run-input.ts +23 -0
  25. package/src/activities/sandbox-lease.ts +33 -2
  26. package/src/activities/scheduled-tasks.ts +21 -0
  27. package/src/activities/session-state.ts +72 -0
  28. package/src/activities/streaming.ts +1 -0
  29. package/src/activities/toolspace-token-renewal.ts +57 -0
  30. package/src/activities/types.ts +44 -6
  31. package/src/activities/workspace-capture.ts +113 -93
  32. package/src/activities.ts +10 -2
  33. package/src/host-export-pump.ts +416 -0
  34. package/src/host-export.ts +22 -0
  35. package/src/http.ts +77 -28
  36. package/src/index.ts +360 -156
  37. package/src/observability-metrics.ts +83 -15
  38. package/src/sandbox-resume.ts +47 -1
  39. package/src/sandbox-routing.ts +1 -1
  40. package/src/worker-service-lifecycle.ts +104 -0
  41. package/src/workflows/activities.ts +13 -7
  42. package/src/workflows/rig-verification.ts +3 -1
  43. package/src/workflows/session.ts +219 -23
  44. package/src/workflows.ts +1 -0
@@ -0,0 +1,70 @@
1
+ import type { NormalizedRunCredentialMaterial } from "@opengeni/runtime";
2
+ import {
3
+ nextExpiringMaterialRenewalDelay,
4
+ startExpiringMaterialRenewalLoop,
5
+ type ExpiringMaterialRenewalController,
6
+ } from "./expiring-material-renewal";
7
+
8
+ export const RUN_CREDENTIAL_DEFAULT_REFRESH_MS = 30 * 60_000;
9
+ export const RUN_CREDENTIAL_EXPIRY_LEAD_MS = 5 * 60_000;
10
+ export const RUN_CREDENTIAL_MIN_REFRESH_MS = 5_000;
11
+ export const RUN_CREDENTIAL_MAX_RETRY_MS = 5 * 60_000;
12
+
13
+ export type RunCredentialRenewalController = ExpiringMaterialRenewalController;
14
+
15
+ type RenewalTimer = unknown;
16
+
17
+ export type RunCredentialRenewalOptions = {
18
+ initialExpiresAt: Date | null;
19
+ resolve: () => Promise<NormalizedRunCredentialMaterial | null>;
20
+ write: (material: NormalizedRunCredentialMaterial | null) => Promise<void>;
21
+ now?: () => number;
22
+ schedule?: (callback: () => void, delayMs: number) => RenewalTimer;
23
+ clearSchedule?: (timer: RenewalTimer) => void;
24
+ onSuccess?: (result: { nextDelayMs: number; authNeeded: boolean }) => void;
25
+ onFailure?: (failure: { retryDelayMs: number; errorClass: string }) => void;
26
+ };
27
+
28
+ export function nextRunCredentialRenewalDelay(expiresAt: Date | null, nowMs = Date.now()): number {
29
+ return nextExpiringMaterialRenewalDelay(
30
+ expiresAt,
31
+ {
32
+ defaultRefreshMs: RUN_CREDENTIAL_DEFAULT_REFRESH_MS,
33
+ expiryLeadMs: RUN_CREDENTIAL_EXPIRY_LEAD_MS,
34
+ minRefreshMs: RUN_CREDENTIAL_MIN_REFRESH_MS,
35
+ maxRetryMs: RUN_CREDENTIAL_MAX_RETRY_MS,
36
+ },
37
+ nowMs,
38
+ );
39
+ }
40
+
41
+ /** Single-flight renewal; late resolves cannot mutate a stopped attempt. */
42
+ export function startRunCredentialRenewalLoop(
43
+ options: RunCredentialRenewalOptions,
44
+ ): RunCredentialRenewalController {
45
+ return startExpiringMaterialRenewalLoop<NormalizedRunCredentialMaterial | null>({
46
+ initialExpiresAt: options.initialExpiresAt,
47
+ resolve: options.resolve,
48
+ write: options.write,
49
+ expiresAt: (material) => material?.expiresAt ?? null,
50
+ policy: {
51
+ defaultRefreshMs: RUN_CREDENTIAL_DEFAULT_REFRESH_MS,
52
+ expiryLeadMs: RUN_CREDENTIAL_EXPIRY_LEAD_MS,
53
+ minRefreshMs: RUN_CREDENTIAL_MIN_REFRESH_MS,
54
+ maxRetryMs: RUN_CREDENTIAL_MAX_RETRY_MS,
55
+ },
56
+ ...(options.now ? { now: options.now } : {}),
57
+ ...(options.schedule ? { schedule: options.schedule } : {}),
58
+ ...(options.clearSchedule ? { clearSchedule: options.clearSchedule } : {}),
59
+ ...(options.onSuccess
60
+ ? {
61
+ onSuccess: ({ material, nextDelayMs }) =>
62
+ options.onSuccess?.({
63
+ nextDelayMs,
64
+ authNeeded: (material?.authNeeded.length ?? 0) > 0,
65
+ }),
66
+ }
67
+ : {}),
68
+ ...(options.onFailure ? { onFailure: options.onFailure } : {}),
69
+ });
70
+ }
@@ -0,0 +1,132 @@
1
+ import type {
2
+ ConnectionCredentialsPort,
3
+ CredentialAuthNeededPayload,
4
+ SandboxBackend,
5
+ RunCredentialsRequest,
6
+ Session,
7
+ SessionTurn,
8
+ } from "@opengeni/contracts";
9
+ import { getSessionRootId, type Database } from "@opengeni/db";
10
+ import {
11
+ normalizeRunCredentialsResolution,
12
+ type NormalizedRunCredentialMaterial,
13
+ } from "@opengeni/runtime";
14
+
15
+ export type RunCredentialResolutionContext = {
16
+ db: Database;
17
+ connectionCredentials?: ConnectionCredentialsPort | null;
18
+ accountId: string;
19
+ workspaceId: string;
20
+ session: Session;
21
+ turn: SessionTurn;
22
+ attemptId: string;
23
+ effectiveSandboxBackend: SandboxBackend;
24
+ variableSet: { id: string; name: string } | null;
25
+ };
26
+
27
+ export type BoundRunCredentialResolver = {
28
+ resolve(input: {
29
+ purpose: "provision" | "renewal";
30
+ forceRefresh: boolean;
31
+ }): Promise<NormalizedRunCredentialMaterial | null>;
32
+ };
33
+
34
+ export function buildRunCredentialsRequest(
35
+ input: Omit<RunCredentialResolutionContext, "db" | "connectionCredentials"> & {
36
+ rootSessionId: string;
37
+ purpose: "provision" | "renewal";
38
+ forceRefresh: boolean;
39
+ },
40
+ ): RunCredentialsRequest {
41
+ return {
42
+ accountId: input.accountId,
43
+ workspaceId: input.workspaceId,
44
+ sessionId: input.session.id,
45
+ parentSessionId: input.session.parentSessionId,
46
+ rootSessionId: input.rootSessionId,
47
+ sandboxGroupId: input.session.sandboxGroupId,
48
+ turnId: input.turn.id,
49
+ attemptId: input.attemptId,
50
+ executionGeneration: input.turn.executionGeneration,
51
+ initiator: input.turn.initiator,
52
+ initiatorContext: input.turn.initiatorContext,
53
+ effectiveSandboxBackend: input.effectiveSandboxBackend,
54
+ sandboxOs: input.turn.sandboxOs ?? input.session.sandboxOs,
55
+ purpose: input.purpose,
56
+ forceRefresh: input.forceRefresh,
57
+ variableSet: input.variableSet,
58
+ };
59
+ }
60
+
61
+ export function runCredentialAuthNeededPayloads(
62
+ material: NormalizedRunCredentialMaterial,
63
+ ): CredentialAuthNeededPayload[] {
64
+ return material.authNeeded.map((notice) => ({
65
+ credentialClass: "run",
66
+ reason: notice.reason,
67
+ ...(notice.providerDomain ? { providerDomain: notice.providerDomain } : {}),
68
+ ...(notice.connectionId ? { connectionId: notice.connectionId } : {}),
69
+ ...(notice.scopes?.length ? { scopes: notice.scopes } : {}),
70
+ ...(notice.resource ? { resource: notice.resource } : {}),
71
+ ...(notice.authorizationUrl ? { authorizationUrl: notice.authorizationUrl } : {}),
72
+ ...(notice.message ? { message: notice.message } : {}),
73
+ }));
74
+ }
75
+
76
+ export function runCredentialModelNote(
77
+ material: NormalizedRunCredentialMaterial,
78
+ ): string | undefined {
79
+ if (material.authNeeded.length === 0) return undefined;
80
+ return [
81
+ "[OpenGeni connected-service status]",
82
+ "One or more host-managed credentials need user attention. Continue with available capabilities, but do not claim the affected service is usable until it is reconnected.",
83
+ JSON.stringify({
84
+ credentials: material.authNeeded.map((notice) => ({
85
+ reason: notice.reason,
86
+ ...(notice.providerDomain ? { providerDomain: notice.providerDomain } : {}),
87
+ ...(notice.resource ? { resource: notice.resource } : {}),
88
+ ...(notice.message ? { message: notice.message } : {}),
89
+ })),
90
+ }),
91
+ ].join("\n");
92
+ }
93
+
94
+ /**
95
+ * Freeze the provider-neutral host credential request to this exact admitted
96
+ * turn. The host selects connections and material; the worker never infers a
97
+ * provider from repositories, environment names, or an OpenGeni variable set.
98
+ */
99
+ export async function bindRunCredentialResolver(
100
+ input: RunCredentialResolutionContext,
101
+ ): Promise<BoundRunCredentialResolver | null> {
102
+ const resolver = input.connectionCredentials?.runCredentials;
103
+ if (!resolver) return null;
104
+ const rootSessionId = await getSessionRootId(input.db, input.workspaceId, input.session.id);
105
+ if (!rootSessionId) {
106
+ throw new Error(`cannot resolve run credentials for missing session ${input.session.id}`);
107
+ }
108
+ const scope = {
109
+ accountId: input.accountId,
110
+ workspaceId: input.workspaceId,
111
+ sessionId: input.session.id,
112
+ };
113
+ return {
114
+ resolve: async ({ purpose, forceRefresh }) => {
115
+ const resolution = await resolver(
116
+ buildRunCredentialsRequest({
117
+ accountId: input.accountId,
118
+ workspaceId: input.workspaceId,
119
+ session: input.session,
120
+ turn: input.turn,
121
+ attemptId: input.attemptId,
122
+ effectiveSandboxBackend: input.effectiveSandboxBackend,
123
+ variableSet: input.variableSet,
124
+ rootSessionId,
125
+ purpose,
126
+ forceRefresh,
127
+ }),
128
+ );
129
+ return normalizeRunCredentialsResolution(resolution, scope);
130
+ },
131
+ };
132
+ }
@@ -2,6 +2,7 @@ import type { FileAsset, ResourceRef, SessionSystemUpdate } from "@opengeni/cont
2
2
  import {
3
3
  getActiveSessionHistoryItems,
4
4
  getLatestRunState,
5
+ getHumanInputResumeForEvent,
5
6
  getSandboxSessionEnvelope,
6
7
  getSessionEvent,
7
8
  listSessionSystemUpdatesForTurn,
@@ -144,6 +145,7 @@ export type TurnInputOptions = {
144
145
  turnId: string;
145
146
  recovering?: boolean;
146
147
  unavailableSandboxFilesNote?: string;
148
+ runCredentialsNote?: string;
147
149
  };
148
150
 
149
151
  export async function turnInput(
@@ -173,6 +175,7 @@ export async function turnInput(
173
175
  : undefined,
174
176
  updateContext,
175
177
  options.unavailableSandboxFilesNote,
178
+ options.runCredentialsNote,
176
179
  );
177
180
  if (trigger.type === "user.message") {
178
181
  const payload = trigger.payload as { text?: unknown; resources?: unknown };
@@ -228,6 +231,26 @@ export async function turnInput(
228
231
  modelHistoryFromItems: false,
229
232
  };
230
233
  }
234
+ if (trigger.type === "user.humanInputResponse") {
235
+ const [state, resume] = await Promise.all([
236
+ getLatestRunState(db, trigger.workspaceId, trigger.sessionId),
237
+ getHumanInputResumeForEvent(db, trigger.workspaceId, trigger.sessionId, trigger),
238
+ ]);
239
+ if (!state) {
240
+ throw new Error("No saved run state is available for human-input response");
241
+ }
242
+ if (!resume) {
243
+ throw new Error("Human-input response does not resolve to a durable request");
244
+ }
245
+ return {
246
+ input: await runtime.prepareInput(agent, {
247
+ kind: "human_input",
248
+ serializedRunState: resumeRunStateForCodexAccount(state, current),
249
+ toolCallId: resume.toolCallId,
250
+ }),
251
+ modelHistoryFromItems: false,
252
+ };
253
+ }
231
254
  throw new Error(`Unsupported trigger event type: ${trigger.type}`);
232
255
  }
233
256
 
@@ -63,6 +63,7 @@ import {
63
63
  isProviderSandboxNotFoundError,
64
64
  sweepModalOrphanSandboxes,
65
65
  terminateModalSandboxById,
66
+ type ModalOrphanSweepTermination,
66
67
  } from "@opengeni/runtime";
67
68
  import type { ActivityServices } from "./types";
68
69
  import { reconcilePendingParentSystemUpdates } from "./parent-wake";
@@ -447,6 +448,31 @@ async function forceDrainOverLimitWorkspaces(
447
448
  return forceDrained;
448
449
  }
449
450
 
451
+ export function modalOrphanTerminationStillEligible(
452
+ latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
453
+ candidate: ModalOrphanSweepTermination,
454
+ ): boolean {
455
+ if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
456
+ return false;
457
+ }
458
+ const leaseId = candidate.tags.opengeni_lease_id;
459
+ const workspaceId = candidate.tags.opengeni_workspace_id;
460
+ const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;
461
+ if (!leaseId || !workspaceId || !sandboxGroupId) {
462
+ return true;
463
+ }
464
+ const activeAttribution = latest.find(
465
+ (lease) =>
466
+ lease.leaseId === leaseId &&
467
+ lease.workspaceId === workspaceId &&
468
+ lease.sandboxGroupId === sandboxGroupId,
469
+ );
470
+ return Boolean(
471
+ !activeAttribution ||
472
+ (activeAttribution.instanceId !== null && activeAttribution.instanceId !== candidate.sandboxId),
473
+ );
474
+ }
475
+
450
476
  async function sweepModalOrphansForConfiguredBackend(
451
477
  settings: ActivityServices["settings"],
452
478
  db: ActivityServices["db"],
@@ -468,7 +494,12 @@ async function sweepModalOrphansForConfiguredBackend(
468
494
  return 0;
469
495
  }
470
496
 
471
- const result = await sweepModalOrphanSandboxes(settings, liveLeases);
497
+ const result = await sweepModalOrphanSandboxes(settings, liveLeases, {
498
+ revalidateTermination: async (candidate) => {
499
+ const latest = await listLiveModalSandboxLeaseAttributions(db);
500
+ return modalOrphanTerminationStillEligible(latest, candidate);
501
+ },
502
+ });
472
503
  for (const terminated of result.terminated) {
473
504
  observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
474
505
  sandboxId: terminated.sandboxId,
@@ -647,7 +678,7 @@ export async function terminateProviderBox(
647
678
  return true;
648
679
  }
649
680
 
650
- // CRITICAL SAFETY (bring-your-own-compute, dossier §19/§21): NEVER provider-stop
681
+ // CRITICAL SAFETY (bring-your-own-compute): NEVER provider-stop
651
682
  // a selfhosted box. The "box" is a user's PHYSICAL machine — you cannot kill it,
652
683
  // and a delete()/kill() reaching the agent would be catastrophic. The reaper
653
684
  // DRAINS the lease to cold only (refcount→0 → draining → this returns true so
@@ -77,6 +77,9 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
77
77
  unit: "run",
78
78
  sourceResourceType: "scheduled_task_run",
79
79
  sourceResourceId: run.id,
80
+ initiator: { kind: "service", subjectId: "scheduler" },
81
+ initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
82
+ origin: "scheduled_task",
80
83
  idempotencyKey: `usage:scheduled_task.fired:${run.id}`,
81
84
  });
82
85
  if (run.status === "dispatched" && run.sessionId && run.triggerEventId) {
@@ -88,6 +91,10 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
88
91
  unit: "run",
89
92
  sourceResourceType: "scheduled_task_run",
90
93
  sourceResourceId: run.id,
94
+ sessionId: run.sessionId,
95
+ initiator: { kind: "service", subjectId: "scheduler" },
96
+ initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
97
+ origin: "scheduled_task",
91
98
  idempotencyKey:
92
99
  input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`,
93
100
  });
@@ -164,6 +171,15 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
164
171
  scheduledTaskId: task.id,
165
172
  scheduledTaskRunId: run.id,
166
173
  },
174
+ createdBy: {
175
+ kind: "service",
176
+ subjectId: "scheduler",
177
+ label: "OpenGeni scheduler",
178
+ },
179
+ createdByContext: {
180
+ scheduledTaskId: task.id,
181
+ scheduledTaskRunId: run.id,
182
+ },
167
183
  model,
168
184
  sandboxBackend,
169
185
  variableSetId: task.variableSetId ?? null,
@@ -193,6 +209,7 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
193
209
  type: "session.created",
194
210
  payload: {
195
211
  status: session.status,
212
+ createdBy: session.createdBy,
196
213
  scheduledTaskId: task.id,
197
214
  scheduledTaskRunId: run.id,
198
215
  // Names/ids only; never values.
@@ -390,6 +407,10 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
390
407
  unit: "run",
391
408
  sourceResourceType: "scheduled_task_run",
392
409
  sourceResourceId: run.id,
410
+ sessionId: result.sessionId,
411
+ initiator: { kind: "service", subjectId: "scheduler" },
412
+ initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
413
+ origin: "scheduled_task",
393
414
  idempotencyKey:
394
415
  input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`,
395
416
  });
@@ -6,6 +6,8 @@ import {
6
6
  countQueuedTurns,
7
7
  getSessionEvent,
8
8
  getSessionTurnForAttempt,
9
+ expireSessionHumanInputRequest,
10
+ markSessionAttemptQuiesced,
9
11
  requireSession,
10
12
  settleSessionIdleWithParentOutbox,
11
13
  } from "@opengeni/db";
@@ -14,10 +16,13 @@ import { deliverFailedChildTurnToParent, notifyParentOfChildIdle } from "./paren
14
16
  import { recordTurnsQueuedGauge } from "../observability-metrics";
15
17
  import type {
16
18
  ActivityServices,
19
+ ExpireSessionHumanInputInput,
20
+ ExpireSessionHumanInputResult,
17
21
  PeekSessionWorkInput,
18
22
  FailSessionAttemptInput,
19
23
  SettleSessionInterruptionsInput,
20
24
  MarkSessionIdleInput,
25
+ PersistSessionAttemptQuiescenceInput,
21
26
  RecoverDispatchInput,
22
27
  RecoverDispatchResult,
23
28
  } from "./types";
@@ -30,8 +35,10 @@ export type SessionStateActivityOverrides = Partial<{
30
35
  countQueuedTurns: typeof countQueuedTurns;
31
36
  getSessionEvent: typeof getSessionEvent;
32
37
  getSessionTurnForAttempt: typeof getSessionTurnForAttempt;
38
+ expireSessionHumanInputRequest: typeof expireSessionHumanInputRequest;
33
39
  requireSession: typeof requireSession;
34
40
  settleSessionIdleWithParentOutbox: typeof settleSessionIdleWithParentOutbox;
41
+ markSessionAttemptQuiesced: typeof markSessionAttemptQuiesced;
35
42
  publishDurableSessionEvents: typeof publishDurableSessionEvents;
36
43
  deliverFailedChildTurnToParent: typeof deliverFailedChildTurnToParent;
37
44
  notifyParentOfChildIdle: typeof notifyParentOfChildIdle;
@@ -57,9 +64,13 @@ export function createSessionStateActivities(
57
64
  const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns;
58
65
  const getSessionEventFn = overrides.getSessionEvent ?? getSessionEvent;
59
66
  const getSessionTurnForAttemptFn = overrides.getSessionTurnForAttempt ?? getSessionTurnForAttempt;
67
+ const expireSessionHumanInputRequestFn =
68
+ overrides.expireSessionHumanInputRequest ?? expireSessionHumanInputRequest;
60
69
  const requireSessionFn = overrides.requireSession ?? requireSession;
61
70
  const settleSessionIdleWithParentOutboxFn =
62
71
  overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
72
+ const markSessionAttemptQuiescedFn =
73
+ overrides.markSessionAttemptQuiesced ?? markSessionAttemptQuiesced;
63
74
  const publishDurableSessionEventsFn =
64
75
  overrides.publishDurableSessionEvents ?? publishDurableSessionEvents;
65
76
  const deliverFailedChildTurnToParentFn =
@@ -117,6 +128,19 @@ export function createSessionStateActivities(
117
128
  input: SettleSessionInterruptionsInput,
118
129
  ): Promise<{ action: "paused" | "continue" | "stale" }> {
119
130
  const { db, bus, observability } = await services();
131
+ if (input.phase === "attempt_quiesced") {
132
+ // Replay compatibility only: v1 histories scheduled this idempotent
133
+ // fallback after WAIT_CANCELLATION_COMPLETED. Receipt-gated v2 workflows
134
+ // never call it; runAgentTurn writes immediately after its hard fence.
135
+ const events = await markSessionAttemptQuiescedFn(db, {
136
+ workspaceId: input.workspaceId,
137
+ sessionId: input.sessionId,
138
+ attemptId: input.attemptId,
139
+ temporalWorkflowId: input.workflowId,
140
+ });
141
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, events);
142
+ return { action: "stale" };
143
+ }
120
144
  const applied = await settleSessionAttemptInterruptionsFn(
121
145
  db,
122
146
  input.workspaceId,
@@ -130,6 +154,38 @@ export function createSessionStateActivities(
130
154
  return { action: applied.action };
131
155
  }
132
156
 
157
+ /** Persist an exact activity-owned physical-quiescence proof through the
158
+ * workflow control-activity retry policy. The DB transaction remains the
159
+ * sole receipt/wake authority; duplicate signals and activity retries reuse
160
+ * its attempt-scoped idempotency key. */
161
+ async function persistSessionAttemptQuiescence(
162
+ input: PersistSessionAttemptQuiescenceInput,
163
+ ): Promise<void> {
164
+ const { db, bus, observability } = await services();
165
+ const events = await markSessionAttemptQuiescedFn(db, {
166
+ accountId: input.accountId,
167
+ workspaceId: input.workspaceId,
168
+ sessionId: input.sessionId,
169
+ attemptId: input.attemptId,
170
+ temporalWorkflowId: input.workflowId,
171
+ temporalWorkflowRunId: input.workflowRunId,
172
+ temporalActivityId: input.activityId,
173
+ allowUninterrupted: true,
174
+ });
175
+ try {
176
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, events);
177
+ } catch (error) {
178
+ // The receipt and exact workflow wake already committed atomically in
179
+ // Postgres. NATS is best-effort live fanout and must not keep this
180
+ // control activity retrying or delay receipt-gated admission.
181
+ observability.error("session-attempt quiescence event fanout failed", {
182
+ "opengeni.session_id": input.sessionId,
183
+ "opengeni.attempt_id": input.attemptId,
184
+ error: error instanceof Error ? error.message : String(error),
185
+ });
186
+ }
187
+ }
188
+
133
189
  /**
134
190
  * Recover the same current inference when its worker dies without completing
135
191
  * a graceful checkpoint (heartbeat timeout, SIGKILL, OOM, or node loss).
@@ -177,6 +233,20 @@ export function createSessionStateActivities(
177
233
  return peek;
178
234
  }
179
235
 
236
+ async function expireSessionHumanInput(
237
+ input: ExpireSessionHumanInputInput,
238
+ ): Promise<ExpireSessionHumanInputResult> {
239
+ const { db, bus } = await services();
240
+ const result = await expireSessionHumanInputRequestFn(db, input);
241
+ if (result.action === "not_found") return { action: "not_found" };
242
+ if (result.events.length > 0) {
243
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
244
+ }
245
+ return {
246
+ action: result.request.status === "expired" ? "expired" : "stale",
247
+ };
248
+ }
249
+
180
250
  async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
181
251
  const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
182
252
  const settled = await settleSessionIdleWithParentOutboxFn(
@@ -207,8 +277,10 @@ export function createSessionStateActivities(
207
277
  return {
208
278
  failSessionAttempt,
209
279
  settleSessionInterruptions,
280
+ persistSessionAttemptQuiescence,
210
281
  recoverDispatch,
211
282
  peekSessionWork,
283
+ expireSessionHumanInput,
212
284
  markSessionIdle,
213
285
  };
214
286
  }
@@ -53,6 +53,7 @@ export function createRuntimeBatcher(
53
53
  "agent.toolCall.output",
54
54
  "agent.message.completed",
55
55
  "tool.auth_needed",
56
+ "credential.auth_needed",
56
57
  "session.requiresAction",
57
58
  "turn.completed",
58
59
  "turn.failed",
@@ -0,0 +1,57 @@
1
+ import {
2
+ startExpiringMaterialRenewalLoop,
3
+ type ExpiringMaterialRenewalController,
4
+ } from "./expiring-material-renewal";
5
+ import type { MintedSandboxToolspaceToken } from "./environment";
6
+
7
+ export const TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS = 30 * 60_000;
8
+ export const TOOLSPACE_TOKEN_EXPIRY_LEAD_MS = 5 * 60_000;
9
+ export const TOOLSPACE_TOKEN_MIN_REFRESH_MS = 5_000;
10
+ export const TOOLSPACE_TOKEN_MAX_RETRY_MS = 5 * 60_000;
11
+
12
+ export type ToolspaceTokenRenewalController = ExpiringMaterialRenewalController;
13
+
14
+ export type ToolspaceTokenRenewalOptions = {
15
+ initialExpiresAt: Date;
16
+ mint: () => Promise<MintedSandboxToolspaceToken | undefined>;
17
+ write: (material: MintedSandboxToolspaceToken) => Promise<void>;
18
+ now?: () => number;
19
+ schedule?: (callback: () => void, delayMs: number) => unknown;
20
+ clearSchedule?: (timer: unknown) => void;
21
+ onSuccess?: (result: { nextDelayMs: number }) => void;
22
+ onFailure?: (failure: { retryDelayMs: number; errorClass: string }) => void;
23
+ };
24
+
25
+ class ToolspaceTokenRenewalError extends Error {
26
+ override readonly name = "ToolspaceTokenRenewalError";
27
+ }
28
+
29
+ export function startToolspaceTokenRenewalLoop(
30
+ options: ToolspaceTokenRenewalOptions,
31
+ ): ToolspaceTokenRenewalController {
32
+ return startExpiringMaterialRenewalLoop<MintedSandboxToolspaceToken>({
33
+ initialExpiresAt: options.initialExpiresAt,
34
+ resolve: async () => {
35
+ const minted = await options.mint();
36
+ if (!minted) {
37
+ throw new ToolspaceTokenRenewalError("Toolspace token mint is no longer available");
38
+ }
39
+ return minted;
40
+ },
41
+ write: options.write,
42
+ expiresAt: (material) => material.expiresAt,
43
+ policy: {
44
+ defaultRefreshMs: TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS,
45
+ expiryLeadMs: TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
46
+ minRefreshMs: TOOLSPACE_TOKEN_MIN_REFRESH_MS,
47
+ maxRetryMs: TOOLSPACE_TOKEN_MAX_RETRY_MS,
48
+ },
49
+ ...(options.now ? { now: options.now } : {}),
50
+ ...(options.schedule ? { schedule: options.schedule } : {}),
51
+ ...(options.clearSchedule ? { clearSchedule: options.clearSchedule } : {}),
52
+ ...(options.onSuccess
53
+ ? { onSuccess: ({ nextDelayMs }) => options.onSuccess?.({ nextDelayMs }) }
54
+ : {}),
55
+ ...(options.onFailure ? { onFailure: options.onFailure } : {}),
56
+ });
57
+ }
@@ -36,6 +36,21 @@ export type SignalCodexCapacityWorkflow = (input: {
36
36
  wakeRevision: number;
37
37
  }) => Promise<void>;
38
38
 
39
+ /** Exact activity-owned proof that the hard sandbox/tool fence physically
40
+ * drained. This is delivery evidence only: the workflow still validates the
41
+ * persisted attempt dispatch and commits the authoritative Postgres receipt. */
42
+ export type SessionAttemptQuiescenceProof = {
43
+ accountId: string;
44
+ workspaceId: string;
45
+ sessionId: string;
46
+ attemptId: string;
47
+ workflowId: string;
48
+ workflowRunId: string;
49
+ activityId: string;
50
+ };
51
+
52
+ export type SignalSessionAttemptQuiesced = (input: SessionAttemptQuiescenceProof) => Promise<void>;
53
+
39
54
  export type ActivityServices = {
40
55
  settings: Settings;
41
56
  db: Database;
@@ -45,6 +60,9 @@ export type ActivityServices = {
45
60
  documentServices: DocumentServices;
46
61
  observability: Observability;
47
62
  wakeSessionWorkflow: WakeSessionWorkflowSignal | null;
63
+ /** Durable signalWithStart fallback used only after the activity's direct
64
+ * physical-quiescence receipt write exhausts its bounded DB retries. */
65
+ signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced | null;
48
66
  /** Revision-carrying capacity nudge; generic outbox repair is also sufficient. */
49
67
  signalCodexCapacityWorkflow?: SignalCodexCapacityWorkflow | null;
50
68
  // §7.5 P3 — host-entitlements port, the WORKER half of the same seam the API
@@ -61,7 +79,7 @@ export type ActivityServices = {
61
79
  // admission and metering are separate operations, and only metering carries
62
80
  // the idempotency key.
63
81
  entitlements?: EntitlementsPort | null;
64
- // §7.6 P4a — host connection-credential provider, the WORKER half of the
82
+ // §7.6 connection-credential provider — host connection-credential provider, the WORKER half of the
65
83
  // federated-connection boundary. When set, the run's per-run credential mint
66
84
  // delegates to the host instead of self-minting from `settings`:
67
85
  // - `gitCredentials` REPLACES `createGitHubAppInstallationToken(settings,…)`
@@ -72,7 +90,7 @@ export type ActivityServices = {
72
90
  // self-mint for THAT leg. null/undefined (standalone default) → both legs
73
91
  // self-mint byte-for-byte as today.
74
92
  //
75
- // FORK-7 CROSS-CHECK: a provider echoes the `workspaceId` it scoped the
93
+ // workspace-scope cross-check CROSS-CHECK: a provider echoes the `workspaceId` it scoped the
76
94
  // credential to; the consuming activity ASSERTS agreement with the run's
77
95
  // workspace BEFORE injecting `GH_TOKEN` (or applying decrypted values). A host
78
96
  // mapping bug returning tenant B's creds for a tenant-A run is caught here.
@@ -123,8 +141,16 @@ export type SettleSessionInterruptionsInput = {
123
141
  sessionId: string;
124
142
  attemptId: string;
125
143
  workflowId: string;
144
+ /**
145
+ * Replay-only compatibility for session workflow histories created before
146
+ * the receipt-gated cancellation v2 patch. New histories never send this
147
+ * phase; the exact activity writes the authoritative receipt itself.
148
+ */
149
+ phase?: "logical" | "attempt_quiesced";
126
150
  };
127
151
 
152
+ export type PersistSessionAttemptQuiescenceInput = SessionAttemptQuiescenceProof;
153
+
128
154
  export type FailSessionAttemptInput = {
129
155
  accountId: string;
130
156
  workspaceId: string;
@@ -160,6 +186,17 @@ export type PeekSessionWorkInput = {
160
186
  sessionId: string;
161
187
  };
162
188
 
189
+ export type ExpireSessionHumanInputInput = {
190
+ accountId: string;
191
+ workspaceId: string;
192
+ sessionId: string;
193
+ requestId: string;
194
+ };
195
+
196
+ export type ExpireSessionHumanInputResult = {
197
+ action: "expired" | "stale" | "not_found";
198
+ };
199
+
163
200
  export type MarkSessionIdleInput = {
164
201
  workspaceId: string;
165
202
  sessionId: string;
@@ -220,10 +257,11 @@ type ClaimedRunAgentTurnResult = {
220
257
  // persisted in Postgres and reconstructed after workflow/worker restart.
221
258
  // The workflow must not call maybeContinueGoal while this waiter is active.
222
259
  capacityWait?: CodexCapacityWaitRef;
223
- // A maintenance execution could not run (for example, every Codex account
224
- // is unavailable). End this workflow run without consuming the durable
225
- // request; a later prompt/control wake starts the workflow again and normal
226
- // prompt claim ordering consumes the request in-turn.
260
+ // This execution reached a durable terminal-for-now boundary (for example,
261
+ // maintenance could not run or same-turn context recovery failed). End this
262
+ // workflow run without synthesizing another goal continuation from unchanged
263
+ // state. A later prompt/control/new-update wake may retry through normal claim
264
+ // ordering.
227
265
  deferredUntilWake?: boolean;
228
266
  };
229
267