@opengeni/worker-bundle 0.23.4 → 0.26.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +17 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  3. package/dist/activities/agent-turn/claim.d.ts +8 -0
  4. package/dist/activities/agent-turn/codex.d.ts +2 -2
  5. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  6. package/dist/activities/agent-turn/errors.d.ts +1 -1
  7. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +15 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +11 -6
  11. package/dist/activities/agent-turn/run.d.ts +9 -0
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
  13. package/dist/activities/agent-turn/session-title.d.ts +21 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +4 -13
  16. package/dist/activities/agent-turn/tool-policy.d.ts +40 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +13 -2
  18. package/dist/activities/agent-turn.d.ts +1 -1
  19. package/dist/activities/automations.d.ts +1 -0
  20. package/dist/activities/capabilities.d.ts +3 -1
  21. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  22. package/dist/activities/codex-capacity.d.ts +32 -3
  23. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  24. package/dist/activities/codex-image-generation.d.ts +1 -1
  25. package/dist/activities/codex-rotation.d.ts +8 -67
  26. package/dist/activities/context-compaction.d.ts +2 -0
  27. package/dist/activities/goals.d.ts +12 -1
  28. package/dist/activities/google-drive-publication.d.ts +2 -13
  29. package/dist/activities/image-generation-operation.d.ts +13 -4
  30. package/dist/activities/image-generation-references.d.ts +2 -1
  31. package/dist/activities/scheduled-tasks.d.ts +1 -0
  32. package/dist/activities/session-state.d.ts +4 -2
  33. package/dist/activities/types.d.ts +19 -1
  34. package/dist/{activities-control-52W5IDWM.js → activities-control-CTEMLTHG.js} +339 -195
  35. package/dist/activities-control-CTEMLTHG.js.map +1 -0
  36. package/dist/activities-control.d.ts +4 -0
  37. package/dist/{activities-turn-A2G2KLSY.js → activities-turn-S5IU347A.js} +1846 -1136
  38. package/dist/activities-turn-S5IU347A.js.map +1 -0
  39. package/dist/activities.d.ts +4 -0
  40. package/dist/{chunk-GFNXVZVQ.js → chunk-D4ZEFOYH.js} +107 -7
  41. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  42. package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
  43. package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
  44. package/dist/{chunk-IPVMCAW5.js → chunk-TAUE3GYT.js} +225 -154
  45. package/dist/chunk-TAUE3GYT.js.map +1 -0
  46. package/dist/github-rest-mcp.d.ts +0 -2
  47. package/dist/index.d.ts +2 -0
  48. package/dist/index.js +43 -7
  49. package/dist/index.js.map +1 -1
  50. package/dist/observability-metrics.d.ts +35 -8
  51. package/dist/sandbox-reaper-timeout.d.ts +2 -2
  52. package/dist/sandbox-resume.d.ts +6 -1
  53. package/dist/workflow-bundle.js +96 -10
  54. package/dist/workflows/activities.d.ts +6 -1
  55. package/dist/workflows/automations.d.ts +5 -0
  56. package/dist/workflows/session.d.ts +3 -0
  57. package/package.json +17 -17
  58. package/src/activities/agent-run-admission.ts +7 -1
  59. package/src/activities/agent-turn/admission.ts +47 -6
  60. package/src/activities/agent-turn/agent-build.ts +20 -26
  61. package/src/activities/agent-turn/claim.ts +59 -6
  62. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  63. package/src/activities/agent-turn/codex.ts +2 -2
  64. package/src/activities/agent-turn/compaction-prep.ts +8 -1
  65. package/src/activities/agent-turn/credential-leases.ts +36 -0
  66. package/src/activities/agent-turn/errors.ts +29 -1
  67. package/src/activities/agent-turn/failure-settlement.ts +451 -323
  68. package/src/activities/agent-turn/finalization.ts +31 -17
  69. package/src/activities/agent-turn/list-models.ts +101 -0
  70. package/src/activities/agent-turn/model-usage.ts +120 -45
  71. package/src/activities/agent-turn/quiescence.ts +32 -17
  72. package/src/activities/agent-turn/run.ts +98 -9
  73. package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
  74. package/src/activities/agent-turn/session-title.ts +55 -2
  75. package/src/activities/agent-turn/stream-attempt.ts +281 -136
  76. package/src/activities/agent-turn/tool-environment.ts +132 -39
  77. package/src/activities/agent-turn/tool-policy.ts +71 -12
  78. package/src/activities/agent-turn/turn-context.ts +21 -2
  79. package/src/activities/agent-turn.ts +5 -1
  80. package/src/activities/automations.ts +122 -67
  81. package/src/activities/capabilities.ts +129 -2
  82. package/src/activities/codemode-dispatcher.ts +235 -59
  83. package/src/activities/codex-capacity.ts +89 -10
  84. package/src/activities/codex-fleet-shadow.ts +8 -1
  85. package/src/activities/codex-image-generation.ts +19 -2
  86. package/src/activities/codex-rotation.ts +45 -142
  87. package/src/activities/context-compaction.ts +3 -0
  88. package/src/activities/goals.ts +116 -39
  89. package/src/activities/google-drive-publication.ts +4 -81
  90. package/src/activities/image-generation-operation.ts +28 -3
  91. package/src/activities/image-generation-references.ts +9 -9
  92. package/src/activities/mcp-credentials.ts +53 -0
  93. package/src/activities/rig-verification.ts +5 -3
  94. package/src/activities/sandbox-lease.ts +29 -12
  95. package/src/activities/scheduled-tasks.ts +217 -110
  96. package/src/activities/session-state.ts +17 -0
  97. package/src/activities/site-auth-maintenance.ts +12 -12
  98. package/src/activities/types.ts +22 -8
  99. package/src/activities.ts +2 -0
  100. package/src/activity-services.ts +4 -1
  101. package/src/github-rest-mcp.ts +2 -83
  102. package/src/index.ts +34 -4
  103. package/src/observability-metrics.ts +158 -14
  104. package/src/sandbox-reaper-timeout.ts +10 -4
  105. package/src/sandbox-resume.ts +58 -8
  106. package/src/sandbox-routing.ts +71 -16
  107. package/src/workflows/activities.ts +16 -0
  108. package/src/workflows/automations.ts +21 -2
  109. package/src/workflows/session.ts +85 -7
  110. package/dist/activities/api-integrations.d.ts +0 -21
  111. package/dist/activities-control-52W5IDWM.js.map +0 -1
  112. package/dist/activities-turn-A2G2KLSY.js.map +0 -1
  113. package/dist/chunk-GFNXVZVQ.js.map +0 -1
  114. package/dist/chunk-IPVMCAW5.js.map +0 -1
  115. package/src/activities/api-integrations.ts +0 -141
@@ -10,6 +10,7 @@ import {
10
10
  normalizeAutomaticSessionTitle,
11
11
  scheduledOccurrencePayloadUtf8Bytes,
12
12
  stableJson,
13
+ TurnExecutionPolicyV1,
13
14
  type ScheduledTask,
14
15
  type ScheduledTaskRun,
15
16
  } from "@opengeni/contracts";
@@ -17,7 +18,10 @@ import {
17
18
  defaultSessionMcpServerIds,
18
19
  openGeniSlackBotMetadata,
19
20
  requireOpenGeniSlackBotConnection,
21
+ resolveWorkspaceCatalogSettings,
20
22
  resolveSessionToolPolicy,
23
+ workspaceCustomModelReference,
24
+ lockActiveCustomModelForAdmission,
21
25
  scheduledSlackBotConnectionId,
22
26
  swapActiveSandbox,
23
27
  } from "@opengeni/core";
@@ -67,8 +71,9 @@ import {
67
71
  withSessionActivityRlsContext,
68
72
  } from "@opengeni/db";
69
73
  import { publishDurableSessionEvents } from "@opengeni/events";
70
- import { resolveFirstPartyMcpToolPolicy } from "@opengeni/config";
74
+ import { resolveFirstPartyMcpToolPolicy, resolveTurnExecutionPolicyV1 } from "@opengeni/config";
71
75
  import { Context } from "@temporalio/activity";
76
+ import { createHash } from "node:crypto";
72
77
  import {
73
78
  assertReusableSessionRevivable,
74
79
  scheduledUserMessagePayload,
@@ -128,6 +133,14 @@ export function scheduledTaskRunProducerKey(
128
133
  })}`;
129
134
  }
130
135
 
136
+ export function scheduledTaskGeneratedSessionCreateIdempotencyKey(producerKey: string): string {
137
+ const digest = createHash("sha256")
138
+ .update("scheduled-task-run\0", "utf8")
139
+ .update(producerKey, "utf8")
140
+ .digest("hex");
141
+ return `scheduled-task-run:${digest}`;
142
+ }
143
+
131
144
  /**
132
145
  * Display title for a session the scheduler generates for a task.
133
146
  *
@@ -234,14 +247,26 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
234
247
  dispatchScheduledTaskRun: async (
235
248
  input: DispatchScheduledTaskRunInput,
236
249
  ): Promise<DispatchScheduledTaskRunResult> => {
237
- const service = await services();
238
- const { settings, db, bus, wakeSessionWorkflow } = service;
250
+ const baseService = await services();
239
251
  // Histories created before the manual-initiator workflow patch already
240
252
  // contain this activity command. Reject their incomplete wire input here
241
253
  // so replay consumes the recorded command and the retry loop settles.
242
254
  if (input.triggerType !== "scheduled" && !input.initiator) {
243
255
  return { action: "blocked", reason: "malformed_manual_trigger" };
244
256
  }
257
+ const { db, bus, wakeSessionWorkflow } = baseService;
258
+ const settingsForTask = async (task: ScheduledTask, retainedProductModelId?: string | null) =>
259
+ (
260
+ await resolveWorkspaceCatalogSettings(
261
+ db,
262
+ baseService.catalogSourceSettings ?? baseService.settings,
263
+ {
264
+ accountId: task.accountId,
265
+ workspaceId: task.workspaceId,
266
+ ...(retainedProductModelId !== undefined ? { retainedProductModelId } : {}),
267
+ },
268
+ )
269
+ ).settings;
245
270
  const stableProducerKey = scheduledTaskRunProducerKey(input);
246
271
  const priorRun = await getScheduledTaskRunByProducerKey(db, {
247
272
  workspaceId: input.workspaceId,
@@ -275,6 +300,15 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
275
300
  });
276
301
  }
277
302
  if (priorRun.status === "queued") {
303
+ const acceptedPolicy = acceptedExecution.turnExecutionPolicy
304
+ ? TurnExecutionPolicyV1.parse(acceptedExecution.turnExecutionPolicy)
305
+ : null;
306
+ const settings = await settingsForTask(
307
+ acceptedExecution.task,
308
+ acceptedPolicy?.productModelId ??
309
+ acceptedExecution.targetSessionExecution?.model ??
310
+ acceptedExecution.resolvedModel,
311
+ );
278
312
  try {
279
313
  return await recoverBoundScheduledTaskDispatch({
280
314
  db,
@@ -444,6 +478,27 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
444
478
  taskRevisionAuthority?.subjectId ??
445
479
  taskPersonalResourceAuthoritySubjectId ??
446
480
  taskConnectionAuthoritySubjectId;
481
+ const acceptedTargetSessionId =
482
+ task.runMode === "existing_session"
483
+ ? task.targetSessionId
484
+ : task.runMode === "reusable_session"
485
+ ? task.reusableSessionId
486
+ : null;
487
+ // A targeted task whose exact session was deleted has no target to
488
+ // resolve; database admission settles that occurrence terminally
489
+ // (`scheduled_target_session_unavailable`) instead of retrying forever.
490
+ // Resolve this before the catalog so an existing session's retired
491
+ // custom Gateway model remains executable without becoming selectable
492
+ // for a new scheduled session.
493
+ const targetSessionExecutionBase = acceptedTargetSessionId
494
+ ? await getScheduledTargetSessionExecution(
495
+ db,
496
+ task.workspaceId,
497
+ acceptedTargetSessionId,
498
+ taskAuthoritySubjectId,
499
+ )
500
+ : null;
501
+ const settings = await settingsForTask(task, targetSessionExecutionBase?.model);
447
502
  const model = task.agentConfig.model ?? settings.openaiModel;
448
503
  const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
449
504
  let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
@@ -515,22 +570,6 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
515
570
  };
516
571
  })()
517
572
  : null;
518
- const acceptedTargetSessionId = generatedTarget
519
- ? null
520
- : task.runMode === "existing_session"
521
- ? task.targetSessionId
522
- : task.reusableSessionId;
523
- // A targeted task whose exact session was deleted has no target to
524
- // resolve; database admission settles that occurrence terminally
525
- // (`scheduled_target_session_unavailable`) instead of retrying forever.
526
- const targetSessionExecutionBase = acceptedTargetSessionId
527
- ? await getScheduledTargetSessionExecution(
528
- db,
529
- task.workspaceId,
530
- acceptedTargetSessionId,
531
- taskAuthoritySubjectId,
532
- )
533
- : null;
534
573
  const targetSessionExecution = targetSessionExecutionBase
535
574
  ? await (async () => {
536
575
  const [capabilityServerIds, apiIntegrationServerIds] = await Promise.all([
@@ -727,15 +766,56 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
727
766
  ) {
728
767
  throw new Error("scheduled personal-resource execution has no causal human");
729
768
  }
730
- const admissionDenial = await agentRunAdmissionDenial(service, {
731
- accountId: task.accountId,
732
- workspaceId: task.workspaceId,
733
- model: targetSessionExecution?.model ?? model,
734
- requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1,
735
- });
769
+ const admissionDenial = await agentRunAdmissionDenial(
770
+ { ...baseService, settings },
771
+ {
772
+ accountId: task.accountId,
773
+ workspaceId: task.workspaceId,
774
+ model: targetSessionExecution?.model ?? model,
775
+ requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1,
776
+ },
777
+ );
736
778
  if (admissionDenial) {
737
779
  return { action: "blocked", reason: admissionDenial };
738
780
  }
781
+ const acceptedModel = targetSessionExecution?.model ?? model;
782
+ const acceptedReasoningEffort = targetSessionExecution?.reasoningEffort ?? reasoningEffort;
783
+ const acceptedLatencyMode = targetSessionExecution?.latencyMode ?? "standard";
784
+ const acceptedCustomModel = generatedTarget
785
+ ? workspaceCustomModelReference(settings, acceptedModel)
786
+ : null;
787
+ const beforeFreshAgentRunCommit = acceptedCustomModel
788
+ ? async (tx: Database): Promise<void> => {
789
+ const active = await lockActiveCustomModelForAdmission(tx, {
790
+ accountId: task.accountId,
791
+ workspaceId: task.workspaceId,
792
+ reference: acceptedCustomModel,
793
+ });
794
+ if (!active) {
795
+ throw new ScheduledRunTerminalAuthorityError(
796
+ "scheduled_model_unavailable",
797
+ `model is not available: ${acceptedModel}`,
798
+ );
799
+ }
800
+ }
801
+ : undefined;
802
+ const turnExecutionPolicy: TurnExecutionPolicyV1 = resolveTurnExecutionPolicyV1(settings, {
803
+ modelId: acceptedModel,
804
+ requestedModelId: generatedTarget && task.agentConfig.model ? task.agentConfig.model : null,
805
+ modelSource: generatedTarget
806
+ ? task.agentConfig.model
807
+ ? "explicit"
808
+ : "deployment"
809
+ : "session",
810
+ reasoningEffort: acceptedReasoningEffort,
811
+ reasoningSource: generatedTarget
812
+ ? task.agentConfig.reasoningEffort
813
+ ? "explicit"
814
+ : "deployment"
815
+ : "session",
816
+ latencyMode: acceptedLatencyMode,
817
+ latencyModeSource: generatedTarget ? "deployment" : "session",
818
+ });
739
819
  const deferredEvents: Array<{
740
820
  sessionId: string;
741
821
  events: Awaited<ReturnType<typeof appendSessionEvents>>;
@@ -748,7 +828,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
748
828
  alertResponderSessionCreateIdempotencyKey ??
749
829
  (task.runMode === "reusable_session"
750
830
  ? `scheduled-task-reusable:${task.id}:${task.authorityRevision}:${task.executionDigest}`
751
- : `scheduled-task-run:${admittedRunId}`),
831
+ : scheduledTaskGeneratedSessionCreateIdempotencyKey(stableProducerKey)),
752
832
  effectiveMaxNestedAgentDepth:
753
833
  generatedSessionDepthPolicy!.effectiveMaxNestedAgentDepth,
754
834
  nestedAgentDepthPolicySource:
@@ -762,6 +842,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
762
842
  resolvedModel: model,
763
843
  resolvedReasoningEffort: reasoningEffort,
764
844
  resolvedLatencyMode: "standard",
845
+ turnExecutionPolicy,
765
846
  resolvedSandboxBackend: sandboxBackend,
766
847
  resolvedSandboxOs: sandboxOs,
767
848
  resolvedTools: taskTools,
@@ -832,6 +913,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
832
913
  producerKey: stableProducerKey,
833
914
  scheduledAt: null,
834
915
  acceptedExecutionSnapshot: acceptedExecution,
916
+ ...(beforeFreshAgentRunCommit ? { beforeFreshAgentRunCommit } : {}),
835
917
  });
836
918
  await recordScheduledTaskFiredUsage(dispatchDb, acceptedExecution, run);
837
919
  if (run.status === "failed" && run.error === "scheduled_authority_exhausted") {
@@ -1308,15 +1390,16 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1308
1390
  xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
1309
1391
  scheduledTaskRunId: run.id,
1310
1392
  },
1311
- async (tx, wakeEventId) => {
1393
+ async (tx, wakeEventId, _updateId, rejectionReason) => {
1312
1394
  if (!wakeEventId) {
1395
+ const error = scheduledSessionRejectionError(rejectionReason);
1313
1396
  await settleScheduledTaskRunInTransaction(tx, {
1314
1397
  workspaceId: task.workspaceId,
1315
1398
  runId: scheduledRun.id,
1316
1399
  sessionId: session.id,
1317
1400
  triggerEventId: null,
1318
1401
  status: "skipped",
1319
- error: "session_cancelled",
1402
+ error,
1320
1403
  });
1321
1404
  return;
1322
1405
  }
@@ -1328,26 +1411,33 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1328
1411
  status: "dispatched",
1329
1412
  });
1330
1413
  },
1331
- incidentPreflightRequired
1332
- ? {
1333
- prepareSource: async (tx, sessionId) =>
1334
- await prepareIncidentTelemetrySource({
1335
- tx,
1336
- settings,
1337
- taskId: task.id,
1338
- workspaceId: task.workspaceId,
1339
- sessionId,
1340
- alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
1341
- acceptedExecution,
1342
- }),
1343
- }
1344
- : undefined,
1414
+ {
1415
+ requireIdleSession: task.overlapPolicy === "skip" && !sessionCreated,
1416
+ ...(incidentPreflightRequired
1417
+ ? {
1418
+ prepareSource: async (tx, sessionId) =>
1419
+ await prepareIncidentTelemetrySource({
1420
+ tx,
1421
+ settings,
1422
+ taskId: task.id,
1423
+ workspaceId: task.workspaceId,
1424
+ sessionId,
1425
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
1426
+ acceptedExecution,
1427
+ }),
1428
+ }
1429
+ : {}),
1430
+ },
1345
1431
  );
1346
- if (scheduledUpdate.reason === "session_cancelled") {
1432
+ if (
1433
+ scheduledUpdate.reason === "session_cancelled" ||
1434
+ scheduledUpdate.reason === "session_not_idle"
1435
+ ) {
1436
+ const error = scheduledSessionRejectionError(scheduledUpdate.reason);
1347
1437
  return {
1348
1438
  kind: "blocked" as const,
1349
1439
  result: { action: "blocked" as const, reason: "scheduled_run_terminal" as const },
1350
- run: { ...run, status: "skipped" as const, error: "session_cancelled" },
1440
+ run: { ...run, status: "skipped" as const, error },
1351
1441
  };
1352
1442
  }
1353
1443
  if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
@@ -1385,39 +1475,14 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1385
1475
  sessionId: session.id,
1386
1476
  });
1387
1477
  // A user-cancelled (terminal) reusable session must not be revived and
1388
- // re-billed on the next fire. Early check avoids the pre-lock goal
1389
- // upsert side-effect; the locked-callback check below is the
1390
- // authoritative atomic guard. Mirrors apps/api/src/domain/sessions.ts.
1478
+ // re-billed on the next fire. This cheap check gives an early answer;
1479
+ // the locked admission check below remains authoritative. Mirrors
1480
+ // apps/api/src/domain/sessions.ts.
1391
1481
  assertReusableSessionRevivable(session.status);
1392
1482
  // Defensive backstop for the API-level 409: a reusable session keeps
1393
1483
  // its creation-time attachment, so a diverged task attachment must
1394
1484
  // fail the run instead of silently running with the wrong secrets.
1395
1485
  assertReusableSessionBindingMatches(session, task);
1396
- // A recurring "maintain X" task re-establishes its objective on every
1397
- // fire: replace the goal text, reactivate it, and reset the counters.
1398
- const goalEvents =
1399
- task.runMode === "reusable_session" && goalSpec && run.status === "queued"
1400
- ? await upsertScheduledSessionGoalForRun(dispatchDb, {
1401
- accountId: task.accountId,
1402
- workspaceId: task.workspaceId,
1403
- sessionId: session.id,
1404
- runId: run.id,
1405
- text: goalSpec.text,
1406
- successCriteria: goalSpec.successCriteria ?? null,
1407
- maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
1408
- ...(goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {}),
1409
- })
1410
- : [];
1411
- if (goalEvents.length > 0) {
1412
- if (deferPublications) {
1413
- deferredEvents.push({
1414
- sessionId: session.id,
1415
- events: goalEvents,
1416
- });
1417
- } else {
1418
- await publishDurableSessionEvents(bus, task.workspaceId, session.id, goalEvents);
1419
- }
1420
- }
1421
1486
  const bundled = await addSessionSystemUpdateWithSourceMutation(
1422
1487
  dispatchDb,
1423
1488
  {
@@ -1452,15 +1517,16 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1452
1517
  xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
1453
1518
  scheduledTaskRunId: run.id,
1454
1519
  },
1455
- async (tx, wakeEventId) => {
1520
+ async (tx, wakeEventId, _updateId, rejectionReason) => {
1456
1521
  if (!wakeEventId) {
1522
+ const error = scheduledSessionRejectionError(rejectionReason);
1457
1523
  await settleScheduledTaskRunInTransaction(tx, {
1458
1524
  workspaceId: task.workspaceId,
1459
1525
  runId: scheduledRun.id,
1460
1526
  sessionId: session.id,
1461
1527
  triggerEventId: null,
1462
1528
  status: "skipped",
1463
- error: "session_cancelled",
1529
+ error,
1464
1530
  });
1465
1531
  return;
1466
1532
  }
@@ -1472,10 +1538,26 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1472
1538
  status: "dispatched",
1473
1539
  });
1474
1540
  },
1475
- incidentPreflightRequired
1476
- ? {
1477
- prepareSource: async (tx, sessionId) =>
1478
- await prepareIncidentTelemetrySource({
1541
+ {
1542
+ requireIdleSession: task.overlapPolicy === "skip",
1543
+ prepareSource: async (tx, sessionId) => {
1544
+ const goalEvents =
1545
+ task.runMode === "reusable_session" && goalSpec && run.status === "queued"
1546
+ ? await upsertScheduledSessionGoalForRun(tx, {
1547
+ accountId: task.accountId,
1548
+ workspaceId: task.workspaceId,
1549
+ sessionId,
1550
+ runId: run.id,
1551
+ text: goalSpec.text,
1552
+ successCriteria: goalSpec.successCriteria ?? null,
1553
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
1554
+ ...(goalSpec.mutationPolicy
1555
+ ? { mutationPolicy: goalSpec.mutationPolicy }
1556
+ : {}),
1557
+ })
1558
+ : [];
1559
+ const incidentSource = incidentPreflightRequired
1560
+ ? await prepareIncidentTelemetrySource({
1479
1561
  tx,
1480
1562
  settings,
1481
1563
  taskId: task.id,
@@ -1483,15 +1565,21 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1483
1565
  sessionId,
1484
1566
  alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
1485
1567
  acceptedExecution,
1486
- }),
1487
- }
1488
- : undefined,
1568
+ })
1569
+ : undefined;
1570
+ return {
1571
+ ...(incidentSource ?? {}),
1572
+ ...(goalEvents.length > 0 ? { events: goalEvents } : {}),
1573
+ };
1574
+ },
1575
+ },
1489
1576
  );
1490
- if (bundled.reason === "session_cancelled") {
1577
+ if (bundled.reason === "session_cancelled" || bundled.reason === "session_not_idle") {
1578
+ const error = scheduledSessionRejectionError(bundled.reason);
1491
1579
  return {
1492
1580
  kind: "blocked" as const,
1493
1581
  result: { action: "blocked" as const, reason: "scheduled_run_terminal" as const },
1494
- run: { ...run, status: "skipped" as const, error: "session_cancelled" },
1582
+ run: { ...run, status: "skipped" as const, error },
1495
1583
  };
1496
1584
  }
1497
1585
  if (bundled.added && bundled.events.length > 0) {
@@ -1571,6 +1659,10 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1571
1659
  if (error instanceof IncidentTelemetryPreflightBlockedError) {
1572
1660
  return { action: "blocked", reason: error.reason };
1573
1661
  }
1662
+ const terminalError = scheduledRunRecoveryTerminalError(error);
1663
+ if (terminalError?.code === "scheduled_model_unavailable") {
1664
+ return scheduledRunTerminalResult(terminalError.code);
1665
+ }
1574
1666
  throw error;
1575
1667
  }
1576
1668
  if (dispatchOutcome.kind === "replay") {
@@ -1711,6 +1803,14 @@ function scheduledRunTerminalResult(
1711
1803
  return { action: "blocked", reason: "scheduled_run_terminal" };
1712
1804
  }
1713
1805
 
1806
+ function scheduledSessionRejectionError(
1807
+ rejectionReason: "session_cancelled" | "session_not_idle" | null,
1808
+ ): "session_cancelled" | "scheduled_session_not_idle" {
1809
+ return rejectionReason === "session_not_idle"
1810
+ ? "scheduled_session_not_idle"
1811
+ : "session_cancelled";
1812
+ }
1813
+
1714
1814
  async function prepareIncidentTelemetrySource(input: {
1715
1815
  tx: Database;
1716
1816
  settings: ControlActivityServices["settings"];
@@ -2218,23 +2318,6 @@ async function recoverBoundScheduledTaskDispatch(input: {
2218
2318
  await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, titleEvents);
2219
2319
  }
2220
2320
  }
2221
- if (!generatedSession && task.runMode === "reusable_session" && task.agentConfig.goal) {
2222
- const goalEvents = await upsertScheduledSessionGoalForRun(input.db, {
2223
- accountId: task.accountId,
2224
- workspaceId: task.workspaceId,
2225
- sessionId: session.id,
2226
- runId: input.run.id,
2227
- text: task.agentConfig.goal.text,
2228
- successCriteria: task.agentConfig.goal.successCriteria ?? null,
2229
- maxAutoContinuations: task.agentConfig.goal.maxAutoContinuations ?? null,
2230
- ...(task.agentConfig.goal.mutationPolicy
2231
- ? { mutationPolicy: task.agentConfig.goal.mutationPolicy }
2232
- : {}),
2233
- });
2234
- if (goalEvents.length > 0) {
2235
- await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, goalEvents);
2236
- }
2237
- }
2238
2321
  const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
2239
2322
  input.db,
2240
2323
  {
@@ -2273,15 +2356,16 @@ async function recoverBoundScheduledTaskDispatch(input: {
2273
2356
  input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
2274
2357
  scheduledTaskRunId: input.run.id,
2275
2358
  },
2276
- async (tx, wakeEventId) => {
2359
+ async (tx, wakeEventId, _updateId, rejectionReason) => {
2277
2360
  if (!wakeEventId) {
2361
+ const error = scheduledSessionRejectionError(rejectionReason);
2278
2362
  await settleScheduledTaskRunInTransaction(tx, {
2279
2363
  workspaceId: task.workspaceId,
2280
2364
  runId: input.run.id,
2281
2365
  sessionId: session.id,
2282
2366
  triggerEventId: null,
2283
2367
  status: "skipped",
2284
- error: "session_cancelled",
2368
+ error,
2285
2369
  });
2286
2370
  return;
2287
2371
  }
@@ -2293,10 +2377,25 @@ async function recoverBoundScheduledTaskDispatch(input: {
2293
2377
  status: "dispatched",
2294
2378
  });
2295
2379
  },
2296
- input.acceptedExecution.incidentPreflightRequired
2297
- ? {
2298
- prepareSource: async (tx, sessionId) =>
2299
- await prepareIncidentTelemetrySource({
2380
+ {
2381
+ requireIdleSession: task.overlapPolicy === "skip" && !generatedSession,
2382
+ prepareSource: async (tx, sessionId) => {
2383
+ const goal = task.agentConfig.goal;
2384
+ const goalEvents =
2385
+ !generatedSession && task.runMode === "reusable_session" && goal
2386
+ ? await upsertScheduledSessionGoalForRun(tx, {
2387
+ accountId: task.accountId,
2388
+ workspaceId: task.workspaceId,
2389
+ sessionId,
2390
+ runId: input.run.id,
2391
+ text: goal.text,
2392
+ successCriteria: goal.successCriteria ?? null,
2393
+ maxAutoContinuations: goal.maxAutoContinuations ?? null,
2394
+ ...(goal.mutationPolicy ? { mutationPolicy: goal.mutationPolicy } : {}),
2395
+ })
2396
+ : [];
2397
+ const incidentSource = input.acceptedExecution.incidentPreflightRequired
2398
+ ? await prepareIncidentTelemetrySource({
2300
2399
  tx,
2301
2400
  settings: input.settings,
2302
2401
  taskId: task.id,
@@ -2304,11 +2403,19 @@ async function recoverBoundScheduledTaskDispatch(input: {
2304
2403
  sessionId,
2305
2404
  alertOccurrenceLabels: input.acceptedExecution.alertOccurrenceLabels,
2306
2405
  acceptedExecution: input.acceptedExecution,
2307
- }),
2308
- }
2309
- : undefined,
2406
+ })
2407
+ : undefined;
2408
+ return {
2409
+ ...(incidentSource ?? {}),
2410
+ ...(goalEvents.length > 0 ? { events: goalEvents } : {}),
2411
+ };
2412
+ },
2413
+ },
2310
2414
  );
2311
- if (scheduledUpdate.reason === "session_cancelled") {
2415
+ if (
2416
+ scheduledUpdate.reason === "session_cancelled" ||
2417
+ scheduledUpdate.reason === "session_not_idle"
2418
+ ) {
2312
2419
  return { action: "blocked", reason: "scheduled_run_terminal" };
2313
2420
  }
2314
2421
  if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
@@ -7,6 +7,7 @@ import {
7
7
  recoverSessionDispatch,
8
8
  reconcileSessionAttemptQuiescence,
9
9
  peekSessionWork as peekSessionWorkDb,
10
+ settleSessionInputWait as settleSessionInputWaitDb,
10
11
  countQueuedTurns,
11
12
  getSessionAttemptActivityRef,
12
13
  getSessionEvent,
@@ -42,6 +43,8 @@ import type {
42
43
  RecoverDispatchResult,
43
44
  RecoverEscapedMcpTimeoutInput,
44
45
  RecoverEscapedMcpTimeoutResult,
46
+ SettleSessionInputWaitInput,
47
+ SettleSessionInputWaitResult,
45
48
  } from "./types";
46
49
 
47
50
  export type SessionStateActivityOverrides = Partial<{
@@ -53,6 +56,7 @@ export type SessionStateActivityOverrides = Partial<{
53
56
  recoverSessionDispatch: typeof recoverSessionDispatch;
54
57
  reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
55
58
  peekSessionWork: typeof peekSessionWorkDb;
59
+ settleSessionInputWait: typeof settleSessionInputWaitDb;
56
60
  countQueuedTurns: typeof countQueuedTurns;
57
61
  getSessionAttemptActivityRef: typeof getSessionAttemptActivityRef;
58
62
  getSessionEvent: typeof getSessionEvent;
@@ -93,6 +97,7 @@ export function createSessionStateActivities(
93
97
  const reconcileSessionAttemptQuiescenceFn =
94
98
  overrides.reconcileSessionAttemptQuiescence ?? reconcileSessionAttemptQuiescence;
95
99
  const peekSessionWorkFn = overrides.peekSessionWork ?? peekSessionWorkDb;
100
+ const settleSessionInputWaitFn = overrides.settleSessionInputWait ?? settleSessionInputWaitDb;
96
101
  const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns;
97
102
  const getSessionAttemptActivityRefFn =
98
103
  overrides.getSessionAttemptActivityRef ?? getSessionAttemptActivityRef;
@@ -486,6 +491,17 @@ export function createSessionStateActivities(
486
491
  return peek;
487
492
  }
488
493
 
494
+ async function settleSessionInputWait(
495
+ input: SettleSessionInputWaitInput,
496
+ ): Promise<SettleSessionInputWaitResult> {
497
+ const { db, bus } = await services();
498
+ const result = await settleSessionInputWaitFn(db, input);
499
+ if (result.events.length > 0) {
500
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
501
+ }
502
+ return { action: result.action };
503
+ }
504
+
489
505
  async function expireSessionHumanInput(
490
506
  input: ExpireSessionHumanInputInput,
491
507
  ): Promise<ExpireSessionHumanInputResult> {
@@ -546,6 +562,7 @@ export function createSessionStateActivities(
546
562
  recoverDispatch,
547
563
  recoverEscapedMcpTimeout,
548
564
  peekSessionWork,
565
+ settleSessionInputWait,
549
566
  expireSessionHumanInput,
550
567
  expireSessionInteractionIntervention,
551
568
  markSessionIdle,
@@ -10,6 +10,7 @@ import {
10
10
  canonicalConfiguredModel,
11
11
  createAndStartSessionWithOutcome,
12
12
  recordWorkspaceUsage,
13
+ resolveCatalogSettings,
13
14
  } from "@opengeni/core";
14
15
  import {
15
16
  claimSiteAuthMaintenance,
@@ -127,23 +128,22 @@ async function dispatchMaintenanceSession(
127
128
  assertModelPolicy: typeof assertWorkspaceModelPolicyAllows;
128
129
  },
129
130
  ): Promise<void> {
130
- const model = canonicalConfiguredModel(service.settings, service.settings.openaiModel);
131
+ const catalogSettings = (
132
+ await resolveCatalogSettings(service.db, service.catalogSourceSettings ?? service.settings)
133
+ ).settings;
134
+ const catalogService = { ...service, settings: catalogSettings };
135
+ const model = canonicalConfiguredModel(catalogSettings, catalogSettings.openaiModel);
131
136
  if (!model) throw new Error("site auth maintenance has no configured model");
132
- await dependencies.assertModelPolicy(
133
- service.db,
134
- service.settings,
135
- maintenance.workspaceId,
136
- model,
137
- );
138
- const denial = await dependencies.admit(service, {
137
+ await dependencies.assertModelPolicy(service.db, catalogSettings, maintenance.workspaceId, model);
138
+ const denial = await dependencies.admit(catalogService, {
139
139
  accountId: maintenance.accountId,
140
140
  workspaceId: maintenance.workspaceId,
141
141
  model,
142
142
  requestedAgentRuns: 1,
143
143
  });
144
144
  if (denial) throw new Error(`site auth maintenance admission denied: ${denial}`);
145
- const reasoningEffort = service.settings.openaiReasoningEffort;
146
- const turnExecutionPolicy = resolveTurnExecutionPolicyV1(service.settings, {
145
+ const reasoningEffort = catalogSettings.openaiReasoningEffort;
146
+ const turnExecutionPolicy = resolveTurnExecutionPolicyV1(catalogSettings, {
147
147
  modelId: model,
148
148
  requestedModelId: null,
149
149
  modelSource: "deployment",
@@ -180,7 +180,7 @@ async function dispatchMaintenanceSession(
180
180
  model,
181
181
  reasoningEffort,
182
182
  turnExecutionPolicy,
183
- sandboxBackend: service.settings.sandboxBackend,
183
+ sandboxBackend: catalogSettings.sandboxBackend,
184
184
  metadata: {
185
185
  role: "site_auth_maintenance",
186
186
  [SITE_AUTH_MAINTENANCE_CONNECTION_CONTEXT_KEY]: maintenance.siteAuthConnectionId,
@@ -205,7 +205,7 @@ async function dispatchMaintenanceSession(
205
205
  });
206
206
  try {
207
207
  await dependencies.recordUsage(
208
- { db: service.db, settings: service.settings },
208
+ { db: service.db, settings: catalogSettings },
209
209
  {
210
210
  accountId: maintenance.accountId,
211
211
  workspaceId: maintenance.workspaceId,