@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +10 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  3. package/dist/activities/agent-turn/claim.d.ts +3 -4
  4. package/dist/activities/agent-turn/codex.d.ts +8 -7
  5. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  6. package/dist/activities/agent-turn/errors.d.ts +13 -0
  7. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +10 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  11. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  13. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
  16. package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +3 -8
  18. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  19. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  20. package/dist/activities/automations.d.ts +1 -0
  21. package/dist/activities/capabilities.d.ts +2 -1
  22. package/dist/activities/codex-rotation.d.ts +2 -0
  23. package/dist/activities/goals.d.ts +11 -0
  24. package/dist/activities/image-generation-references.d.ts +2 -1
  25. package/dist/activities/scheduled-tasks.d.ts +1 -0
  26. package/dist/activities/session-state.d.ts +2 -1
  27. package/dist/activities/types.d.ts +8 -0
  28. package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
  29. package/dist/activities-control-SGU2WEVB.js.map +1 -0
  30. package/dist/activities-control.d.ts +2 -0
  31. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
  32. package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
  33. package/dist/activities.d.ts +2 -0
  34. package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
  35. package/dist/chunk-S2QVRVJW.js.map +1 -0
  36. package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
  37. package/dist/chunk-TJIPCHMP.js.map +1 -0
  38. package/dist/index.d.ts +1 -0
  39. package/dist/index.js +69 -6
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +29 -0
  42. package/dist/personal-github-git-credentials.d.ts +1 -0
  43. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  44. package/dist/workflow-bundle.js +26 -3
  45. package/dist/workflows/activities.d.ts +5 -0
  46. package/dist/workflows/automations.d.ts +5 -0
  47. package/package.json +18 -18
  48. package/src/activities/agent-run-admission.ts +7 -1
  49. package/src/activities/agent-turn/admission.ts +28 -6
  50. package/src/activities/agent-turn/agent-build.ts +15 -31
  51. package/src/activities/agent-turn/claim.ts +53 -72
  52. package/src/activities/agent-turn/codex.ts +21 -12
  53. package/src/activities/agent-turn/compaction-prep.ts +52 -1
  54. package/src/activities/agent-turn/errors.ts +78 -0
  55. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  56. package/src/activities/agent-turn/finalization.ts +24 -21
  57. package/src/activities/agent-turn/history.ts +5 -1
  58. package/src/activities/agent-turn/list-models.ts +101 -0
  59. package/src/activities/agent-turn/model-usage.ts +88 -42
  60. package/src/activities/agent-turn/quiescence.ts +86 -0
  61. package/src/activities/agent-turn/run-credentials.ts +12 -1
  62. package/src/activities/agent-turn/run.ts +20 -14
  63. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  64. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  65. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  66. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  67. package/src/activities/agent-turn/stream-attempt.ts +106 -14
  68. package/src/activities/agent-turn/tool-environment.ts +72 -14
  69. package/src/activities/agent-turn/tool-policy.ts +7 -6
  70. package/src/activities/agent-turn/turn-context.ts +12 -14
  71. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  72. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  73. package/src/activities/automations.ts +122 -67
  74. package/src/activities/capabilities.ts +72 -2
  75. package/src/activities/codex-capacity.ts +13 -5
  76. package/src/activities/codex-rotation.ts +17 -0
  77. package/src/activities/goals.ts +117 -35
  78. package/src/activities/image-generation-references.ts +9 -9
  79. package/src/activities/retained-screenshots.ts +104 -21
  80. package/src/activities/sandbox-lease.ts +7 -4
  81. package/src/activities/scheduled-tasks.ts +248 -30
  82. package/src/activities/session-state.ts +8 -1
  83. package/src/activities/site-auth-maintenance.ts +12 -12
  84. package/src/activities/types.ts +6 -0
  85. package/src/activities.ts +1 -0
  86. package/src/activity-services.ts +4 -1
  87. package/src/github-rest-mcp.ts +8 -4
  88. package/src/http.ts +3 -0
  89. package/src/index.ts +38 -4
  90. package/src/observability-metrics.ts +202 -0
  91. package/src/personal-github-git-credentials.ts +45 -4
  92. package/src/sandbox-resume.ts +1 -1
  93. package/src/sandbox-routing.ts +7 -4
  94. package/src/unhandled-rejection-boundary.ts +73 -0
  95. package/src/workflows/activities.ts +15 -0
  96. package/src/workflows/automations.ts +21 -2
  97. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  98. package/dist/activities/recording.d.ts +0 -90
  99. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  100. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  101. package/dist/chunk-NAVSAZ56.js.map +0 -1
  102. package/dist/chunk-O4HDX5SC.js.map +0 -1
  103. package/src/activities/recording.ts +0 -279
@@ -10,6 +10,7 @@ import {
10
10
  normalizeAutomaticSessionTitle,
11
11
  scheduledOccurrencePayloadUtf8Bytes,
12
12
  stableJson,
13
+ TurnExecutionPolicyV1,
13
14
  type ScheduledTask,
14
15
  type ScheduledTaskRun,
15
16
  } from "@opengeni/contracts";
@@ -17,8 +18,11 @@ import {
17
18
  defaultSessionMcpServerIds,
18
19
  openGeniSlackBotMetadata,
19
20
  requireOpenGeniSlackBotConnection,
21
+ resolveWorkspaceCatalogSettings,
20
22
  resolveSessionToolPolicy,
23
+ workspaceCustomModelReference,
21
24
  scheduledSlackBotConnectionId,
25
+ swapActiveSandbox,
22
26
  } from "@opengeni/core";
23
27
  import {
24
28
  appendSessionEvents,
@@ -28,6 +32,7 @@ import {
28
32
  createSession,
29
33
  createSessionWithIdempotencyKeyResult,
30
34
  enqueueSessionWorkflowWakeIfRunnable,
35
+ failScheduledGeneratedSessionRoute,
31
36
  getScheduledTask,
32
37
  getScheduledTaskRunAcceptedExecution,
33
38
  getScheduledTaskRunByProducerKey,
@@ -38,6 +43,8 @@ import {
38
43
  getScheduledTaskPersonalResourceAuthoritySubject,
39
44
  getScheduledTaskRevisionAuthority,
40
45
  getScheduledTaskXaiProviderAccountAuthoritySnapshot,
46
+ getEnrollment,
47
+ getSandbox,
41
48
  getRig,
42
49
  getScheduledScopedRigVersionMetadata,
43
50
  getNestedAgentDepthDeploymentPolicy,
@@ -45,6 +52,8 @@ import {
45
52
  getVariableSet,
46
53
  isCodexBilledModel,
47
54
  initializeSessionStartAtomically,
55
+ lockActiveWorkspaceGatewayCustomModelForAdmission,
56
+ lockActiveWorkspaceOpenRouterCustomModelForAdmission,
48
57
  materializeScheduledTaskReusableSessionFromRun,
49
58
  listEnabledMcpCapabilityServerIds,
50
59
  listInstalledApiIntegrationServerIdsForDelegations,
@@ -63,8 +72,9 @@ import {
63
72
  withSessionActivityRlsContext,
64
73
  } from "@opengeni/db";
65
74
  import { publishDurableSessionEvents } from "@opengeni/events";
66
- import { resolveFirstPartyMcpToolPolicy } from "@opengeni/config";
75
+ import { resolveFirstPartyMcpToolPolicy, resolveTurnExecutionPolicyV1 } from "@opengeni/config";
67
76
  import { Context } from "@temporalio/activity";
77
+ import { createHash } from "node:crypto";
68
78
  import {
69
79
  assertReusableSessionRevivable,
70
80
  scheduledUserMessagePayload,
@@ -124,6 +134,14 @@ export function scheduledTaskRunProducerKey(
124
134
  })}`;
125
135
  }
126
136
 
137
+ export function scheduledTaskGeneratedSessionCreateIdempotencyKey(producerKey: string): string {
138
+ const digest = createHash("sha256")
139
+ .update("scheduled-task-run\0", "utf8")
140
+ .update(producerKey, "utf8")
141
+ .digest("hex");
142
+ return `scheduled-task-run:${digest}`;
143
+ }
144
+
127
145
  /**
128
146
  * Display title for a session the scheduler generates for a task.
129
147
  *
@@ -230,14 +248,26 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
230
248
  dispatchScheduledTaskRun: async (
231
249
  input: DispatchScheduledTaskRunInput,
232
250
  ): Promise<DispatchScheduledTaskRunResult> => {
233
- const service = await services();
234
- const { settings, db, bus, wakeSessionWorkflow } = service;
251
+ const baseService = await services();
235
252
  // Histories created before the manual-initiator workflow patch already
236
253
  // contain this activity command. Reject their incomplete wire input here
237
254
  // so replay consumes the recorded command and the retry loop settles.
238
255
  if (input.triggerType !== "scheduled" && !input.initiator) {
239
256
  return { action: "blocked", reason: "malformed_manual_trigger" };
240
257
  }
258
+ const { db, bus, wakeSessionWorkflow } = baseService;
259
+ const settingsForTask = async (task: ScheduledTask, retainedProductModelId?: string | null) =>
260
+ (
261
+ await resolveWorkspaceCatalogSettings(
262
+ db,
263
+ baseService.catalogSourceSettings ?? baseService.settings,
264
+ {
265
+ accountId: task.accountId,
266
+ workspaceId: task.workspaceId,
267
+ ...(retainedProductModelId !== undefined ? { retainedProductModelId } : {}),
268
+ },
269
+ )
270
+ ).settings;
241
271
  const stableProducerKey = scheduledTaskRunProducerKey(input);
242
272
  const priorRun = await getScheduledTaskRunByProducerKey(db, {
243
273
  workspaceId: input.workspaceId,
@@ -271,6 +301,15 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
271
301
  });
272
302
  }
273
303
  if (priorRun.status === "queued") {
304
+ const acceptedPolicy = acceptedExecution.turnExecutionPolicy
305
+ ? TurnExecutionPolicyV1.parse(acceptedExecution.turnExecutionPolicy)
306
+ : null;
307
+ const settings = await settingsForTask(
308
+ acceptedExecution.task,
309
+ acceptedPolicy?.productModelId ??
310
+ acceptedExecution.targetSessionExecution?.model ??
311
+ acceptedExecution.resolvedModel,
312
+ );
274
313
  try {
275
314
  return await recoverBoundScheduledTaskDispatch({
276
315
  db,
@@ -440,14 +479,64 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
440
479
  taskRevisionAuthority?.subjectId ??
441
480
  taskPersonalResourceAuthoritySubjectId ??
442
481
  taskConnectionAuthoritySubjectId;
482
+ const acceptedTargetSessionId =
483
+ task.runMode === "existing_session"
484
+ ? task.targetSessionId
485
+ : task.runMode === "reusable_session"
486
+ ? task.reusableSessionId
487
+ : null;
488
+ // A targeted task whose exact session was deleted has no target to
489
+ // resolve; database admission settles that occurrence terminally
490
+ // (`scheduled_target_session_unavailable`) instead of retrying forever.
491
+ // Resolve this before the catalog so an existing session's retired
492
+ // custom Gateway model remains executable without becoming selectable
493
+ // for a new scheduled session.
494
+ const targetSessionExecutionBase = acceptedTargetSessionId
495
+ ? await getScheduledTargetSessionExecution(
496
+ db,
497
+ task.workspaceId,
498
+ acceptedTargetSessionId,
499
+ taskAuthoritySubjectId,
500
+ )
501
+ : null;
502
+ const settings = await settingsForTask(task, targetSessionExecutionBase?.model);
443
503
  const model = task.agentConfig.model ?? settings.openaiModel;
444
504
  const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
445
- const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
505
+ let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
506
+ let sandboxOs: "linux" | "macos" | "windows" = "linux";
446
507
  const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
447
508
  const firstPartyMcpTools = resolveFirstPartyMcpToolPolicy(settings).default;
448
509
  const generatedTarget =
449
510
  task.runMode === "new_session_per_run" ||
450
511
  (task.runMode === "reusable_session" && task.reusableSessionId === null);
512
+ if (generatedTarget && task.agentConfig.machineTarget) {
513
+ const access = {
514
+ accountId: task.accountId,
515
+ workspaceId: task.workspaceId,
516
+ subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId,
517
+ };
518
+ const machine = await getSandbox(
519
+ db,
520
+ access,
521
+ task.agentConfig.machineTarget.targetSandboxId,
522
+ );
523
+ if (
524
+ !machine ||
525
+ machine.kind !== "selfhosted" ||
526
+ !machine.enrollmentId ||
527
+ machine.scope === "user"
528
+ ) {
529
+ throw new Error("scheduled Connected Machine target is unavailable");
530
+ }
531
+ const enrollment = await getEnrollment(db, access, machine.enrollmentId);
532
+ if (!enrollment || enrollment.status !== "active") {
533
+ throw new Error("scheduled Connected Machine enrollment is unavailable");
534
+ }
535
+ sandboxBackend = "selfhosted";
536
+ sandboxOs = enrollment.os;
537
+ } else if (generatedTarget && sandboxBackend === "selfhosted") {
538
+ throw new Error("self-hosted scheduled task has no Connected Machine target");
539
+ }
451
540
  const generatedSessionDepthPolicy = generatedTarget
452
541
  ? await (async () => {
453
542
  const workspace = await requireWorkspace(db, task.workspaceId);
@@ -482,22 +571,6 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
482
571
  };
483
572
  })()
484
573
  : null;
485
- const acceptedTargetSessionId = generatedTarget
486
- ? null
487
- : task.runMode === "existing_session"
488
- ? task.targetSessionId
489
- : task.reusableSessionId;
490
- // A targeted task whose exact session was deleted has no target to
491
- // resolve; database admission settles that occurrence terminally
492
- // (`scheduled_target_session_unavailable`) instead of retrying forever.
493
- const targetSessionExecutionBase = acceptedTargetSessionId
494
- ? await getScheduledTargetSessionExecution(
495
- db,
496
- task.workspaceId,
497
- acceptedTargetSessionId,
498
- taskAuthoritySubjectId,
499
- )
500
- : null;
501
574
  const targetSessionExecution = targetSessionExecutionBase
502
575
  ? await (async () => {
503
576
  const [capabilityServerIds, apiIntegrationServerIds] = await Promise.all([
@@ -694,15 +767,63 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
694
767
  ) {
695
768
  throw new Error("scheduled personal-resource execution has no causal human");
696
769
  }
697
- const admissionDenial = await agentRunAdmissionDenial(service, {
698
- accountId: task.accountId,
699
- workspaceId: task.workspaceId,
700
- model: targetSessionExecution?.model ?? model,
701
- requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1,
702
- });
770
+ const admissionDenial = await agentRunAdmissionDenial(
771
+ { ...baseService, settings },
772
+ {
773
+ accountId: task.accountId,
774
+ workspaceId: task.workspaceId,
775
+ model: targetSessionExecution?.model ?? model,
776
+ requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1,
777
+ },
778
+ );
703
779
  if (admissionDenial) {
704
780
  return { action: "blocked", reason: admissionDenial };
705
781
  }
782
+ const acceptedModel = targetSessionExecution?.model ?? model;
783
+ const acceptedReasoningEffort = targetSessionExecution?.reasoningEffort ?? reasoningEffort;
784
+ const acceptedLatencyMode = targetSessionExecution?.latencyMode ?? "standard";
785
+ const acceptedCustomModel = generatedTarget
786
+ ? workspaceCustomModelReference(settings, acceptedModel)
787
+ : null;
788
+ const beforeFreshAgentRunCommit = acceptedCustomModel
789
+ ? async (tx: Database): Promise<void> => {
790
+ const active =
791
+ acceptedCustomModel.providerKind === "openrouter"
792
+ ? await lockActiveWorkspaceOpenRouterCustomModelForAdmission(tx, {
793
+ accountId: task.accountId,
794
+ workspaceId: task.workspaceId,
795
+ upstreamModelId: acceptedCustomModel.upstreamModelId,
796
+ })
797
+ : await lockActiveWorkspaceGatewayCustomModelForAdmission(tx, {
798
+ accountId: task.accountId,
799
+ workspaceId: task.workspaceId,
800
+ upstreamModelId: acceptedCustomModel.upstreamModelId,
801
+ });
802
+ if (!active) {
803
+ throw new ScheduledRunTerminalAuthorityError(
804
+ "scheduled_model_unavailable",
805
+ `model is not available: ${acceptedModel}`,
806
+ );
807
+ }
808
+ }
809
+ : undefined;
810
+ const turnExecutionPolicy: TurnExecutionPolicyV1 = resolveTurnExecutionPolicyV1(settings, {
811
+ modelId: acceptedModel,
812
+ requestedModelId: generatedTarget && task.agentConfig.model ? task.agentConfig.model : null,
813
+ modelSource: generatedTarget
814
+ ? task.agentConfig.model
815
+ ? "explicit"
816
+ : "deployment"
817
+ : "session",
818
+ reasoningEffort: acceptedReasoningEffort,
819
+ reasoningSource: generatedTarget
820
+ ? task.agentConfig.reasoningEffort
821
+ ? "explicit"
822
+ : "deployment"
823
+ : "session",
824
+ latencyMode: acceptedLatencyMode,
825
+ latencyModeSource: generatedTarget ? "deployment" : "session",
826
+ });
706
827
  const deferredEvents: Array<{
707
828
  sessionId: string;
708
829
  events: Awaited<ReturnType<typeof appendSessionEvents>>;
@@ -715,7 +836,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
715
836
  alertResponderSessionCreateIdempotencyKey ??
716
837
  (task.runMode === "reusable_session"
717
838
  ? `scheduled-task-reusable:${task.id}:${task.authorityRevision}:${task.executionDigest}`
718
- : `scheduled-task-run:${admittedRunId}`),
839
+ : scheduledTaskGeneratedSessionCreateIdempotencyKey(stableProducerKey)),
719
840
  effectiveMaxNestedAgentDepth:
720
841
  generatedSessionDepthPolicy!.effectiveMaxNestedAgentDepth,
721
842
  nestedAgentDepthPolicySource:
@@ -729,8 +850,9 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
729
850
  resolvedModel: model,
730
851
  resolvedReasoningEffort: reasoningEffort,
731
852
  resolvedLatencyMode: "standard",
853
+ turnExecutionPolicy,
732
854
  resolvedSandboxBackend: sandboxBackend,
733
- resolvedSandboxOs: "linux",
855
+ resolvedSandboxOs: sandboxOs,
734
856
  resolvedTools: taskTools,
735
857
  resolvedFirstPartyMcpTools: firstPartyMcpTools,
736
858
  resolvedFirstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
@@ -799,6 +921,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
799
921
  producerKey: stableProducerKey,
800
922
  scheduledAt: null,
801
923
  acceptedExecutionSnapshot: acceptedExecution,
924
+ ...(beforeFreshAgentRunCommit ? { beforeFreshAgentRunCommit } : {}),
802
925
  });
803
926
  await recordScheduledTaskFiredUsage(dispatchDb, acceptedExecution, run);
804
927
  if (run.status === "failed" && run.error === "scheduled_authority_exhausted") {
@@ -972,7 +1095,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
972
1095
  reasoningEffort,
973
1096
  latencyMode: "standard",
974
1097
  sandboxBackend,
975
- sandboxOs: "linux",
1098
+ sandboxOs,
976
1099
  variableSetId: task.variableSetId ?? null,
977
1100
  rigId: frozenRigId,
978
1101
  rigVersionId: frozenRigVersionId,
@@ -1079,6 +1202,16 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1079
1202
  );
1080
1203
  }
1081
1204
  }
1205
+ session = await seedScheduledGeneratedSessionRoute({
1206
+ db: dispatchDb,
1207
+ bus,
1208
+ settings,
1209
+ task,
1210
+ runId: run.id,
1211
+ session,
1212
+ deferPublications,
1213
+ deferredEvents,
1214
+ });
1082
1215
  let workflowId: string;
1083
1216
  if (alertOccurrence) {
1084
1217
  const started = await initializeSessionStartAtomically(dispatchDb, {
@@ -1528,6 +1661,10 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1528
1661
  if (error instanceof IncidentTelemetryPreflightBlockedError) {
1529
1662
  return { action: "blocked", reason: error.reason };
1530
1663
  }
1664
+ const terminalError = scheduledRunRecoveryTerminalError(error);
1665
+ if (terminalError?.code === "scheduled_model_unavailable") {
1666
+ return scheduledRunTerminalResult(terminalError.code);
1667
+ }
1531
1668
  throw error;
1532
1669
  }
1533
1670
  if (dispatchOutcome.kind === "replay") {
@@ -1539,7 +1676,6 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1539
1676
  acceptedExecution: dispatchOutcome.acceptedExecution,
1540
1677
  });
1541
1678
  }
1542
- if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
1543
1679
  for (const deferred of deferredEvents) {
1544
1680
  await publishDurableSessionEvents(
1545
1681
  bus,
@@ -1548,6 +1684,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1548
1684
  deferred.events,
1549
1685
  );
1550
1686
  }
1687
+ if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
1551
1688
  const result = dispatchOutcome.result;
1552
1689
  if (wakeSessionWorkflow && result.workflowWakeRevision !== null) {
1553
1690
  await wakeSessionWorkflow({
@@ -1773,6 +1910,74 @@ async function recordScheduledTaskFiredUsage(
1773
1910
  });
1774
1911
  }
1775
1912
 
1913
+ async function seedScheduledGeneratedSessionRoute(input: {
1914
+ db: Database;
1915
+ bus: ControlActivityServices["bus"];
1916
+ settings: ControlActivityServices["settings"];
1917
+ task: ScheduledTask;
1918
+ runId: string;
1919
+ session: Awaited<ReturnType<typeof createSession>>;
1920
+ deferPublications: boolean;
1921
+ deferredEvents: Array<{
1922
+ sessionId: string;
1923
+ events: Awaited<ReturnType<typeof appendSessionEvents>>;
1924
+ }>;
1925
+ }): Promise<Awaited<ReturnType<typeof createSession>>> {
1926
+ const target = input.task.agentConfig.machineTarget;
1927
+ if (!target) return input.session;
1928
+ const seeded = await swapActiveSandbox(
1929
+ {
1930
+ db: input.db,
1931
+ settings: input.settings,
1932
+ bus: input.bus,
1933
+ },
1934
+ {
1935
+ accountId: input.task.accountId,
1936
+ workspaceId: input.task.workspaceId,
1937
+ sessionId: input.session.id,
1938
+ sessionBackend: input.session.sandboxBackend,
1939
+ sessionGroupId: input.session.sandboxGroupId,
1940
+ },
1941
+ target.targetSandboxId,
1942
+ target.workingDir ?? null,
1943
+ );
1944
+ if (seeded.swapped) {
1945
+ return await requireSession(input.db, input.task.workspaceId, input.session.id);
1946
+ }
1947
+
1948
+ const failed = await failScheduledGeneratedSessionRoute(input.db, {
1949
+ accountId: input.task.accountId,
1950
+ workspaceId: input.task.workspaceId,
1951
+ taskId: input.task.id,
1952
+ runId: input.runId,
1953
+ sessionId: input.session.id,
1954
+ error: "scheduled_machine_unavailable",
1955
+ });
1956
+ if (failed.action === "advanced") {
1957
+ throw new Error(
1958
+ `scheduled run advanced to ${failed.status} while its Connected Machine route was being established`,
1959
+ );
1960
+ }
1961
+ if (failed.events.length > 0) {
1962
+ if (input.deferPublications) {
1963
+ input.deferredEvents.push({ sessionId: input.session.id, events: failed.events });
1964
+ } else {
1965
+ await publishDurableSessionEvents(
1966
+ input.bus,
1967
+ input.task.workspaceId,
1968
+ input.session.id,
1969
+ failed.events,
1970
+ );
1971
+ }
1972
+ }
1973
+ const error =
1974
+ failed.action === "terminal" ? (failed.error ?? "scheduled_run_terminal") : failed.error;
1975
+ throw new ScheduledRunTerminalAuthorityError(
1976
+ error,
1977
+ seeded.reason ?? "scheduled Connected Machine target is unavailable",
1978
+ );
1979
+ }
1980
+
1776
1981
  async function recoverBoundScheduledTaskDispatch(input: {
1777
1982
  db: Database;
1778
1983
  bus: ControlActivityServices["bus"];
@@ -1952,6 +2157,18 @@ async function recoverBoundScheduledTaskDispatch(input: {
1952
2157
  sessionId: session.id,
1953
2158
  });
1954
2159
  }
2160
+ if (generatedSession) {
2161
+ session = await seedScheduledGeneratedSessionRoute({
2162
+ db: input.db,
2163
+ bus: input.bus,
2164
+ settings: input.settings,
2165
+ task,
2166
+ runId: input.run.id,
2167
+ session,
2168
+ deferPublications: false,
2169
+ deferredEvents: [],
2170
+ });
2171
+ }
1955
2172
  if (session.status === "cancelled") {
1956
2173
  await markScheduledTaskRunSkippedIfQueued(input.db, {
1957
2174
  workspaceId: task.workspaceId,
@@ -2024,6 +2241,7 @@ async function recoverBoundScheduledTaskDispatch(input: {
2024
2241
  session.latencyMode !== input.acceptedExecution.resolvedLatencyMode ||
2025
2242
  session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend ||
2026
2243
  session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs ||
2244
+ session.activeSandboxId !== (task.agentConfig.machineTarget?.targetSandboxId ?? null) ||
2027
2245
  stableJson(session.resources) !== stableJson(task.agentConfig.resources) ||
2028
2246
  stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) ||
2029
2247
  stableJson(session.firstPartyMcpTools) !==
@@ -19,7 +19,7 @@ import {
19
19
  } from "@opengeni/db";
20
20
  import { publishDurableSessionEvents } from "@opengeni/events";
21
21
  import { deliverFailedChildTurnToParent, notifyParentOfChildIdle } from "./parent-wake";
22
- import { recordTurnsQueuedGauge } from "../observability-metrics";
22
+ import { recordTurnsQueuedGauge, recordWorkerDeathRecoveryMetrics } from "../observability-metrics";
23
23
  import {
24
24
  MAX_AUTOMATIC_PROVIDER_RECOVERIES,
25
25
  providerRecoveryCountFromMetadata,
@@ -66,6 +66,7 @@ export type SessionStateActivityOverrides = Partial<{
66
66
  deliverFailedChildTurnToParent: typeof deliverFailedChildTurnToParent;
67
67
  notifyParentOfChildIdle: typeof notifyParentOfChildIdle;
68
68
  recordTurnsQueuedGauge: typeof recordTurnsQueuedGauge;
69
+ recordWorkerDeathRecoveryMetrics: typeof recordWorkerDeathRecoveryMetrics;
69
70
  }>;
70
71
 
71
72
  // Crash-loop guard for worker-death re-dispatch: a turn that takes a worker
@@ -112,6 +113,8 @@ export function createSessionStateActivities(
112
113
  overrides.deliverFailedChildTurnToParent ?? deliverFailedChildTurnToParent;
113
114
  const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
114
115
  const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
116
+ const recordWorkerDeathRecoveryMetricsFn =
117
+ overrides.recordWorkerDeathRecoveryMetrics ?? recordWorkerDeathRecoveryMetrics;
115
118
 
116
119
  async function failSessionAttempt(
117
120
  input: FailSessionAttemptInput,
@@ -397,6 +400,10 @@ export function createSessionStateActivities(
397
400
  if (result.action === "stale" || result.action === "unclaimed") {
398
401
  return { action: result.action };
399
402
  }
403
+ recordWorkerDeathRecoveryMetricsFn(observability, {
404
+ outcome: result.action === "exceeded" ? "exhausted" : "recovering",
405
+ timeoutType: input.timeoutType,
406
+ });
400
407
  await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
401
408
  await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
402
409
  if (result.action === "exceeded") {
@@ -10,6 +10,7 @@ import {
10
10
  canonicalConfiguredModel,
11
11
  createAndStartSessionWithOutcome,
12
12
  recordWorkspaceUsage,
13
+ resolveCatalogSettings,
13
14
  } from "@opengeni/core";
14
15
  import {
15
16
  claimSiteAuthMaintenance,
@@ -127,23 +128,22 @@ async function dispatchMaintenanceSession(
127
128
  assertModelPolicy: typeof assertWorkspaceModelPolicyAllows;
128
129
  },
129
130
  ): Promise<void> {
130
- const model = canonicalConfiguredModel(service.settings, service.settings.openaiModel);
131
+ const catalogSettings = (
132
+ await resolveCatalogSettings(service.db, service.catalogSourceSettings ?? service.settings)
133
+ ).settings;
134
+ const catalogService = { ...service, settings: catalogSettings };
135
+ const model = canonicalConfiguredModel(catalogSettings, catalogSettings.openaiModel);
131
136
  if (!model) throw new Error("site auth maintenance has no configured model");
132
- await dependencies.assertModelPolicy(
133
- service.db,
134
- service.settings,
135
- maintenance.workspaceId,
136
- model,
137
- );
138
- const denial = await dependencies.admit(service, {
137
+ await dependencies.assertModelPolicy(service.db, catalogSettings, maintenance.workspaceId, model);
138
+ const denial = await dependencies.admit(catalogService, {
139
139
  accountId: maintenance.accountId,
140
140
  workspaceId: maintenance.workspaceId,
141
141
  model,
142
142
  requestedAgentRuns: 1,
143
143
  });
144
144
  if (denial) throw new Error(`site auth maintenance admission denied: ${denial}`);
145
- const reasoningEffort = service.settings.openaiReasoningEffort;
146
- const turnExecutionPolicy = resolveTurnExecutionPolicyV1(service.settings, {
145
+ const reasoningEffort = catalogSettings.openaiReasoningEffort;
146
+ const turnExecutionPolicy = resolveTurnExecutionPolicyV1(catalogSettings, {
147
147
  modelId: model,
148
148
  requestedModelId: null,
149
149
  modelSource: "deployment",
@@ -180,7 +180,7 @@ async function dispatchMaintenanceSession(
180
180
  model,
181
181
  reasoningEffort,
182
182
  turnExecutionPolicy,
183
- sandboxBackend: service.settings.sandboxBackend,
183
+ sandboxBackend: catalogSettings.sandboxBackend,
184
184
  metadata: {
185
185
  role: "site_auth_maintenance",
186
186
  [SITE_AUTH_MAINTENANCE_CONNECTION_CONTEXT_KEY]: maintenance.siteAuthConnectionId,
@@ -205,7 +205,7 @@ async function dispatchMaintenanceSession(
205
205
  });
206
206
  try {
207
207
  await dependencies.recordUsage(
208
- { db: service.db, settings: service.settings },
208
+ { db: service.db, settings: catalogSettings },
209
209
  {
210
210
  accountId: maintenance.accountId,
211
211
  workspaceId: maintenance.workspaceId,
@@ -75,6 +75,11 @@ export type InspectSessionAttemptActivity = (input: {
75
75
  * agent runtime and document parser so each process can load only its role. */
76
76
  export type SharedActivityServices = {
77
77
  settings: Settings;
78
+ /**
79
+ * Original environment/host settings retained across live database-catalog
80
+ * refreshes. Older embedded test hosts may omit it and fall back to settings.
81
+ */
82
+ catalogSourceSettings?: Settings;
78
83
  db: Database;
79
84
  bus: EventBus;
80
85
  objectStorage: ObjectStorage | null;
@@ -461,6 +466,7 @@ export type DispatchAutomationRunResult =
461
466
  | { action: "started"; sessionId: string }
462
467
  | { action: "already_dispatched"; sessionId: string }
463
468
  | { action: "skipped"; reason: string }
469
+ | { action: "failed"; reason: string }
464
470
  | { action: "not_found" };
465
471
 
466
472
  export type RunKnowledgeSourceSyncBatchResult =
package/src/activities.ts CHANGED
@@ -107,6 +107,7 @@ export const expireSessionInteractionIntervention =
107
107
  export const markSessionIdle = defaultControlActivities.markSessionIdle;
108
108
  export const dispatchScheduledTaskRun = defaultControlActivities.dispatchScheduledTaskRun;
109
109
  export const dispatchAutomationRun = defaultControlActivities.dispatchAutomationRun;
110
+ export const settleAutomationRunFailure = defaultControlActivities.settleAutomationRunFailure;
110
111
  export const runKnowledgeSourceSyncBatch = defaultControlActivities.runKnowledgeSourceSyncBatch;
111
112
  export const enqueueGoalRetryWake = defaultControlActivities.enqueueGoalRetryWake;
112
113
  export const maybeContinueGoal = defaultControlActivities.maybeContinueGoal;
@@ -2,6 +2,7 @@ import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengen
2
2
  import { configureChildLifecycleNotices, createDb } from "@opengeni/db";
3
3
  import { createNatsEventBus } from "@opengeni/events";
4
4
  import { createObservability } from "@opengeni/observability";
5
+ import { resolveCatalogSettings } from "@opengeni/core";
5
6
  import { createObjectStorage } from "@opengeni/storage";
6
7
  import type { ActivityDependencies, SharedActivityServices } from "./activities/types";
7
8
  import { observabilityEventLogger } from "./observability-metrics";
@@ -35,11 +36,13 @@ export function createSharedActivityServices(
35
36
  });
36
37
  const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
37
38
  const db = dependencies.db ?? dbClient!.db;
39
+ const resolvedSettings = (await resolveCatalogSettings(db, settings)).settings;
38
40
  // Child lifecycle notice producers are process-global in @opengeni/db;
39
41
  // install the boot-validated rollout flag once for this worker.
40
42
  configureChildLifecycleNotices({ enabled: settings.childLifecycleNoticesEnabled });
41
43
  return {
42
- settings,
44
+ settings: resolvedSettings,
45
+ catalogSourceSettings: settings,
43
46
  db,
44
47
  bus:
45
48
  dependencies.bus ??
@@ -31,6 +31,7 @@ import {
31
31
  import { createGitHubAppInstallationTokenWithExpiry, githubAppBotIdentity } from "@opengeni/github";
32
32
  import {
33
33
  type AttemptConnectorActionBinding,
34
+ ConnectorActionBindingRejectedError,
34
35
  type LocalMcpServerRegistration,
35
36
  prefixedMcpToolName,
36
37
  } from "@opengeni/runtime";
@@ -258,9 +259,12 @@ function appRepositoryAuthority(resources: ResourceRef[]): GitHubRestRepository[
258
259
  );
259
260
  const repositoryId = positiveSafeInteger(resource.githubRepositoryId ?? resource.repositoryId);
260
261
  const fullName = canonicalGitHubFullName(resource.uri);
261
- if (!installationId || !repositoryId || !fullName) {
262
+ if (!installationId || !repositoryId) {
262
263
  throw new GitHubRestAuthorityError("GitHub App repository authority is incomplete");
263
264
  }
265
+ if (!fullName) {
266
+ throw new GitHubRestAuthorityError("GitHub App repository URL is invalid");
267
+ }
264
268
  return [
265
269
  {
266
270
  repositoryId: String(repositoryId),
@@ -621,7 +625,7 @@ function githubConnectorBindings(
621
625
  const repository =
622
626
  typeof args.repository === "string" ? byName.get(args.repository.toLowerCase()) : null;
623
627
  if (!repository) {
624
- throw new GitHubRestAuthorityError(
628
+ throw new ConnectorActionBindingRejectedError(
625
629
  "GitHub action repository does not match an accepted resource",
626
630
  );
627
631
  }
@@ -645,7 +649,8 @@ function githubConnectorBindings(
645
649
  function canonicalGitHubFullName(uri: string): string | null {
646
650
  try {
647
651
  const url = new URL(uri);
648
- const path = url.pathname.replace(/^\/+|\/+$/gu, "");
652
+ const clonePath = url.pathname.replace(/^\/+|\/+$/gu, "");
653
+ const path = clonePath.endsWith(".git") ? clonePath.slice(0, -4) : clonePath;
649
654
  if (
650
655
  url.protocol !== "https:" ||
651
656
  url.hostname !== "github.com" ||
@@ -654,7 +659,6 @@ function canonicalGitHubFullName(uri: string): string | null {
654
659
  url.password ||
655
660
  url.search ||
656
661
  url.hash ||
657
- path.endsWith(".git") ||
658
662
  !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(path)
659
663
  ) {
660
664
  return null;
package/src/http.ts CHANGED
@@ -6,6 +6,7 @@ import {
6
6
  type RuntimeDatabasePostureOptions,
7
7
  } from "@opengeni/db";
8
8
  import { requireSessionEventDurableFanoutCapability, type EventBus } from "@opengeni/events";
9
+ import { githubAppBotIdentityWarnings } from "@opengeni/github";
9
10
  import type { Observability } from "@opengeni/observability";
10
11
 
11
12
  export type ReadinessCheckName = "db" | "nats" | "temporal";
@@ -45,12 +46,14 @@ export function createWorkerHttpHandler(
45
46
  if (url.pathname === "/healthz") {
46
47
  const state = lifecycle?.state();
47
48
  const ok = state !== "failed" && state !== "stopped";
49
+ const warnings = githubAppBotIdentityWarnings(settings);
48
50
  return Response.json(
49
51
  {
50
52
  service: settings.serviceName,
51
53
  environment: settings.environment,
52
54
  deploymentRevision: settings.deploymentRevision,
53
55
  ...(lifecycle ? { role: lifecycle.role, state } : {}),
56
+ ...(warnings.length > 0 ? { warnings } : {}),
54
57
  ok,
55
58
  },
56
59
  { status: ok ? 200 : 503 },