@opengeni/worker-bundle 0.22.7-canary.0 → 0.23.3-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  2. package/dist/activities/agent-turn/claim.d.ts +2 -4
  3. package/dist/activities/agent-turn/codex.d.ts +6 -5
  4. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  5. package/dist/activities/agent-turn/errors.d.ts +13 -0
  6. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  7. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  8. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  9. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  10. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  11. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  12. package/dist/activities/agent-turn/tool-policy.d.ts +3 -32
  13. package/dist/activities/agent-turn/turn-context.d.ts +1 -8
  14. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  15. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  16. package/dist/activities/codex-rotation.d.ts +2 -0
  17. package/dist/activities/session-state.d.ts +2 -1
  18. package/dist/{activities-control-VZIIV7RX.js → activities-control-52W5IDWM.js} +125 -13
  19. package/dist/activities-control-52W5IDWM.js.map +1 -0
  20. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-A2G2KLSY.js} +534 -482
  21. package/dist/activities-turn-A2G2KLSY.js.map +1 -0
  22. package/dist/{chunk-NAVSAZ56.js → chunk-GFNXVZVQ.js} +160 -1
  23. package/dist/chunk-GFNXVZVQ.js.map +1 -0
  24. package/dist/{chunk-O4HDX5SC.js → chunk-IPVMCAW5.js} +144 -15
  25. package/dist/chunk-IPVMCAW5.js.map +1 -0
  26. package/dist/index.js +42 -2
  27. package/dist/index.js.map +1 -1
  28. package/dist/observability-metrics.d.ts +28 -0
  29. package/dist/personal-github-git-credentials.d.ts +1 -0
  30. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  31. package/package.json +18 -18
  32. package/src/activities/agent-turn/agent-build.ts +15 -31
  33. package/src/activities/agent-turn/claim.ts +29 -68
  34. package/src/activities/agent-turn/codex.ts +19 -10
  35. package/src/activities/agent-turn/compaction-prep.ts +50 -1
  36. package/src/activities/agent-turn/errors.ts +78 -0
  37. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  38. package/src/activities/agent-turn/finalization.ts +24 -21
  39. package/src/activities/agent-turn/history.ts +5 -1
  40. package/src/activities/agent-turn/quiescence.ts +86 -0
  41. package/src/activities/agent-turn/run-credentials.ts +12 -1
  42. package/src/activities/agent-turn/run.ts +14 -11
  43. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  44. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  45. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  46. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  47. package/src/activities/agent-turn/stream-attempt.ts +98 -14
  48. package/src/activities/agent-turn/tool-environment.ts +10 -13
  49. package/src/activities/agent-turn/tool-policy.ts +3 -55
  50. package/src/activities/agent-turn/turn-context.ts +8 -14
  51. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  52. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  53. package/src/activities/codex-capacity.ts +13 -5
  54. package/src/activities/codex-rotation.ts +17 -0
  55. package/src/activities/goals.ts +10 -4
  56. package/src/activities/retained-screenshots.ts +104 -21
  57. package/src/activities/sandbox-lease.ts +7 -4
  58. package/src/activities/scheduled-tasks.ts +128 -4
  59. package/src/activities/session-state.ts +8 -1
  60. package/src/github-rest-mcp.ts +6 -3
  61. package/src/http.ts +3 -0
  62. package/src/index.ts +19 -0
  63. package/src/observability-metrics.ts +200 -0
  64. package/src/personal-github-git-credentials.ts +45 -4
  65. package/src/sandbox-resume.ts +1 -1
  66. package/src/sandbox-routing.ts +7 -4
  67. package/src/unhandled-rejection-boundary.ts +73 -0
  68. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  69. package/dist/activities/recording.d.ts +0 -90
  70. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  71. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  72. package/dist/chunk-NAVSAZ56.js.map +0 -1
  73. package/dist/chunk-O4HDX5SC.js.map +0 -1
  74. package/src/activities/recording.ts +0 -279
@@ -32,7 +32,7 @@ import {
32
32
  scheduledUserMessagePayload,
33
33
  withFirstPartyTools,
34
34
  workflowIdForSession
35
- } from "./chunk-O4HDX5SC.js";
35
+ } from "./chunk-IPVMCAW5.js";
36
36
  import {
37
37
  inspectOpenSandboxKubernetesInventory,
38
38
  recordCreditBalanceGauges,
@@ -50,8 +50,9 @@ import {
50
50
  recordSandboxOrphansTerminated,
51
51
  recordSandboxRotationBacklogGauges,
52
52
  recordTurnsQueuedGauge,
53
+ recordWorkerDeathRecoveryMetrics,
53
54
  runtimeMetricsHooksForObservability
54
- } from "./chunk-NAVSAZ56.js";
55
+ } from "./chunk-GFNXVZVQ.js";
55
56
 
56
57
  // src/activities/browser-state-artifact-reaper.ts
57
58
  import {
@@ -6082,7 +6083,7 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
6082
6083
  }
6083
6084
  }
6084
6085
  try {
6085
- await settleRetainedProcess(db, {
6086
+ const settlement = await settleRetainedProcess(db, {
6086
6087
  accountId: process.accountId,
6087
6088
  workspaceId: process.workspaceId,
6088
6089
  sessionId: process.sessionId,
@@ -6094,14 +6095,17 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
6094
6095
  reason: proof.reason,
6095
6096
  idleGraceMs: settings.sandboxIdleGraceMs
6096
6097
  });
6098
+ if (settlement.process.state === "active") {
6099
+ throw new Error("Retained-process reconciliation returned an active durable process");
6100
+ }
6097
6101
  await settleSessionBackgroundCommandForRetainedProcess(db, {
6098
6102
  accountId: process.accountId,
6099
6103
  workspaceId: process.workspaceId,
6100
6104
  sessionId: process.sessionId,
6101
6105
  retainedProcessId: process.id,
6102
- outcome: proof.outcome,
6103
- exitCode: proof.exitCode,
6104
- reason: proof.reason
6106
+ outcome: settlement.process.state,
6107
+ exitCode: settlement.process.exitCode,
6108
+ reason: settlement.process.settlementReason ?? proof.reason
6105
6109
  });
6106
6110
  recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
6107
6111
  } catch (error) {
@@ -7232,7 +7236,8 @@ import {
7232
7236
  openGeniSlackBotMetadata,
7233
7237
  requireOpenGeniSlackBotConnection,
7234
7238
  resolveSessionToolPolicy,
7235
- scheduledSlackBotConnectionId
7239
+ scheduledSlackBotConnectionId,
7240
+ swapActiveSandbox
7236
7241
  } from "@opengeni/core";
7237
7242
  import {
7238
7243
  addSessionSystemUpdateWithSourceMutation,
@@ -7240,6 +7245,7 @@ import {
7240
7245
  createScheduledTaskRun,
7241
7246
  createSessionWithIdempotencyKeyResult,
7242
7247
  enqueueSessionWorkflowWakeIfRunnable,
7248
+ failScheduledGeneratedSessionRoute,
7243
7249
  getScheduledTask as getScheduledTask2,
7244
7250
  getScheduledTaskRunAcceptedExecution,
7245
7251
  getScheduledTaskRunByProducerKey,
@@ -7250,6 +7256,8 @@ import {
7250
7256
  getScheduledTaskPersonalResourceAuthoritySubject,
7251
7257
  getScheduledTaskRevisionAuthority,
7252
7258
  getScheduledTaskXaiProviderAccountAuthoritySnapshot,
7259
+ getEnrollment,
7260
+ getSandbox,
7253
7261
  getRig as getRig2,
7254
7262
  getScheduledScopedRigVersionMetadata,
7255
7263
  getNestedAgentDepthDeploymentPolicy,
@@ -7623,10 +7631,34 @@ function createScheduledTaskActivities(services) {
7623
7631
  const taskAuthoritySubjectId = taskRevisionAuthority?.subjectId ?? taskPersonalResourceAuthoritySubjectId ?? taskConnectionAuthoritySubjectId;
7624
7632
  const model = task.agentConfig.model ?? settings.openaiModel;
7625
7633
  const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
7626
- const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
7634
+ let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
7635
+ let sandboxOs = "linux";
7627
7636
  const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
7628
7637
  const firstPartyMcpTools = resolveFirstPartyMcpToolPolicy(settings).default;
7629
7638
  const generatedTarget = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
7639
+ if (generatedTarget && task.agentConfig.machineTarget) {
7640
+ const access = {
7641
+ accountId: task.accountId,
7642
+ workspaceId: task.workspaceId,
7643
+ subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
7644
+ };
7645
+ const machine = await getSandbox(
7646
+ db,
7647
+ access,
7648
+ task.agentConfig.machineTarget.targetSandboxId
7649
+ );
7650
+ if (!machine || machine.kind !== "selfhosted" || !machine.enrollmentId || machine.scope === "user") {
7651
+ throw new Error("scheduled Connected Machine target is unavailable");
7652
+ }
7653
+ const enrollment = await getEnrollment(db, access, machine.enrollmentId);
7654
+ if (!enrollment || enrollment.status !== "active") {
7655
+ throw new Error("scheduled Connected Machine enrollment is unavailable");
7656
+ }
7657
+ sandboxBackend = "selfhosted";
7658
+ sandboxOs = enrollment.os;
7659
+ } else if (generatedTarget && sandboxBackend === "selfhosted") {
7660
+ throw new Error("self-hosted scheduled task has no Connected Machine target");
7661
+ }
7630
7662
  const generatedSessionDepthPolicy = generatedTarget ? await (async () => {
7631
7663
  const workspace = await requireWorkspace(db, task.workspaceId);
7632
7664
  const deployment = await getNestedAgentDepthDeploymentPolicy(db);
@@ -7832,7 +7864,7 @@ function createScheduledTaskActivities(services) {
7832
7864
  resolvedReasoningEffort: reasoningEffort,
7833
7865
  resolvedLatencyMode: "standard",
7834
7866
  resolvedSandboxBackend: sandboxBackend,
7835
- resolvedSandboxOs: "linux",
7867
+ resolvedSandboxOs: sandboxOs,
7836
7868
  resolvedTools: taskTools,
7837
7869
  resolvedFirstPartyMcpTools: firstPartyMcpTools,
7838
7870
  resolvedFirstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
@@ -8038,7 +8070,7 @@ function createScheduledTaskActivities(services) {
8038
8070
  reasoningEffort,
8039
8071
  latencyMode: "standard",
8040
8072
  sandboxBackend,
8041
- sandboxOs: "linux",
8073
+ sandboxOs,
8042
8074
  variableSetId: task.variableSetId ?? null,
8043
8075
  rigId: frozenRigId,
8044
8076
  rigVersionId: frozenRigVersionId,
@@ -8138,6 +8170,16 @@ function createScheduledTaskActivities(services) {
8138
8170
  );
8139
8171
  }
8140
8172
  }
8173
+ session = await seedScheduledGeneratedSessionRoute({
8174
+ db: dispatchDb,
8175
+ bus,
8176
+ settings,
8177
+ task,
8178
+ runId: run.id,
8179
+ session,
8180
+ deferPublications,
8181
+ deferredEvents
8182
+ });
8141
8183
  let workflowId;
8142
8184
  if (alertOccurrence) {
8143
8185
  const started = await initializeSessionStartAtomically(dispatchDb, {
@@ -8531,7 +8573,6 @@ function createScheduledTaskActivities(services) {
8531
8573
  acceptedExecution: dispatchOutcome.acceptedExecution
8532
8574
  });
8533
8575
  }
8534
- if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
8535
8576
  for (const deferred of deferredEvents) {
8536
8577
  await publishDurableSessionEvents(
8537
8578
  bus,
@@ -8540,6 +8581,7 @@ function createScheduledTaskActivities(services) {
8540
8581
  deferred.events
8541
8582
  );
8542
8583
  }
8584
+ if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
8543
8585
  const result = dispatchOutcome.result;
8544
8586
  if (wakeSessionWorkflow && result.workflowWakeRevision !== null) {
8545
8587
  await wakeSessionWorkflow({
@@ -8714,6 +8756,59 @@ async function recordScheduledTaskFiredUsage(db, acceptedExecution, run) {
8714
8756
  idempotencyKey: `usage:scheduled_task.fired:${run.id}`
8715
8757
  });
8716
8758
  }
8759
+ async function seedScheduledGeneratedSessionRoute(input) {
8760
+ const target = input.task.agentConfig.machineTarget;
8761
+ if (!target) return input.session;
8762
+ const seeded = await swapActiveSandbox(
8763
+ {
8764
+ db: input.db,
8765
+ settings: input.settings,
8766
+ bus: input.bus
8767
+ },
8768
+ {
8769
+ accountId: input.task.accountId,
8770
+ workspaceId: input.task.workspaceId,
8771
+ sessionId: input.session.id,
8772
+ sessionBackend: input.session.sandboxBackend,
8773
+ sessionGroupId: input.session.sandboxGroupId
8774
+ },
8775
+ target.targetSandboxId,
8776
+ target.workingDir ?? null
8777
+ );
8778
+ if (seeded.swapped) {
8779
+ return await requireSession(input.db, input.task.workspaceId, input.session.id);
8780
+ }
8781
+ const failed = await failScheduledGeneratedSessionRoute(input.db, {
8782
+ accountId: input.task.accountId,
8783
+ workspaceId: input.task.workspaceId,
8784
+ taskId: input.task.id,
8785
+ runId: input.runId,
8786
+ sessionId: input.session.id,
8787
+ error: "scheduled_machine_unavailable"
8788
+ });
8789
+ if (failed.action === "advanced") {
8790
+ throw new Error(
8791
+ `scheduled run advanced to ${failed.status} while its Connected Machine route was being established`
8792
+ );
8793
+ }
8794
+ if (failed.events.length > 0) {
8795
+ if (input.deferPublications) {
8796
+ input.deferredEvents.push({ sessionId: input.session.id, events: failed.events });
8797
+ } else {
8798
+ await publishDurableSessionEvents(
8799
+ input.bus,
8800
+ input.task.workspaceId,
8801
+ input.session.id,
8802
+ failed.events
8803
+ );
8804
+ }
8805
+ }
8806
+ const error = failed.action === "terminal" ? failed.error ?? "scheduled_run_terminal" : failed.error;
8807
+ throw new ScheduledRunTerminalAuthorityError(
8808
+ error,
8809
+ seeded.reason ?? "scheduled Connected Machine target is unavailable"
8810
+ );
8811
+ }
8717
8812
  async function recoverBoundScheduledTaskDispatch(input) {
8718
8813
  const { task } = input.acceptedExecution;
8719
8814
  if (input.run.status !== "queued") {
@@ -8866,6 +8961,18 @@ async function recoverBoundScheduledTaskDispatch(input) {
8866
8961
  sessionId: session.id
8867
8962
  });
8868
8963
  }
8964
+ if (generatedSession) {
8965
+ session = await seedScheduledGeneratedSessionRoute({
8966
+ db: input.db,
8967
+ bus: input.bus,
8968
+ settings: input.settings,
8969
+ task,
8970
+ runId: input.run.id,
8971
+ session,
8972
+ deferPublications: false,
8973
+ deferredEvents: []
8974
+ });
8975
+ }
8869
8976
  if (session.status === "cancelled") {
8870
8977
  await markScheduledTaskRunSkippedIfQueued(input.db, {
8871
8978
  workspaceId: task.workspaceId,
@@ -8901,7 +9008,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
8901
9008
  label: "OpenGeni scheduler",
8902
9009
  scheduledTaskId: task.id,
8903
9010
  scheduledTaskRunId: canonicalGeneratedRunId
8904
- }) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || session.model !== input.acceptedExecution.resolvedModel || session.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || session.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
9011
+ }) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || session.model !== input.acceptedExecution.resolvedModel || session.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || session.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || session.activeSandboxId !== (task.agentConfig.machineTarget?.targetSandboxId ?? null) || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
8905
9012
  throw new ScheduledRunTerminalAuthorityError(
8906
9013
  "scheduled_generated_session_changed",
8907
9014
  "scheduled generated session differs from accepted execution"
@@ -9402,6 +9509,7 @@ function createSessionStateActivities(services, overrides = {}) {
9402
9509
  const deliverFailedChildTurnToParentFn = overrides.deliverFailedChildTurnToParent ?? deliverFailedChildTurnToParent;
9403
9510
  const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
9404
9511
  const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
9512
+ const recordWorkerDeathRecoveryMetricsFn = overrides.recordWorkerDeathRecoveryMetrics ?? recordWorkerDeathRecoveryMetrics;
9405
9513
  async function failSessionAttempt(input) {
9406
9514
  const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
9407
9515
  const session = await requireSessionFn(db, input.workspaceId, input.sessionId);
@@ -9624,6 +9732,10 @@ function createSessionStateActivities(services, overrides = {}) {
9624
9732
  if (result.action === "stale" || result.action === "unclaimed") {
9625
9733
  return { action: result.action };
9626
9734
  }
9735
+ recordWorkerDeathRecoveryMetricsFn(observability, {
9736
+ outcome: result.action === "exceeded" ? "exhausted" : "recovering",
9737
+ timeoutType: input.timeoutType
9738
+ });
9627
9739
  await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
9628
9740
  await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
9629
9741
  if (result.action === "exceeded") {
@@ -9825,4 +9937,4 @@ export {
9825
9937
  createControlActivities,
9826
9938
  createControlActivitiesFromServices
9827
9939
  };
9828
- //# sourceMappingURL=activities-control-VZIIV7RX.js.map
9940
+ //# sourceMappingURL=activities-control-52W5IDWM.js.map