@opengeni/worker-bundle 0.23.4 → 0.24.2-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +17 -2
  2. package/dist/activities/agent-turn/claim.d.ts +1 -0
  3. package/dist/activities/agent-turn/codex.d.ts +2 -2
  4. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  5. package/dist/activities/agent-turn/model-usage.d.ts +15 -1
  6. package/dist/activities/agent-turn/quiescence.d.ts +2 -0
  7. package/dist/activities/agent-turn/run.d.ts +9 -0
  8. package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
  9. package/dist/activities/agent-turn/session-title.d.ts +21 -0
  10. package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
  11. package/dist/activities/agent-turn/tool-environment.d.ts +2 -0
  12. package/dist/activities/agent-turn/tool-policy.d.ts +36 -5
  13. package/dist/activities/agent-turn/turn-context.d.ts +4 -1
  14. package/dist/activities/agent-turn.d.ts +1 -1
  15. package/dist/activities/automations.d.ts +1 -0
  16. package/dist/activities/capabilities.d.ts +3 -1
  17. package/dist/activities/context-compaction.d.ts +2 -0
  18. package/dist/activities/goals.d.ts +11 -0
  19. package/dist/activities/image-generation-references.d.ts +2 -1
  20. package/dist/activities/scheduled-tasks.d.ts +1 -0
  21. package/dist/activities/types.d.ts +8 -0
  22. package/dist/{activities-control-52W5IDWM.js → activities-control-ZWH7J4KZ.js} +297 -180
  23. package/dist/activities-control-ZWH7J4KZ.js.map +1 -0
  24. package/dist/activities-control.d.ts +2 -0
  25. package/dist/{activities-turn-A2G2KLSY.js → activities-turn-I4OPLQLS.js} +898 -333
  26. package/dist/activities-turn-I4OPLQLS.js.map +1 -0
  27. package/dist/activities.d.ts +2 -0
  28. package/dist/{chunk-GFNXVZVQ.js → chunk-65JEWP2Q.js} +107 -7
  29. package/dist/chunk-65JEWP2Q.js.map +1 -0
  30. package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
  31. package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
  32. package/dist/{chunk-IPVMCAW5.js → chunk-RQNH6CGR.js} +111 -62
  33. package/dist/chunk-RQNH6CGR.js.map +1 -0
  34. package/dist/index.d.ts +1 -0
  35. package/dist/index.js +43 -7
  36. package/dist/index.js.map +1 -1
  37. package/dist/observability-metrics.d.ts +34 -7
  38. package/dist/sandbox-reaper-timeout.d.ts +2 -2
  39. package/dist/sandbox-resume.d.ts +6 -1
  40. package/dist/workflow-bundle.js +26 -3
  41. package/dist/workflows/activities.d.ts +5 -0
  42. package/dist/workflows/automations.d.ts +5 -0
  43. package/package.json +17 -17
  44. package/src/activities/agent-run-admission.ts +7 -1
  45. package/src/activities/agent-turn/admission.ts +47 -6
  46. package/src/activities/agent-turn/claim.ts +33 -5
  47. package/src/activities/agent-turn/codex.ts +2 -2
  48. package/src/activities/agent-turn/compaction-prep.ts +8 -1
  49. package/src/activities/agent-turn/finalization.ts +31 -17
  50. package/src/activities/agent-turn/list-models.ts +101 -0
  51. package/src/activities/agent-turn/model-usage.ts +120 -45
  52. package/src/activities/agent-turn/quiescence.ts +11 -2
  53. package/src/activities/agent-turn/run.ts +80 -4
  54. package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
  55. package/src/activities/agent-turn/session-title.ts +55 -2
  56. package/src/activities/agent-turn/stream-attempt.ts +266 -135
  57. package/src/activities/agent-turn/tool-environment.ts +102 -7
  58. package/src/activities/agent-turn/tool-policy.ts +58 -5
  59. package/src/activities/agent-turn/turn-context.ts +8 -2
  60. package/src/activities/agent-turn.ts +5 -1
  61. package/src/activities/api-integrations.ts +11 -1
  62. package/src/activities/automations.ts +122 -67
  63. package/src/activities/capabilities.ts +129 -2
  64. package/src/activities/context-compaction.ts +3 -0
  65. package/src/activities/goals.ts +108 -31
  66. package/src/activities/image-generation-references.ts +9 -9
  67. package/src/activities/mcp-credentials.ts +53 -0
  68. package/src/activities/rig-verification.ts +5 -3
  69. package/src/activities/scheduled-tasks.ts +217 -110
  70. package/src/activities/site-auth-maintenance.ts +12 -12
  71. package/src/activities/types.ts +6 -0
  72. package/src/activities.ts +1 -0
  73. package/src/activity-services.ts +4 -1
  74. package/src/github-rest-mcp.ts +2 -1
  75. package/src/index.ts +34 -4
  76. package/src/observability-metrics.ts +154 -14
  77. package/src/sandbox-reaper-timeout.ts +10 -4
  78. package/src/sandbox-resume.ts +58 -8
  79. package/src/workflows/activities.ts +15 -0
  80. package/src/workflows/automations.ts +21 -2
  81. package/dist/activities-control-52W5IDWM.js.map +0 -1
  82. package/dist/activities-turn-A2G2KLSY.js.map +0 -1
  83. package/dist/chunk-GFNXVZVQ.js.map +0 -1
  84. package/dist/chunk-IPVMCAW5.js.map +0 -1
package/src/index.ts CHANGED
@@ -11,6 +11,7 @@ import {
11
11
  assertRuntimeDatabasePosture,
12
12
  countSessionRecoveryBacklog,
13
13
  createDb,
14
+ getContextCompactionPendingSummary,
14
15
  markSessionWorkflowWakeDelivered,
15
16
  type Database,
16
17
  type RuntimeDatabasePostureOptions,
@@ -56,14 +57,17 @@ import {
56
57
  type WorkerLifecycleState,
57
58
  } from "./http";
58
59
  import {
60
+ initializeContextCompactionMetrics,
59
61
  initializeWorkerOutcomeMetrics,
60
62
  normalizeTurnTaskQueueStats,
61
63
  observabilityEventLogger,
64
+ startContextCompactionPendingMonitor,
62
65
  startSessionRecoveryMonitor,
63
66
  startTurnCapacityMonitor,
64
67
  type TurnTaskQueueStats,
65
68
  } from "./observability-metrics";
66
69
  import {
70
+ resolveCatalogSettings,
67
71
  SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
68
72
  SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
69
73
  SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
@@ -224,6 +228,9 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
224
228
  options.activityDependencies?.observability ??
225
229
  createObservability(settings, { component: `worker-${options.role}` });
226
230
  initializeWorkerOutcomeMetrics(observability);
231
+ if (options.role === "turn") {
232
+ initializeContextCompactionMetrics(observability);
233
+ }
227
234
  if (options.role === "turn" && options.workflowBundle) {
228
235
  throw new Error("workflowBundle is valid only for the control worker role");
229
236
  }
@@ -799,11 +806,24 @@ export async function createOpenGeniWorkerService(
799
806
  let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
800
807
  let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
801
808
  let sessionRecoveryMonitor: ReturnType<typeof startSessionRecoveryMonitor> | undefined;
809
+ let contextCompactionPendingMonitor:
810
+ | ReturnType<typeof startContextCompactionPendingMonitor>
811
+ | undefined;
802
812
  const schedules: Array<{ close: () => Promise<void> }> = [];
803
813
  let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
804
814
  let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
805
815
 
806
816
  try {
817
+ const resolvedCatalog = await retryStartupDependency(
818
+ "model catalog",
819
+ () => resolveCatalogSettings(options.activityDependencies.db, settings),
820
+ { ...retryOptions, onRetry },
821
+ );
822
+ observability.info("OpenGeni model catalog resolved", {
823
+ role: options.role,
824
+ catalogSource: resolvedCatalog.source,
825
+ catalogVersion: resolvedCatalog.version,
826
+ });
807
827
  const needsSignaler =
808
828
  options.role === "turn" ||
809
829
  !options.activityDependencies.wakeSessionWorkflow ||
@@ -877,6 +897,10 @@ export async function createOpenGeniWorkerService(
877
897
  observability,
878
898
  read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db),
879
899
  });
900
+ contextCompactionPendingMonitor = startContextCompactionPendingMonitor({
901
+ observability,
902
+ read: async () => await getContextCompactionPendingSummary(options.activityDependencies.db),
903
+ });
880
904
  }
881
905
 
882
906
  if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
@@ -911,14 +935,18 @@ export async function createOpenGeniWorkerService(
911
935
  }
912
936
 
913
937
  if (options.http !== false) {
938
+ const databaseReady = dbReadyCheck(
939
+ options.http?.readinessDb ?? options.activityDependencies.db,
940
+ options.databasePosture,
941
+ );
914
942
  httpServer = startWorkerHttpServer({
915
943
  settings,
916
944
  observability,
917
945
  checks: {
918
- db: dbReadyCheck(
919
- options.http?.readinessDb ?? options.activityDependencies.db,
920
- options.databasePosture,
921
- ),
946
+ db: async () => {
947
+ await databaseReady();
948
+ await resolveCatalogSettings(options.activityDependencies.db, settings);
949
+ },
922
950
  nats: natsReadyCheck(options.activityDependencies.bus),
923
951
  temporal: temporalReadyCheck(workerBundle.connection),
924
952
  },
@@ -931,6 +959,7 @@ export async function createOpenGeniWorkerService(
931
959
  await Promise.allSettled([
932
960
  turnCapacityMonitor?.close(),
933
961
  sessionRecoveryMonitor?.close(),
962
+ contextCompactionPendingMonitor?.close(),
934
963
  workerBundle?.connection.close(),
935
964
  signaler?.close(),
936
965
  ...schedules.map((schedule) => schedule.close()),
@@ -954,6 +983,7 @@ export async function createOpenGeniWorkerService(
954
983
  await Promise.allSettled([
955
984
  turnCapacityMonitor?.close(),
956
985
  sessionRecoveryMonitor?.close(),
986
+ contextCompactionPendingMonitor?.close(),
957
987
  activeWorkerBundle.connection.close(),
958
988
  activeSignaler?.close(),
959
989
  ...schedules.map((schedule) => schedule.close()),
@@ -34,6 +34,10 @@ export type SessionRecoveryBacklog = {
34
34
  quiescence_missing: number;
35
35
  projection_stale: number;
36
36
  };
37
+ export type ContextCompactionPendingSummary = {
38
+ pendingCount: number;
39
+ oldestStartedAt: Date | null;
40
+ };
37
41
 
38
42
  export type TemporalTurnTaskQueueStats = {
39
43
  approximateBacklogCount?: unknown;
@@ -49,6 +53,7 @@ const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
49
53
  const modelRequestTrackers = new WeakMap<Observability, ModelRequestLifecycleMetrics>();
50
54
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
51
55
  const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
56
+ const initializedContextCompactionMetrics = new WeakSet<Observability>();
52
57
 
53
58
  // A worker process reaching one trillion tokens in one provider/cache counter is
54
59
  // already far outside an ordinary scrape lifetime. Refuse further increments
@@ -61,6 +66,18 @@ const WORKER_DEATH_RECOVERY_OUTCOMES: readonly WorkerDeathRecoveryOutcome[] = [
61
66
  "exhausted",
62
67
  ];
63
68
  const WORKER_DEATH_TIMEOUT_TYPES = ["heartbeat", "schedule_to_start"] as const;
69
+ const CONTEXT_COMPACTION_TRIGGERS = ["auto", "operator", "proactive", "overflow"] as const;
70
+
71
+ export type ContextCompactionTrigger = (typeof CONTEXT_COMPACTION_TRIGGERS)[number];
72
+
73
+ const CONTEXT_COMPACTION_STARTS_METRIC = {
74
+ name: "opengeni_context_compaction_starts_total",
75
+ help: "Total durable context compaction starts, by trigger.",
76
+ } as const;
77
+ const CONTEXT_COMPACTIONS_METRIC = {
78
+ name: "opengeni_context_compactions_total",
79
+ help: "Total completed context compactions, by trigger.",
80
+ } as const;
64
81
 
65
82
  export function observabilityEventLogger(observability: Observability): EventLogger {
66
83
  return {
@@ -346,6 +363,29 @@ export function initializeWorkerOutcomeMetrics(observability: Observability): vo
346
363
  }
347
364
  }
348
365
 
366
+ /**
367
+ * Publish every bounded compaction lifecycle series before the first event.
368
+ * The zero counters make the closed trigger catalog explicit before a scrape.
369
+ * Alerting uses the separate durable control-worker projection below; these
370
+ * process-local counters are lifecycle-rate diagnostics only.
371
+ */
372
+ export function initializeContextCompactionMetrics(observability: Observability): void {
373
+ if (initializedContextCompactionMetrics.has(observability)) return;
374
+ for (const trigger of CONTEXT_COMPACTION_TRIGGERS) {
375
+ observability.incrementCounter({
376
+ ...CONTEXT_COMPACTION_STARTS_METRIC,
377
+ labels: { trigger },
378
+ amount: 0,
379
+ });
380
+ observability.incrementCounter({
381
+ ...CONTEXT_COMPACTIONS_METRIC,
382
+ labels: { trigger },
383
+ amount: 0,
384
+ });
385
+ }
386
+ initializedContextCompactionMetrics.add(observability);
387
+ }
388
+
349
389
  export function recordWorkerDeathRecoveryMetrics(
350
390
  observability: Observability,
351
391
  input: {
@@ -830,6 +870,96 @@ export function startSessionRecoveryMonitor(input: {
830
870
  };
831
871
  }
832
872
 
873
+ export function recordContextCompactionPendingGauges(
874
+ observability: Observability,
875
+ summary: ContextCompactionPendingSummary,
876
+ nowMs = Date.now(),
877
+ ): void {
878
+ const oldestStartedAtMs = summary.oldestStartedAt?.getTime();
879
+ const oldestPendingAgeSeconds =
880
+ oldestStartedAtMs === undefined || !Number.isFinite(oldestStartedAtMs)
881
+ ? 0
882
+ : Math.max(0, (nowMs - oldestStartedAtMs) / 1_000);
883
+ observability.setGauge({
884
+ name: "opengeni_context_compaction_pending",
885
+ help: "Current exact active attempts whose latest automatic compaction landmark is still started.",
886
+ value: nonnegativeFinite(summary.pendingCount),
887
+ });
888
+ observability.setGauge({
889
+ name: "opengeni_context_compaction_oldest_pending_age_seconds",
890
+ help: "Age in seconds of the oldest durably pending automatic compaction, or zero when none are pending.",
891
+ value: oldestPendingAgeSeconds,
892
+ });
893
+ }
894
+
895
+ export function startContextCompactionPendingMonitor(input: {
896
+ observability: Observability;
897
+ read: () => Promise<ContextCompactionPendingSummary>;
898
+ intervalMs?: number;
899
+ now?: () => number;
900
+ }): { close: () => Promise<void> } {
901
+ const intervalMs = input.intervalMs ?? 60_000;
902
+ const now = input.now ?? Date.now;
903
+ const startedAt = now();
904
+ let lastSuccessAt: number | null = null;
905
+ let lastReadSucceeded = false;
906
+ let stopped = false;
907
+ let running: Promise<void> | null = null;
908
+ const recordStatus = () => {
909
+ const observedAt = now();
910
+ const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
911
+ const set = (name: string, help: string, value: number) =>
912
+ input.observability.setGauge({ name, help, value });
913
+ set(
914
+ "opengeni_context_compaction_monitor_last_read_success",
915
+ "Whether the latest durable context-compaction aggregate read completed successfully.",
916
+ lastReadSucceeded ? 1 : 0,
917
+ );
918
+ set(
919
+ "opengeni_context_compaction_monitor_last_success_timestamp_seconds",
920
+ "Unix timestamp of the latest successful durable context-compaction aggregate read, or zero before one succeeds.",
921
+ lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
922
+ );
923
+ set(
924
+ "opengeni_context_compaction_monitor_fresh",
925
+ "Whether durable context-compaction pending gauges have a successful read within three monitor intervals.",
926
+ lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
927
+ );
928
+ };
929
+ const refresh = () => {
930
+ recordStatus();
931
+ if (stopped || running) return;
932
+ running = input
933
+ .read()
934
+ .then((summary) => {
935
+ recordContextCompactionPendingGauges(input.observability, summary, now());
936
+ lastSuccessAt = now();
937
+ lastReadSucceeded = true;
938
+ recordStatus();
939
+ })
940
+ .catch((error) => {
941
+ lastReadSucceeded = false;
942
+ recordStatus();
943
+ input.observability.warn("context compaction monitor: durable aggregate read failed", {
944
+ error: error instanceof Error ? error.message : String(error),
945
+ });
946
+ })
947
+ .finally(() => {
948
+ running = null;
949
+ });
950
+ };
951
+ refresh();
952
+ const timer = setInterval(refresh, intervalMs);
953
+ timer.unref?.();
954
+ return {
955
+ close: async () => {
956
+ stopped = true;
957
+ clearInterval(timer);
958
+ await running;
959
+ },
960
+ };
961
+ }
962
+
833
963
  export function recordSandboxLeaseGauges(
834
964
  observability: Observability,
835
965
  counts: Partial<Record<SandboxLeaseLiveness, number>>,
@@ -1023,6 +1153,7 @@ export function recordSandboxRotationBacklogGauges(
1023
1153
  turnBlocked: number;
1024
1154
  directBlocked: number;
1025
1155
  processBlocked: number;
1156
+ interactionBlocked: number;
1026
1157
  },
1027
1158
  ): void {
1028
1159
  const values = {
@@ -1031,6 +1162,7 @@ export function recordSandboxRotationBacklogGauges(
1031
1162
  turn_blocked: backlog.turnBlocked,
1032
1163
  direct_blocked: backlog.directBlocked,
1033
1164
  process_blocked: backlog.processBlocked,
1165
+ interaction_blocked: backlog.interactionBlocked,
1034
1166
  } as const;
1035
1167
  for (const [kind, value] of Object.entries(values)) {
1036
1168
  observability.setGauge({
@@ -1749,15 +1881,14 @@ export function recordSessionEventPublishLatency(
1749
1881
  });
1750
1882
  }
1751
1883
 
1752
- // Context tokens per response span a wide range; buckets track the pressure toward
1753
- // a model's window so "sessions are running hot but never compacting" is queryable.
1884
+ // Context tokens per response span a wide range; buckets retain context-pressure
1885
+ // diagnostics while compaction alerting follows the durable model-aware start.
1754
1886
  const MODEL_INPUT_TOKENS_BUCKETS = [
1755
1887
  1_000, 5_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 300_000, 500_000, 1_000_000,
1756
1888
  ];
1757
1889
 
1758
- /** Observed input (context) tokens per model response — the context-pressure
1759
- * signal. Paired with `opengeni_context_compactions_total`, it makes the
1760
- * "compaction never firing while contexts run hot" failure mode expressible. */
1890
+ /** Observed input (context) tokens per model response for context-pressure
1891
+ * dashboards and provider diagnostics. */
1761
1892
  export function recordModelInputTokens(
1762
1893
  observability: Observability,
1763
1894
  provider: string,
@@ -1776,15 +1907,24 @@ export function recordModelInputTokens(
1776
1907
  });
1777
1908
  }
1778
1909
 
1779
- /** A context compaction actually fired, by trigger (operator | overflow | proactive
1780
- * | auto). The rate of this — against the input-tokens histogram above — is how an
1781
- * operator sees compaction working (or silently not). */
1782
- export function recordContextCompaction(observability: Observability, trigger: string): void {
1783
- observability.incrementCounter({
1784
- name: "opengeni_context_compactions_total",
1785
- help: "Total context compactions performed, by trigger.",
1786
- labels: { trigger },
1787
- });
1910
+ /** A context compaction successfully completed, by bounded trigger. */
1911
+ export function recordContextCompaction(
1912
+ observability: Observability,
1913
+ trigger: ContextCompactionTrigger,
1914
+ ): void {
1915
+ observability.incrementCounter({ ...CONTEXT_COMPACTIONS_METRIC, labels: { trigger } });
1916
+ }
1917
+
1918
+ /**
1919
+ * A durable context-compaction start, recorded only after the attempt-fenced
1920
+ * `compaction.started` transaction commits. This is the model-aware threshold
1921
+ * signal; it does not guess from a static token count shared by unlike models.
1922
+ */
1923
+ export function recordContextCompactionStarted(
1924
+ observability: Observability,
1925
+ trigger: ContextCompactionTrigger,
1926
+ ): void {
1927
+ observability.incrementCounter({ ...CONTEXT_COMPACTION_STARTS_METRIC, labels: { trigger } });
1788
1928
  }
1789
1929
 
1790
1930
  const MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2_048, 8_192];
@@ -1,6 +1,7 @@
1
1
  import {
2
2
  SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
3
3
  SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS,
4
+ effectiveSandboxDrainSnapshotTimeoutMs,
4
5
  sandboxArchiveCaptureTimeoutMs,
5
6
  type Settings,
6
7
  } from "@opengeni/config";
@@ -13,16 +14,21 @@ import {
13
14
 
14
15
  const STRICT_TIMEOUT_FENCE_MS = 1;
15
16
 
16
- export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTimeoutMs">): {
17
+ export function sandboxDrainTiming(
18
+ settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
19
+ ): {
17
20
  snapshotTimeoutMs: number;
18
21
  captureTimeoutMs: number;
19
22
  timeoutClass: SandboxDrainTimeoutClass;
20
23
  activityTimeoutMs: number;
21
24
  } {
22
- const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
25
+ const snapshotTimeoutMs = effectiveSandboxDrainSnapshotTimeoutMs(settings);
26
+ const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs({
27
+ sandboxSnapshotTimeoutMs: snapshotTimeoutMs,
28
+ });
23
29
  const timeoutClass = sandboxDrainTimeoutClass(captureTimeoutMs);
24
30
  return {
25
- snapshotTimeoutMs: settings.sandboxSnapshotTimeoutMs,
31
+ snapshotTimeoutMs,
26
32
  captureTimeoutMs,
27
33
  timeoutClass,
28
34
  activityTimeoutMs: sandboxDrainActivityTimeoutMs(timeoutClass),
@@ -30,7 +36,7 @@ export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTime
30
36
  }
31
37
 
32
38
  export function assertSandboxReaperActivityTimeout(
33
- settings: Pick<Settings, "sandboxSnapshotTimeoutMs">,
39
+ settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
34
40
  ): void {
35
41
  const timing = sandboxDrainTiming(settings);
36
42
  if (timing.snapshotTimeoutMs >= timing.captureTimeoutMs) {
@@ -709,7 +709,13 @@ export async function persistSandboxDeadlineRotationCheckpoint(
709
709
  * the user's machine IS the persistence), or a snapshot newer than the
710
710
  * interval already rides the lease.
711
711
  */
712
- export async function maybePersistWarmWorkspaceSnapshot(
712
+ export type WarmWorkspaceSnapshotPromise = Promise<boolean> & {
713
+ /** Settles only after any provider capture continuation has published or
714
+ * failed and released its exact workspace-capture admission. */
715
+ readonly settled: Promise<void>;
716
+ };
717
+
718
+ export function maybePersistWarmWorkspaceSnapshot(
713
719
  services: SandboxResumeServices,
714
720
  ids: {
715
721
  accountId: string;
@@ -723,6 +729,50 @@ export async function maybePersistWarmWorkspaceSnapshot(
723
729
  leaseEpoch: number,
724
730
  signal?: AbortSignal,
725
731
  force = false,
732
+ ): WarmWorkspaceSnapshotPromise {
733
+ let continuationRegistered = false;
734
+ let resolvePhysicalSettlement: () => void = () => undefined;
735
+ const physicalSettlement = new Promise<void>((resolve) => {
736
+ resolvePhysicalSettlement = resolve;
737
+ });
738
+ const result = persistWarmWorkspaceSnapshot(
739
+ services,
740
+ ids,
741
+ session,
742
+ leaseEpoch,
743
+ signal,
744
+ force,
745
+ (continuation) => {
746
+ continuationRegistered = true;
747
+ void continuation.then(resolvePhysicalSettlement, resolvePhysicalSettlement);
748
+ },
749
+ );
750
+ void result.then(
751
+ () => {
752
+ if (!continuationRegistered) resolvePhysicalSettlement();
753
+ },
754
+ () => {
755
+ if (!continuationRegistered) resolvePhysicalSettlement();
756
+ },
757
+ );
758
+ return Object.assign(result, { settled: physicalSettlement });
759
+ }
760
+
761
+ async function persistWarmWorkspaceSnapshot(
762
+ services: SandboxResumeServices,
763
+ ids: {
764
+ accountId: string;
765
+ workspaceId: string;
766
+ sessionId: string;
767
+ turnId: string;
768
+ attemptId: string;
769
+ sandboxGroupId: string;
770
+ },
771
+ session: unknown,
772
+ leaseEpoch: number,
773
+ signal: AbortSignal | undefined,
774
+ force: boolean,
775
+ registerOwnedContinuation: (continuation: Promise<void>) => void,
726
776
  ): Promise<boolean> {
727
777
  const { db, settings } = services;
728
778
  const intervalMs = settings.sandboxSnapshotIntervalMs;
@@ -989,6 +1039,7 @@ export async function maybePersistWarmWorkspaceSnapshot(
989
1039
  return { kind: "settled" as const, persisted: false };
990
1040
  },
991
1041
  );
1042
+ registerOwnedContinuation(settled.then(() => undefined));
992
1043
  let cancelListener: (() => void) | undefined;
993
1044
  const outcome = await Promise.race([
994
1045
  settled,
@@ -1223,13 +1274,12 @@ export async function resumeBoxForTurn(
1223
1274
  // holder is gone / its attempt is no longer the active writer" from "the lease
1224
1275
  // could not be extended" (rotation requested, draining, another epoch). The
1225
1276
  // second is NOT a reason to release: once a provider-deadline rotation is
1226
- // requested the lease heartbeat is fenced by design while the still-running
1227
- // turn checkpoints, and the turn-side heartbeat owns that orderly
1228
- // SandboxDeadlineRotationError abort. Dropping the live holder here instead
1229
- // flips the lease to draining under the running turn, the reaper terminates
1230
- // the box, and the checkpoint then fails `attempt_fenced` (the staging
1231
- // ~61-minute box-age failure). Both loops therefore converge on
1232
- // "holder alive -> never release"; this loop keeps touching the holder.
1277
+ // requested the turn-side heartbeat immediately preempts the attempt, whose
1278
+ // finalizer retains this holder until every attempt-owned writer is physically
1279
+ // drained. Dropping it here would let the reaper capture or terminate while a
1280
+ // provider operation may still be mutating /workspace. Both loops therefore
1281
+ // converge on "holder alive -> never release"; finalization performs the
1282
+ // proof-bearing release.
1233
1283
  const releaseDeadHolder = (reason: string): void => {
1234
1284
  const message = "sandbox resume holder released: holder no longer live";
1235
1285
  const fields = {
@@ -68,6 +68,21 @@ export const automationActivity = proxyActivities<Pick<typeof activities, "dispa
68
68
  },
69
69
  );
70
70
 
71
+ /** Dispatch has a bounded retry window, but an accepted automation event must
72
+ * not remain durably `dispatching` after that window closes. This compensating
73
+ * database write is idempotent and retries without a cap until the run reaches
74
+ * a terminal state (or is observed as already dispatched). */
75
+ export const automationFailureActivity = proxyActivities<
76
+ Pick<typeof activities, "settleAutomationRunFailure">
77
+ >({
78
+ startToCloseTimeout: "2 minutes",
79
+ retry: {
80
+ initialInterval: "1 second",
81
+ backoffCoefficient: 2,
82
+ maximumInterval: "30 seconds",
83
+ },
84
+ });
85
+
71
86
  /** Goal continuation evaluates a durable Postgres obligation at an idle
72
87
  * boundary. A transient failure gets a short retry window, then records an
73
88
  * explicit delayed outbox wake instead of relying on an unrelated mutation or
@@ -1,4 +1,4 @@
1
- import { automationActivity } from "./activities";
1
+ import { automationActivity, automationFailureActivity } from "./activities";
2
2
 
3
3
  export type AutomationRunWorkflowInput = {
4
4
  accountId: string;
@@ -6,6 +6,25 @@ export type AutomationRunWorkflowInput = {
6
6
  runId: string;
7
7
  };
8
8
 
9
+ export type AutomationRunWorkflowActivities = {
10
+ dispatchAutomationRun: (input: AutomationRunWorkflowInput) => Promise<unknown>;
11
+ settleAutomationRunFailure: (input: AutomationRunWorkflowInput) => Promise<void>;
12
+ };
13
+
14
+ export async function runAutomationRunWorkflow(
15
+ input: AutomationRunWorkflowInput,
16
+ activities: AutomationRunWorkflowActivities,
17
+ ): Promise<void> {
18
+ try {
19
+ await activities.dispatchAutomationRun(input);
20
+ } catch {
21
+ await activities.settleAutomationRunFailure(input);
22
+ }
23
+ }
24
+
9
25
  export async function automationRunWorkflow(input: AutomationRunWorkflowInput): Promise<void> {
10
- await automationActivity.dispatchAutomationRun(input);
26
+ await runAutomationRunWorkflow(input, {
27
+ dispatchAutomationRun: automationActivity.dispatchAutomationRun,
28
+ settleAutomationRunFailure: automationFailureActivity.settleAutomationRunFailure,
29
+ });
11
30
  }