@opengeni/worker-bundle 0.7.5 → 0.10.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/chunk-BJUW5LCM.js +325 -0
  2. package/dist/chunk-BJUW5LCM.js.map +1 -0
  3. package/dist/host-export.d.ts +70 -0
  4. package/dist/host-export.js +7 -0
  5. package/dist/host-export.js.map +1 -0
  6. package/dist/index.d.ts +98 -6
  7. package/dist/index.js +4097 -1271
  8. package/dist/index.js.map +1 -1
  9. package/dist/workflow-bundle.js +12360 -0
  10. package/package.json +18 -12
  11. package/src/activities/agent-turn.ts +2674 -828
  12. package/src/activities/capabilities.ts +11 -2
  13. package/src/activities/codex-capacity.ts +6 -11
  14. package/src/activities/codex-rotation.ts +75 -86
  15. package/src/activities/context-compaction.ts +104 -24
  16. package/src/activities/environment.ts +503 -97
  17. package/src/activities/expiring-material-renewal.ts +143 -0
  18. package/src/activities/goals.ts +4 -0
  19. package/src/activities/mcp-credentials.ts +38 -0
  20. package/src/activities/packs.ts +115 -2
  21. package/src/activities/rig-verification.ts +897 -244
  22. package/src/activities/run-credential-renewal.ts +70 -0
  23. package/src/activities/run-credentials.ts +132 -0
  24. package/src/activities/run-input.ts +23 -0
  25. package/src/activities/sandbox-lease.ts +33 -2
  26. package/src/activities/scheduled-tasks.ts +21 -0
  27. package/src/activities/session-state.ts +72 -0
  28. package/src/activities/streaming.ts +1 -0
  29. package/src/activities/toolspace-token-renewal.ts +57 -0
  30. package/src/activities/types.ts +44 -6
  31. package/src/activities/workspace-capture.ts +113 -93
  32. package/src/activities.ts +10 -2
  33. package/src/host-export-pump.ts +416 -0
  34. package/src/host-export.ts +22 -0
  35. package/src/http.ts +77 -28
  36. package/src/index.ts +360 -156
  37. package/src/observability-metrics.ts +83 -15
  38. package/src/sandbox-resume.ts +47 -1
  39. package/src/sandbox-routing.ts +1 -1
  40. package/src/worker-service-lifecycle.ts +104 -0
  41. package/src/workflows/activities.ts +13 -7
  42. package/src/workflows/rig-verification.ts +3 -1
  43. package/src/workflows/session.ts +219 -23
  44. package/src/workflows.ts +1 -0
@@ -1,10 +1,12 @@
1
1
  import {
2
2
  ActivityFailure,
3
+ ApplicationFailure,
3
4
  CancellationScope,
4
5
  condition,
5
6
  continueAsNew,
6
7
  defineSignal,
7
8
  isCancellation,
9
+ patched,
8
10
  setHandler,
9
11
  TimeoutFailure,
10
12
  uuid4,
@@ -33,6 +35,7 @@ import {
33
35
  */
34
36
  const TURNS_PER_RUN_BACKSTOP = 2_000;
35
37
  const CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP = 512;
38
+ const HUMAN_INPUT_EXPIRY_STALE_RETRY_MS = 1_000;
36
39
 
37
40
  /**
38
41
  * The minimum hold for a rotation all-capped idle (`idleUntilReset`). A MANDATORY
@@ -68,6 +71,24 @@ export function continuationHoldMs(
68
71
  return Math.max(delay, 0);
69
72
  }
70
73
 
74
+ /**
75
+ * A deferred activity result is terminal for this workflow run unless a new
76
+ * non-control wake committed while the activity was in flight. Keeping this
77
+ * predicate pure makes the compaction-convergence fence directly testable:
78
+ * repeated identical failures cannot synthesize more work from an unchanged
79
+ * durable state. Pause/Steer remain authoritative through the independent
80
+ * signal-version and interruption paths in `sessionWorkflow`.
81
+ */
82
+ export function deferredResultMayContinue(entryWakeups: number, currentWakeups: number): boolean {
83
+ return currentWakeups !== entryWakeups;
84
+ }
85
+
86
+ /** Deterministic Temporal timer delay for a persisted structured-input deadline. */
87
+ export function humanInputDeadlineWaitMs(expiresAt: string, nowMs = Date.now()): number {
88
+ const deadline = Date.parse(expiresAt);
89
+ return Number.isFinite(deadline) ? Math.max(0, deadline - nowMs) : 0;
90
+ }
91
+
71
92
  /**
72
93
  * True when an agent-turn activity failure means "the worker hosting the
73
94
  * turn died or vanished" rather than "the turn itself failed": the server
@@ -98,11 +119,34 @@ function workerDeathFailure(
98
119
  return { timeoutType: cause.timeoutType };
99
120
  }
100
121
 
122
+ /**
123
+ * Classify only the exact turn-fence cancellation protocol used to arbitrate a
124
+ * database control commit that reaches the activity just before its Temporal
125
+ * signal reaches this workflow. This shape is never physical-quiescence proof:
126
+ * only the activity's durable post-tool-fence receipt can open replacement
127
+ * admission. @temporalio/worker 1.20 serializes the pre-request
128
+ * CancelledFailure as an ApplicationFailure with the stable `CancelledFailure`
129
+ * type, so accept that exact wire shape in addition to normal cancellation.
130
+ */
131
+ export function isTurnActivityFenceCancellation(error: unknown): boolean {
132
+ if (isCancellation(error)) return true;
133
+ if (!(error instanceof ActivityFailure)) return false;
134
+ if (error.cause !== undefined && isCancellation(error.cause)) return true;
135
+ return (
136
+ error.activityType === "runAgentTurn" &&
137
+ error.cause instanceof ApplicationFailure &&
138
+ error.cause.type === "CancelledFailure" &&
139
+ error.cause.message === "TURN_ATTEMPT_FENCED"
140
+ );
141
+ }
142
+
101
143
  export const userMessage = defineSignal<[string]>("userMessage");
102
144
  export const queueChanged = defineSignal("queueChanged");
103
145
  export const approvalDecision = defineSignal<[string]>("approvalDecision");
104
146
  export const sessionControl = defineSignal("sessionControl");
105
147
  export const codexCapacityChanged = defineSignal<[number]>("codexCapacityChanged");
148
+ export const sessionAttemptQuiesced =
149
+ defineSignal<[activities.SessionAttemptQuiescenceProof]>("sessionAttemptQuiesced");
106
150
 
107
151
  export type SessionWorkflowInput = {
108
152
  accountId: string;
@@ -120,13 +164,19 @@ export type SessionWorkflowInput = {
120
164
  };
121
165
 
122
166
  export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void> {
123
- const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue);
167
+ // Existing histories recorded the v1 WAIT_CANCELLATION_COMPLETED command
168
+ // order and workflow-side idempotent fallback. Keep that exact replay path;
169
+ // every new run records v2 and uses the activity-owned receipt contract.
170
+ const receiptGatedCancellation = patched("session-attempt-quiescence-v2");
171
+ const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue, receiptGatedCancellation);
124
172
  let approvalWakeups = 0;
125
173
  let interruptionWakeups = 0;
126
174
  let wakeups = 0;
127
175
  let capacityWakeups = 0;
128
176
  let signalVersion = 0;
129
177
  let nonControlSignalVersion = 0;
178
+ const pendingQuiescenceProofs = new Map<string, activities.SessionAttemptQuiescenceProof>();
179
+ const persistedQuiescenceProofs = new Set<string>();
130
180
  // Turns dispatched on THIS run (reset to 0 by continueAsNew). The backstop
131
181
  // for the history-overflow guard below; bounded growth is what makes a
132
182
  // weeks-long session survivable.
@@ -157,6 +207,38 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
157
207
  nonControlSignalVersion += 1;
158
208
  capacityWakeups += 1;
159
209
  });
210
+ setHandler(sessionAttemptQuiesced, (proof) => {
211
+ // Temporal signals are untrusted transport payloads. Accept only the exact
212
+ // workflow/session scope; the DB control activity additionally validates
213
+ // the persisted attempt's account, run id, and activity id under lock.
214
+ if (
215
+ !isSessionAttemptQuiescenceProof(proof) ||
216
+ proof.accountId !== input.accountId ||
217
+ proof.workspaceId !== input.workspaceId ||
218
+ proof.sessionId !== input.sessionId ||
219
+ proof.workflowId !== workflowInfo().workflowId
220
+ ) {
221
+ return;
222
+ }
223
+ const key = sessionAttemptQuiescenceProofKey(proof);
224
+ if (persistedQuiescenceProofs.has(key) || pendingQuiescenceProofs.has(key)) return;
225
+ pendingQuiescenceProofs.set(key, proof);
226
+ signalVersion += 1;
227
+ });
228
+
229
+ async function persistPendingQuiescenceProofs(): Promise<void> {
230
+ while (pendingQuiescenceProofs.size > 0) {
231
+ const entry = pendingQuiescenceProofs.entries().next().value;
232
+ if (!entry) return;
233
+ const [key, proof] = entry;
234
+ // This DB-only activity uses an unbounded Temporal retry policy. The
235
+ // workflow cannot peek, close, or continue-as-new until the exact
236
+ // physical proof commits its idempotent receipt and wake transaction.
237
+ await activity.persistSessionAttemptQuiescence(proof);
238
+ pendingQuiescenceProofs.delete(key);
239
+ persistedQuiescenceProofs.add(key);
240
+ }
241
+ }
160
242
 
161
243
  async function waitForCodexCapacity(
162
244
  initial: activities.CodexCapacityWaitRef,
@@ -217,6 +299,11 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
217
299
  // The waiter/outbox is durable in Postgres. A fresh workflow run reads
218
300
  // it before goal continuation, reconstructs its timer, and turns any
219
301
  // unobserved wake revision into an immediate evaluation.
302
+ // A quiescence proof is the one signal that is not merely a replaceable
303
+ // wake hint. It may have arrived while the reconciliation activity was
304
+ // running, so commit it before crossing this nested continue-as-new
305
+ // boundary.
306
+ await persistPendingQuiescenceProofs();
220
307
  await continueAsNew<typeof sessionWorkflow>({
221
308
  accountId: input.accountId,
222
309
  workspaceId: input.workspaceId,
@@ -232,18 +319,25 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
232
319
  }
233
320
 
234
321
  while (true) {
322
+ // A proof signal can signalWithStart a fresh run or arrive at any prior
323
+ // close/continue-as-new boundary. Persist it before every ordinary workflow
324
+ // decision so no accepted physical fact can be dropped with run history.
325
+ await persistPendingQuiescenceProofs();
235
326
  // History-overflow guard. The top of the loop is the only safe
236
327
  // continueAsNew boundary: no turn is mid-flight (every path that reaches a
237
328
  // new iteration settled or recovered its turn first). Every interruption
238
- // is already durable in Postgres, so all Temporal signals are replaceable
239
- // wake hints and none must be carried into the next workflow run. A
329
+ // is already durable in Postgres, so ordinary Temporal signals are
330
+ // replaceable wake hints and none must be carried into the next workflow
331
+ // run. A
240
332
  // buffered userMessage/queueChanged signal only
241
333
  // bumps `wakeups`, and its turn was written to Postgres BEFORE the signal
242
334
  // was sent, so the fresh run observes it on its first durable work peek
243
335
  // — losing the counter strands nothing. The queue living in Postgres is the
244
336
  // safety net: continueAsNew carries only the self-contained
245
337
  // SessionWorkflowInput (no initialEventId — the new run claims from the
246
- // queue, it does not replay a seed event).
338
+ // queue, it does not replay a seed event). Quiescence-proof signals are the
339
+ // exception: persistPendingQuiescenceProofs() immediately above commits
340
+ // every accepted proof before this boundary.
247
341
  //
248
342
  // Approval signals are wakeups, never conversation truth. A genuinely
249
343
  // accepted decision is already persisted on the turn and is rediscovered
@@ -285,6 +379,9 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
285
379
  workflowId,
286
380
  });
287
381
  if (settlement.action === "paused") {
382
+ if (pendingQuiescenceProofs.size > 0) {
383
+ continue;
384
+ }
288
385
  if (nonControlSignalVersion !== closeNonControlSignalVersion) {
289
386
  continue;
290
387
  }
@@ -292,6 +389,22 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
292
389
  }
293
390
  continue;
294
391
  }
392
+ if (peek.kind === "cancellation-wait") {
393
+ // Logical settlement is complete, but the exact predecessor activity has
394
+ // not yet durably proved sandbox/tool quiescence. Wait only briefly for
395
+ // its transactional queueChanged wake. If provider/tool cancellation is
396
+ // genuinely slow, close this workflow run rather than consuming a turn
397
+ // slot or churning control activities; the outbox uses signalWithStart to
398
+ // restart this exact workflow after the receipt commits.
399
+ const seenSignalVersion = signalVersion;
400
+ const woke = await condition(() => signalVersion !== seenSignalVersion, "5s");
401
+ if (woke) continue;
402
+ // Close only against the same signal snapshot. A proof signal accepted
403
+ // at the timer/completion boundary must loop through the DB-only receipt
404
+ // activity rather than disappearing with this workflow run.
405
+ if (signalVersion !== seenSignalVersion || pendingQuiescenceProofs.size > 0) continue;
406
+ return;
407
+ }
295
408
  if (peek.kind === "capacity-wait") {
296
409
  await waitForCodexCapacity(peek.ref);
297
410
  continue;
@@ -300,12 +413,33 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
300
413
  const seenApprovalWakeups = approvalWakeups;
301
414
  const seenWakeups = wakeups;
302
415
  const seenInterruptionWakeups = interruptionWakeups;
303
- await condition(
304
- () =>
305
- interruptionWakeups !== seenInterruptionWakeups ||
306
- approvalWakeups !== seenApprovalWakeups ||
307
- wakeups !== seenWakeups,
308
- );
416
+ const timeoutMs =
417
+ peek.humanInputRequestId && peek.expiresAt
418
+ ? humanInputDeadlineWaitMs(peek.expiresAt)
419
+ : undefined;
420
+ const wakeCondition = () =>
421
+ interruptionWakeups !== seenInterruptionWakeups ||
422
+ approvalWakeups !== seenApprovalWakeups ||
423
+ wakeups !== seenWakeups;
424
+ const woke =
425
+ timeoutMs === undefined
426
+ ? await condition(wakeCondition)
427
+ : await condition(wakeCondition, timeoutMs);
428
+ if (!woke && peek.humanInputRequestId) {
429
+ const expiry = await activity.expireSessionHumanInput({
430
+ accountId: input.accountId,
431
+ workspaceId: input.workspaceId,
432
+ sessionId: input.sessionId,
433
+ requestId: peek.humanInputRequestId,
434
+ });
435
+ // Temporal and PostgreSQL may observe slightly different wall clocks.
436
+ // If the workflow timer fired first, the DB settler truthfully leaves
437
+ // the request pending. Bound that skew with one interruptible timer
438
+ // instead of spinning peek + activity at zero delay.
439
+ if (expiry.action === "stale") {
440
+ await condition(wakeCondition, HUMAN_INPUT_EXPIRY_STALE_RETRY_MS);
441
+ }
442
+ }
309
443
  continue;
310
444
  }
311
445
  if (peek.kind === "idle") {
@@ -389,7 +523,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
389
523
  trigger,
390
524
  }),
391
525
  );
392
- const outcome:
526
+ const racedOutcome:
393
527
  | { kind: "result"; result: activities.RunAgentTurnResult }
394
528
  | { kind: "control" }
395
529
  | { kind: "failure"; error: unknown } = await Promise.race([
@@ -404,9 +538,55 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
404
538
  kind: "control" as const,
405
539
  })),
406
540
  ]);
541
+ // A Steer/Pause transaction fences the active attempt before its Temporal
542
+ // signal is necessarily handled. That fence can make the activity's typed
543
+ // cancellation win Promise.race by one workflow activation. Give only that
544
+ // confirmed cancellation shape a short deterministic arbitration window;
545
+ // the durable control signal is authoritative once observed. Ordinary
546
+ // failures and timeouts never wait here, and a cancellation with no session
547
+ // control still follows the failure/cancellation path below.
548
+ if (
549
+ racedOutcome.kind === "failure" &&
550
+ isTurnActivityFenceCancellation(racedOutcome.error) &&
551
+ interruptionWakeups === interruptionBaseline
552
+ ) {
553
+ await condition(() => interruptionWakeups !== interruptionBaseline, "250ms");
554
+ }
555
+ const outcome =
556
+ interruptionWakeups !== interruptionBaseline ? ({ kind: "control" } as const) : racedOutcome;
407
557
 
408
558
  if (outcome.kind === "control") {
409
- scope.cancel();
559
+ if (!receiptGatedCancellation) {
560
+ // Replay-only v1 command order. New histories always take the v2 path
561
+ // below. The current runAgentTurn still owns and writes the truthful
562
+ // hard-fence receipt; this call remains an idempotent history command.
563
+ scope.cancel();
564
+ const settlement = await activity.settleSessionInterruptions({
565
+ accountId,
566
+ workspaceId,
567
+ sessionId,
568
+ attemptId,
569
+ workflowId: workflowInfo().workflowId,
570
+ });
571
+ const termination = await turn.then(
572
+ () => ({ kind: "completed" as const }),
573
+ (error: unknown) => ({ kind: "failed" as const, error }),
574
+ );
575
+ const physicalStopConfirmed =
576
+ termination.kind === "completed" || isTurnActivityFenceCancellation(termination.error);
577
+ if (patched("session-attempt-quiescence-v1") && physicalStopConfirmed) {
578
+ await activity.settleSessionInterruptions({
579
+ accountId,
580
+ workspaceId,
581
+ sessionId,
582
+ attemptId,
583
+ workflowId: workflowInfo().workflowId,
584
+ phase: "attempt_quiesced",
585
+ });
586
+ }
587
+ if (!physicalStopConfirmed) throw termination.error;
588
+ return settlement.action !== "paused";
589
+ }
410
590
  const settlement = await activity.settleSessionInterruptions({
411
591
  accountId,
412
592
  workspaceId,
@@ -414,16 +594,12 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
414
594
  attemptId,
415
595
  workflowId: workflowInfo().workflowId,
416
596
  });
417
- // Keep the workflow alive until the cancelled activity has actually
418
- // stopped. Temporal delivers activity cancellation through a heartbeat;
419
- // returning while the activity promise is still detached lets the
420
- // workflow complete first, after which the worker can keep streaming for
421
- // the activity's entire start-to-close window. Postgres settlement above
422
- // is intentionally first: it fences every late write immediately while
423
- // the physical activity winds down. The control outcome is authoritative,
424
- // so a concurrent activity completion or cancellation failure is ignored
425
- // only after Temporal has durably observed that terminal activity state.
426
- await turn.catch(() => undefined);
597
+ // The transaction above is the authority fence: every late model/tool/UI
598
+ // write is rejected from this point onward. Request cancellation only
599
+ // after it commits, then stop observing the Temporal activity promise.
600
+ // TRY_CANCEL allows this workflow to move to cancellation-wait without
601
+ // confusing Temporal completion/cancellation with local process safety.
602
+ scope.cancel();
427
603
  return settlement.action !== "paused";
428
604
  }
429
605
 
@@ -490,7 +666,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
490
666
  }
491
667
 
492
668
  if (outcome.result.deferredUntilWake) {
493
- return wakeups !== capacityWaitEntryBaseline.wakeups;
669
+ return deferredResultMayContinue(capacityWaitEntryBaseline.wakeups, wakeups);
494
670
  }
495
671
 
496
672
  if (outcome.result.status === "requires_action") return true;
@@ -512,3 +688,23 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
512
688
  return true;
513
689
  }
514
690
  }
691
+
692
+ function isSessionAttemptQuiescenceProof(
693
+ value: unknown,
694
+ ): value is activities.SessionAttemptQuiescenceProof {
695
+ if (!value || typeof value !== "object") return false;
696
+ const proof = value as Record<string, unknown>;
697
+ return [
698
+ "accountId",
699
+ "workspaceId",
700
+ "sessionId",
701
+ "attemptId",
702
+ "workflowId",
703
+ "workflowRunId",
704
+ "activityId",
705
+ ].every((field) => typeof proof[field] === "string" && proof[field].length > 0);
706
+ }
707
+
708
+ function sessionAttemptQuiescenceProofKey(proof: activities.SessionAttemptQuiescenceProof): string {
709
+ return `${proof.attemptId}:${proof.workflowRunId}:${proof.activityId}`;
710
+ }
package/src/workflows.ts CHANGED
@@ -3,6 +3,7 @@ export {
3
3
  approvalDecision,
4
4
  codexCapacityChanged,
5
5
  sessionControl,
6
+ sessionAttemptQuiesced,
6
7
  queueChanged,
7
8
  sessionWorkflow,
8
9
  userMessage,