@opengeni/worker-bundle 0.7.5 → 0.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BJUW5LCM.js +325 -0
- package/dist/chunk-BJUW5LCM.js.map +1 -0
- package/dist/host-export.d.ts +70 -0
- package/dist/host-export.js +7 -0
- package/dist/host-export.js.map +1 -0
- package/dist/index.d.ts +98 -6
- package/dist/index.js +4097 -1271
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +12360 -0
- package/package.json +18 -12
- package/src/activities/agent-turn.ts +2674 -828
- package/src/activities/capabilities.ts +11 -2
- package/src/activities/codex-capacity.ts +6 -11
- package/src/activities/codex-rotation.ts +75 -86
- package/src/activities/context-compaction.ts +104 -24
- package/src/activities/environment.ts +503 -97
- package/src/activities/expiring-material-renewal.ts +143 -0
- package/src/activities/goals.ts +4 -0
- package/src/activities/mcp-credentials.ts +38 -0
- package/src/activities/packs.ts +115 -2
- package/src/activities/rig-verification.ts +897 -244
- package/src/activities/run-credential-renewal.ts +70 -0
- package/src/activities/run-credentials.ts +132 -0
- package/src/activities/run-input.ts +23 -0
- package/src/activities/sandbox-lease.ts +33 -2
- package/src/activities/scheduled-tasks.ts +21 -0
- package/src/activities/session-state.ts +72 -0
- package/src/activities/streaming.ts +1 -0
- package/src/activities/toolspace-token-renewal.ts +57 -0
- package/src/activities/types.ts +44 -6
- package/src/activities/workspace-capture.ts +113 -93
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +416 -0
- package/src/host-export.ts +22 -0
- package/src/http.ts +77 -28
- package/src/index.ts +360 -156
- package/src/observability-metrics.ts +83 -15
- package/src/sandbox-resume.ts +47 -1
- package/src/sandbox-routing.ts +1 -1
- package/src/worker-service-lifecycle.ts +104 -0
- package/src/workflows/activities.ts +13 -7
- package/src/workflows/rig-verification.ts +3 -1
- package/src/workflows/session.ts +219 -23
- package/src/workflows.ts +1 -0
|
@@ -5,6 +5,7 @@ import type { EventLogger } from "@opengeni/events";
|
|
|
5
5
|
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
6
6
|
import {
|
|
7
7
|
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
|
|
8
|
+
modelUsageTokenCountOrNull,
|
|
8
9
|
type RuntimeMetricsHooks,
|
|
9
10
|
type SelfhostedOpObservation,
|
|
10
11
|
type SelfhostedOpObserver,
|
|
@@ -17,6 +18,13 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
|
17
18
|
|
|
18
19
|
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
19
20
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
21
|
+
const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
|
|
22
|
+
|
|
23
|
+
// A worker process reaching one trillion tokens in one provider/cache counter is
|
|
24
|
+
// already far outside an ordinary scrape lifetime. Refuse further increments
|
|
25
|
+
// before floating-point precision can degrade; a restart resets both the
|
|
26
|
+
// process-local Prometheus registry and this guard together.
|
|
27
|
+
const MAX_MODEL_CACHE_COUNTER_TOTAL = 1_000_000_000_000;
|
|
20
28
|
|
|
21
29
|
export function observabilityEventLogger(observability: Observability): EventLogger {
|
|
22
30
|
return {
|
|
@@ -553,7 +561,8 @@ export function recordModelInputTokens(
|
|
|
553
561
|
provider: string,
|
|
554
562
|
inputTokens: number,
|
|
555
563
|
): void {
|
|
556
|
-
|
|
564
|
+
const normalizedInputTokens = modelUsageTokenCountOrNull(inputTokens);
|
|
565
|
+
if (normalizedInputTokens === null || normalizedInputTokens === 0) {
|
|
557
566
|
return;
|
|
558
567
|
}
|
|
559
568
|
observability.observeHistogram({
|
|
@@ -561,7 +570,7 @@ export function recordModelInputTokens(
|
|
|
561
570
|
help: "Observed input (context) tokens per model response, by provider.",
|
|
562
571
|
buckets: MODEL_INPUT_TOKENS_BUCKETS,
|
|
563
572
|
labels: { provider },
|
|
564
|
-
value:
|
|
573
|
+
value: normalizedInputTokens,
|
|
565
574
|
});
|
|
566
575
|
}
|
|
567
576
|
|
|
@@ -598,29 +607,56 @@ const MODEL_CACHE_HIT_RATIO_BUCKETS = [
|
|
|
598
607
|
* - `opengeni_model_cached_tokens_total{provider}` — cumulative prompt tokens the
|
|
599
608
|
* provider served from cache. Advanced only when >0, so a provider that never
|
|
600
609
|
* reports cached tokens contributes nothing rather than phantom zero-increments.
|
|
610
|
+
* - `opengeni_model_cache_write_tokens_total{provider}` — cumulative tokens a
|
|
611
|
+
* reporting provider wrote into its prompt cache. Absent telemetry contributes
|
|
612
|
+
* nothing; in particular, Codex subscription usage does not currently expose it.
|
|
601
613
|
* - `opengeni_model_cache_hit_ratio{provider}` — cached/prompt for the call,
|
|
602
|
-
* observed
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
614
|
+
* observed only when prompt tokens are known (>0) AND cached tokens are reported.
|
|
615
|
+
* A reported cached-token zero records a real 0%; absent/null telemetry remains
|
|
616
|
+
* unknown and must not fabricate a low-cache signal.
|
|
617
|
+
* - `opengeni_model_cache_read_telemetry_total{provider,status}` — one bounded
|
|
618
|
+
* availability observation per authoritative call (`reported` or `missing`).
|
|
619
|
+
* Invalid, fractional, unsafe, or over-contract token telemetry is rejected. A call
|
|
620
|
+
* with no prompt tokens has no ratio (skipped).
|
|
607
621
|
*/
|
|
608
622
|
export function recordModelCacheTokens(
|
|
609
623
|
observability: Observability,
|
|
610
624
|
provider: string,
|
|
611
|
-
input: {
|
|
625
|
+
input: {
|
|
626
|
+
cachedTokens: number | null | undefined;
|
|
627
|
+
cacheWriteTokens?: number | null | undefined;
|
|
628
|
+
promptTokens: number | null | undefined;
|
|
629
|
+
},
|
|
612
630
|
): void {
|
|
613
|
-
const cached =
|
|
614
|
-
const
|
|
615
|
-
|
|
616
|
-
|
|
631
|
+
const cached = modelUsageTokenCountOrNull(input.cachedTokens);
|
|
632
|
+
const cacheWrite = modelUsageTokenCountOrNull(input.cacheWriteTokens);
|
|
633
|
+
const prompt = modelUsageTokenCountOrNull(input.promptTokens);
|
|
634
|
+
incrementBoundedModelCacheCounter(observability, {
|
|
635
|
+
name: "opengeni_model_cache_read_telemetry_total",
|
|
636
|
+
help: "Authoritative model calls with reported or missing cache-read telemetry, by provider.",
|
|
637
|
+
provider,
|
|
638
|
+
labels: { provider, status: cached === null ? "missing" : "reported" },
|
|
639
|
+
amount: 1,
|
|
640
|
+
});
|
|
641
|
+
if (cached !== null && cached > 0) {
|
|
642
|
+
incrementBoundedModelCacheCounter(observability, {
|
|
617
643
|
name: "opengeni_model_cached_tokens_total",
|
|
618
644
|
help: "Total prompt tokens served from the provider's prompt cache, by provider.",
|
|
645
|
+
provider,
|
|
619
646
|
labels: { provider },
|
|
620
647
|
amount: cached,
|
|
621
648
|
});
|
|
622
649
|
}
|
|
623
|
-
if (
|
|
650
|
+
if (cacheWrite !== null && cacheWrite > 0) {
|
|
651
|
+
incrementBoundedModelCacheCounter(observability, {
|
|
652
|
+
name: "opengeni_model_cache_write_tokens_total",
|
|
653
|
+
help: "Total prompt tokens written to the provider's prompt cache, by provider.",
|
|
654
|
+
provider,
|
|
655
|
+
labels: { provider },
|
|
656
|
+
amount: cacheWrite,
|
|
657
|
+
});
|
|
658
|
+
}
|
|
659
|
+
if (cached !== null && prompt !== null && prompt > 0) {
|
|
624
660
|
observability.observeHistogram({
|
|
625
661
|
name: "opengeni_model_cache_hit_ratio",
|
|
626
662
|
help: "Per-call prompt-cache hit ratio (cached/prompt tokens) by provider.",
|
|
@@ -633,8 +669,40 @@ export function recordModelCacheTokens(
|
|
|
633
669
|
}
|
|
634
670
|
}
|
|
635
671
|
|
|
636
|
-
function
|
|
637
|
-
|
|
672
|
+
function incrementBoundedModelCacheCounter(
|
|
673
|
+
observability: Observability,
|
|
674
|
+
input: {
|
|
675
|
+
name: string;
|
|
676
|
+
help: string;
|
|
677
|
+
provider: string;
|
|
678
|
+
labels: Record<string, string>;
|
|
679
|
+
amount: number;
|
|
680
|
+
},
|
|
681
|
+
): void {
|
|
682
|
+
const totals = modelCacheCounterTotals.get(observability) ?? new Map<string, number>();
|
|
683
|
+
if (!modelCacheCounterTotals.has(observability)) {
|
|
684
|
+
modelCacheCounterTotals.set(observability, totals);
|
|
685
|
+
}
|
|
686
|
+
const labelKey = Object.entries(input.labels)
|
|
687
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
688
|
+
.map(([key, value]) => `${key}=${value}`)
|
|
689
|
+
.join(",");
|
|
690
|
+
const key = `${input.name}:${labelKey}`;
|
|
691
|
+
const current = totals.get(key) ?? 0;
|
|
692
|
+
if (input.amount > MAX_MODEL_CACHE_COUNTER_TOTAL - current) {
|
|
693
|
+
observability.warn("model cache metric cumulative limit reached", {
|
|
694
|
+
provider: input.provider,
|
|
695
|
+
metric: input.name,
|
|
696
|
+
});
|
|
697
|
+
return;
|
|
698
|
+
}
|
|
699
|
+
observability.incrementCounter({
|
|
700
|
+
name: input.name,
|
|
701
|
+
help: input.help,
|
|
702
|
+
labels: input.labels,
|
|
703
|
+
amount: input.amount,
|
|
704
|
+
});
|
|
705
|
+
totals.set(key, current + input.amount);
|
|
638
706
|
}
|
|
639
707
|
|
|
640
708
|
/**
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -231,11 +231,24 @@ class SnapshotTimeoutError extends Error {
|
|
|
231
231
|
export async function waitForWarmSnapshot(
|
|
232
232
|
snapshot: Promise<unknown>,
|
|
233
233
|
timeoutMs: number,
|
|
234
|
+
signal?: AbortSignal,
|
|
234
235
|
): Promise<boolean> {
|
|
236
|
+
if (signal?.aborted) return false;
|
|
235
237
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
238
|
+
let cancelListener: (() => void) | undefined;
|
|
236
239
|
try {
|
|
237
240
|
await Promise.race([
|
|
238
241
|
snapshot,
|
|
242
|
+
...(signal
|
|
243
|
+
? [
|
|
244
|
+
new Promise<never>((_resolve, reject) => {
|
|
245
|
+
cancelListener = () =>
|
|
246
|
+
reject(signal.reason ?? new Error("workspace snapshot wait cancelled"));
|
|
247
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
248
|
+
if (signal.aborted) cancelListener();
|
|
249
|
+
}),
|
|
250
|
+
]
|
|
251
|
+
: []),
|
|
239
252
|
new Promise<never>((_, reject) => {
|
|
240
253
|
timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
|
|
241
254
|
if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
|
|
@@ -245,6 +258,7 @@ export async function waitForWarmSnapshot(
|
|
|
245
258
|
]);
|
|
246
259
|
return true;
|
|
247
260
|
} catch (error) {
|
|
261
|
+
if (signal?.aborted) return false;
|
|
248
262
|
if (error instanceof SnapshotTimeoutError) {
|
|
249
263
|
console.error("mid-session workspace snapshot wait timed out (turn unaffected)", error);
|
|
250
264
|
return false;
|
|
@@ -254,6 +268,7 @@ export async function waitForWarmSnapshot(
|
|
|
254
268
|
if (timeout) {
|
|
255
269
|
clearTimeout(timeout);
|
|
256
270
|
}
|
|
271
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
257
272
|
}
|
|
258
273
|
}
|
|
259
274
|
|
|
@@ -389,9 +404,17 @@ function preserveWorkspaceArchivesOnResumeState(
|
|
|
389
404
|
*/
|
|
390
405
|
export async function maybePersistWarmWorkspaceSnapshot(
|
|
391
406
|
services: SandboxResumeServices,
|
|
392
|
-
ids: {
|
|
407
|
+
ids: {
|
|
408
|
+
accountId: string;
|
|
409
|
+
workspaceId: string;
|
|
410
|
+
sessionId: string;
|
|
411
|
+
turnId: string;
|
|
412
|
+
attemptId: string;
|
|
413
|
+
sandboxGroupId: string;
|
|
414
|
+
},
|
|
393
415
|
session: unknown,
|
|
394
416
|
leaseEpoch: number,
|
|
417
|
+
signal?: AbortSignal,
|
|
395
418
|
): Promise<boolean> {
|
|
396
419
|
const { db, settings } = services;
|
|
397
420
|
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
@@ -404,6 +427,9 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
404
427
|
if (typeof persistable.persistWorkspace !== "function") {
|
|
405
428
|
return false;
|
|
406
429
|
}
|
|
430
|
+
if (signal?.aborted) {
|
|
431
|
+
return false;
|
|
432
|
+
}
|
|
407
433
|
try {
|
|
408
434
|
// Cheap throttle pre-check before the (potentially slow) capture;
|
|
409
435
|
// persistWarmSnapshot re-checks atomically under the row lock, so this is
|
|
@@ -412,6 +438,9 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
412
438
|
if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm") {
|
|
413
439
|
return false;
|
|
414
440
|
}
|
|
441
|
+
if (signal?.aborted) {
|
|
442
|
+
return false;
|
|
443
|
+
}
|
|
415
444
|
const sessionState =
|
|
416
445
|
lease.resumeState && typeof lease.resumeState === "object"
|
|
417
446
|
? (lease.resumeState as { sessionState?: Record<string, unknown> }).sessionState
|
|
@@ -437,8 +466,18 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
437
466
|
// guard this timeout exists to provide.
|
|
438
467
|
const capture = persistable.persistWorkspace();
|
|
439
468
|
capture.catch(() => undefined);
|
|
469
|
+
let cancelListener: (() => void) | undefined;
|
|
440
470
|
const bytes = await Promise.race([
|
|
441
471
|
capture,
|
|
472
|
+
...(signal
|
|
473
|
+
? [
|
|
474
|
+
new Promise<undefined>((resolve) => {
|
|
475
|
+
cancelListener = () => resolve(undefined);
|
|
476
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
477
|
+
if (signal.aborted) cancelListener();
|
|
478
|
+
}),
|
|
479
|
+
]
|
|
480
|
+
: []),
|
|
442
481
|
new Promise<undefined>((resolve) => {
|
|
443
482
|
timeout = setTimeout(() => resolve(undefined), settings.sandboxSnapshotTimeoutMs);
|
|
444
483
|
if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
|
|
@@ -449,13 +488,20 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
449
488
|
if (timeout) {
|
|
450
489
|
clearTimeout(timeout);
|
|
451
490
|
}
|
|
491
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
452
492
|
});
|
|
453
493
|
if (!bytes || bytes.length === 0) {
|
|
454
494
|
return false;
|
|
455
495
|
}
|
|
496
|
+
if (signal?.aborted) {
|
|
497
|
+
return false;
|
|
498
|
+
}
|
|
456
499
|
const { wrote, priorArchiveForGc } = await persistWarmSnapshot(db, {
|
|
457
500
|
accountId: ids.accountId,
|
|
458
501
|
workspaceId: ids.workspaceId,
|
|
502
|
+
sessionId: ids.sessionId,
|
|
503
|
+
turnId: ids.turnId,
|
|
504
|
+
attemptId: ids.attemptId,
|
|
459
505
|
sandboxGroupId: ids.sandboxGroupId,
|
|
460
506
|
expectedEpoch: leaseEpoch,
|
|
461
507
|
workspaceArchive: Buffer.from(bytes).toString("base64"),
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
//
|
|
5
5
|
// The turn resumes its group box by id (resumeBoxForTurn) and injects it
|
|
6
6
|
// NON-OWNED into the run. With hot-swap, the injected `session` must be the
|
|
7
|
-
// STABLE routing proxy
|
|
7
|
+
// STABLE routing proxy: the SDK binds to it ONCE and calls its
|
|
8
8
|
// methods per tool call; the proxy re-reads `(active_sandbox_id, active_epoch)`
|
|
9
9
|
// per op and dispatches to the currently-active backend (the group Modal box by
|
|
10
10
|
// default, or a swap target — a sibling Modal box or a selfhosted machine).
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import type { Observability } from "@opengeni/observability";
|
|
2
|
+
import type { WorkerLifecycleState } from "./http";
|
|
3
|
+
|
|
4
|
+
export async function constructWithOwnedConnection<Connection, Result>(
|
|
5
|
+
connect: () => Promise<Connection>,
|
|
6
|
+
construct: (connection: Connection) => Promise<Result>,
|
|
7
|
+
close: (connection: Connection) => Promise<void>,
|
|
8
|
+
): Promise<Result> {
|
|
9
|
+
const connection = await connect();
|
|
10
|
+
try {
|
|
11
|
+
return await construct(connection);
|
|
12
|
+
} catch (error) {
|
|
13
|
+
await close(connection).catch(() => undefined);
|
|
14
|
+
throw error;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export type WorkerRunTarget = {
|
|
19
|
+
run(): Promise<void>;
|
|
20
|
+
shutdown(): void;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export type WorkerServiceLifecycle = {
|
|
24
|
+
state(): WorkerLifecycleState;
|
|
25
|
+
run(): Promise<void>;
|
|
26
|
+
drain(reason?: string): void;
|
|
27
|
+
close(): Promise<void>;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export function createWorkerServiceLifecycle(input: {
|
|
31
|
+
role: "control" | "turn";
|
|
32
|
+
worker: WorkerRunTarget;
|
|
33
|
+
observability: Observability;
|
|
34
|
+
closeOwnedResources: () => Promise<void>;
|
|
35
|
+
onReady?: () => void;
|
|
36
|
+
}): WorkerServiceLifecycle {
|
|
37
|
+
let state: WorkerLifecycleState = "starting";
|
|
38
|
+
let runPromise: Promise<void> | undefined;
|
|
39
|
+
let resourcesClosed: Promise<void> | undefined;
|
|
40
|
+
const closeOwnedResources = () => {
|
|
41
|
+
resourcesClosed ??= input.closeOwnedResources();
|
|
42
|
+
return resourcesClosed;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
const lifecycle: WorkerServiceLifecycle = {
|
|
46
|
+
state: () => state,
|
|
47
|
+
run: () => {
|
|
48
|
+
if (!runPromise && state === "draining") {
|
|
49
|
+
runPromise = (async () => {
|
|
50
|
+
state = "stopped";
|
|
51
|
+
await closeOwnedResources();
|
|
52
|
+
})();
|
|
53
|
+
return runPromise;
|
|
54
|
+
}
|
|
55
|
+
if (!runPromise && (state === "stopped" || state === "failed")) {
|
|
56
|
+
return Promise.reject(new Error(`cannot run a worker service that is ${state}`));
|
|
57
|
+
}
|
|
58
|
+
runPromise ??= (async () => {
|
|
59
|
+
if (state === "starting") {
|
|
60
|
+
state = "ready";
|
|
61
|
+
}
|
|
62
|
+
input.onReady?.();
|
|
63
|
+
try {
|
|
64
|
+
await input.worker.run();
|
|
65
|
+
state = "stopped";
|
|
66
|
+
} catch (error) {
|
|
67
|
+
state = "failed";
|
|
68
|
+
throw error;
|
|
69
|
+
} finally {
|
|
70
|
+
await closeOwnedResources();
|
|
71
|
+
}
|
|
72
|
+
})();
|
|
73
|
+
return runPromise;
|
|
74
|
+
},
|
|
75
|
+
drain: (reason = "host request") => {
|
|
76
|
+
if (state === "draining" || state === "stopped" || state === "failed") {
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
state = "draining";
|
|
80
|
+
input.observability.info("OpenGeni worker draining (graceful shutdown)", {
|
|
81
|
+
role: input.role,
|
|
82
|
+
reason,
|
|
83
|
+
});
|
|
84
|
+
try {
|
|
85
|
+
input.worker.shutdown();
|
|
86
|
+
} catch (error) {
|
|
87
|
+
input.observability.warn("worker shutdown request failed", {
|
|
88
|
+
error: error instanceof Error ? error.message : String(error),
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
close: async () => {
|
|
93
|
+
lifecycle.drain("service close");
|
|
94
|
+
if (runPromise) {
|
|
95
|
+
await runPromise.catch(() => undefined);
|
|
96
|
+
} else {
|
|
97
|
+
state = "stopped";
|
|
98
|
+
await closeOwnedResources();
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
return lifecycle;
|
|
104
|
+
}
|
|
@@ -5,10 +5,12 @@ type WorkflowControlActivities = Pick<
|
|
|
5
5
|
typeof activities,
|
|
6
6
|
| "dispatchScheduledTaskRun"
|
|
7
7
|
| "enqueueGoalRetryWake"
|
|
8
|
+
| "expireSessionHumanInput"
|
|
8
9
|
| "failSessionAttempt"
|
|
9
10
|
| "getCodexCapacityWait"
|
|
10
11
|
| "markSessionIdle"
|
|
11
12
|
| "peekSessionWork"
|
|
13
|
+
| "persistSessionAttemptQuiescence"
|
|
12
14
|
| "reconcileCodexCapacityWait"
|
|
13
15
|
| "recoverDispatch"
|
|
14
16
|
| "settleSessionInterruptions"
|
|
@@ -49,7 +51,7 @@ export function turnTaskQueue(baseTaskQueue: string): string {
|
|
|
49
51
|
return `${baseTaskQueue}-turns`;
|
|
50
52
|
}
|
|
51
53
|
|
|
52
|
-
export function turnActivityForTaskQueue(baseTaskQueue: string) {
|
|
54
|
+
export function turnActivityForTaskQueue(baseTaskQueue: string, receiptGatedCancellation = true) {
|
|
53
55
|
return proxyActivities<Pick<typeof activities, "runAgentTurn">>({
|
|
54
56
|
taskQueue: turnTaskQueue(baseTaskQueue),
|
|
55
57
|
// Agent segments legitimately run for days. A started turn heartbeats;
|
|
@@ -57,12 +59,16 @@ export function turnActivityForTaskQueue(baseTaskQueue: string) {
|
|
|
57
59
|
// accepts them and performs the atomic claim.
|
|
58
60
|
startToCloseTimeout: "30 days",
|
|
59
61
|
heartbeatTimeout: "2 minutes",
|
|
60
|
-
// Pause/Steer
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
//
|
|
64
|
-
//
|
|
65
|
-
|
|
62
|
+
// Pause/Steer first closes the exact attempt in Postgres, then asks
|
|
63
|
+
// Temporal to deliver cancellation. The workflow must not wait for the
|
|
64
|
+
// Temporal activity promise: a provider cleanup promise can outlive the
|
|
65
|
+
// fenced activity body, and Temporal terminalization is not proof that
|
|
66
|
+
// sandbox tools or processes are physically quiescent. The activity owns
|
|
67
|
+
// that proof and writes an exact receipt after its hard tool fence; the
|
|
68
|
+
// receipt transaction wakes the workflow to admit a replacement.
|
|
69
|
+
cancellationType: receiptGatedCancellation
|
|
70
|
+
? ActivityCancellationType.TRY_CANCEL
|
|
71
|
+
: ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
|
|
66
72
|
retry: { maximumAttempts: 1 },
|
|
67
73
|
});
|
|
68
74
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { proxyActivities } from "@temporalio/workflow";
|
|
1
|
+
import { ActivityCancellationType, proxyActivities } from "@temporalio/workflow";
|
|
2
2
|
import type * as activities from "../activities";
|
|
3
3
|
import type { RigVerificationWorkflowInput } from "../activities/rig-verification";
|
|
4
4
|
|
|
@@ -6,6 +6,8 @@ const rigVerificationActivities = proxyActivities<
|
|
|
6
6
|
Pick<typeof activities, "verifyRigChange" | "verifyRigVersion">
|
|
7
7
|
>({
|
|
8
8
|
startToCloseTimeout: "15 minutes",
|
|
9
|
+
heartbeatTimeout: "10 seconds",
|
|
10
|
+
cancellationType: ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
|
|
9
11
|
retry: { maximumAttempts: 1 },
|
|
10
12
|
});
|
|
11
13
|
|