@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +10 -2
- package/dist/activities/agent-turn/agent-build.d.ts +1 -4
- package/dist/activities/agent-turn/claim.d.ts +3 -4
- package/dist/activities/agent-turn/codex.d.ts +8 -7
- package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
- package/dist/activities/agent-turn/errors.d.ts +13 -0
- package/dist/activities/agent-turn/finalization.d.ts +1 -3
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +10 -1
- package/dist/activities/agent-turn/quiescence.d.ts +39 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
- package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
- package/dist/activities/agent-turn/turn-context.d.ts +3 -8
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
- package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +2 -1
- package/dist/activities/codex-rotation.d.ts +2 -0
- package/dist/activities/goals.d.ts +11 -0
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/session-state.d.ts +2 -1
- package/dist/activities/types.d.ts +8 -0
- package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
- package/dist/activities-control-SGU2WEVB.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
- package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
- package/dist/chunk-S2QVRVJW.js.map +1 -0
- package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
- package/dist/chunk-TJIPCHMP.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +69 -6
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +29 -0
- package/dist/personal-github-git-credentials.d.ts +1 -0
- package/dist/unhandled-rejection-boundary.d.ts +29 -0
- package/dist/workflow-bundle.js +26 -3
- package/dist/workflows/activities.d.ts +5 -0
- package/dist/workflows/automations.d.ts +5 -0
- package/package.json +18 -18
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +28 -6
- package/src/activities/agent-turn/agent-build.ts +15 -31
- package/src/activities/agent-turn/claim.ts +53 -72
- package/src/activities/agent-turn/codex.ts +21 -12
- package/src/activities/agent-turn/compaction-prep.ts +52 -1
- package/src/activities/agent-turn/errors.ts +78 -0
- package/src/activities/agent-turn/failure-settlement.ts +30 -19
- package/src/activities/agent-turn/finalization.ts +24 -21
- package/src/activities/agent-turn/history.ts +5 -1
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +88 -42
- package/src/activities/agent-turn/quiescence.ts +86 -0
- package/src/activities/agent-turn/run-credentials.ts +12 -1
- package/src/activities/agent-turn/run.ts +20 -14
- package/src/activities/agent-turn/sandbox-establish.ts +46 -11
- package/src/activities/agent-turn/sandbox-route.ts +21 -25
- package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
- package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
- package/src/activities/agent-turn/stream-attempt.ts +106 -14
- package/src/activities/agent-turn/tool-environment.ts +72 -14
- package/src/activities/agent-turn/tool-policy.ts +7 -6
- package/src/activities/agent-turn/turn-context.ts +12 -14
- package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
- package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +72 -2
- package/src/activities/codex-capacity.ts +13 -5
- package/src/activities/codex-rotation.ts +17 -0
- package/src/activities/goals.ts +117 -35
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/retained-screenshots.ts +104 -21
- package/src/activities/sandbox-lease.ts +7 -4
- package/src/activities/scheduled-tasks.ts +248 -30
- package/src/activities/session-state.ts +8 -1
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +6 -0
- package/src/activities.ts +1 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +8 -4
- package/src/http.ts +3 -0
- package/src/index.ts +38 -4
- package/src/observability-metrics.ts +202 -0
- package/src/personal-github-git-credentials.ts +45 -4
- package/src/sandbox-resume.ts +1 -1
- package/src/sandbox-routing.ts +7 -4
- package/src/unhandled-rejection-boundary.ts +73 -0
- package/src/workflows/activities.ts +15 -0
- package/src/workflows/automations.ts +21 -2
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
- package/dist/activities/recording.d.ts +0 -90
- package/dist/activities-control-VZIIV7RX.js.map +0 -1
- package/dist/activities-turn-5RGQDBFV.js.map +0 -1
- package/dist/chunk-NAVSAZ56.js.map +0 -1
- package/dist/chunk-O4HDX5SC.js.map +0 -1
- package/src/activities/recording.ts +0 -279
package/src/index.ts
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
} from "@opengeni/config";
|
|
10
10
|
import {
|
|
11
11
|
assertRuntimeDatabasePosture,
|
|
12
|
+
countSessionRecoveryBacklog,
|
|
12
13
|
createDb,
|
|
13
14
|
markSessionWorkflowWakeDelivered,
|
|
14
15
|
type Database,
|
|
@@ -55,12 +56,15 @@ import {
|
|
|
55
56
|
type WorkerLifecycleState,
|
|
56
57
|
} from "./http";
|
|
57
58
|
import {
|
|
59
|
+
initializeWorkerOutcomeMetrics,
|
|
58
60
|
normalizeTurnTaskQueueStats,
|
|
59
61
|
observabilityEventLogger,
|
|
62
|
+
startSessionRecoveryMonitor,
|
|
60
63
|
startTurnCapacityMonitor,
|
|
61
64
|
type TurnTaskQueueStats,
|
|
62
65
|
} from "./observability-metrics";
|
|
63
66
|
import {
|
|
67
|
+
resolveCatalogSettings,
|
|
64
68
|
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
65
69
|
SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
66
70
|
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
@@ -86,6 +90,7 @@ import {
|
|
|
86
90
|
type TurnWorkerMemoryPressureGuard,
|
|
87
91
|
} from "./memory-pressure-guard";
|
|
88
92
|
import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
|
|
93
|
+
import { installWorkerUnhandledRejectionBoundary } from "./unhandled-rejection-boundary";
|
|
89
94
|
import {
|
|
90
95
|
SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
91
96
|
sandboxLifecycleTaskQueue,
|
|
@@ -219,6 +224,7 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
219
224
|
const observability =
|
|
220
225
|
options.activityDependencies?.observability ??
|
|
221
226
|
createObservability(settings, { component: `worker-${options.role}` });
|
|
227
|
+
initializeWorkerOutcomeMetrics(observability);
|
|
222
228
|
if (options.role === "turn" && options.workflowBundle) {
|
|
223
229
|
throw new Error("workflowBundle is valid only for the control worker role");
|
|
224
230
|
}
|
|
@@ -793,11 +799,22 @@ export async function createOpenGeniWorkerService(
|
|
|
793
799
|
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
794
800
|
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
795
801
|
let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
|
|
802
|
+
let sessionRecoveryMonitor: ReturnType<typeof startSessionRecoveryMonitor> | undefined;
|
|
796
803
|
const schedules: Array<{ close: () => Promise<void> }> = [];
|
|
797
804
|
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
798
805
|
let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
|
|
799
806
|
|
|
800
807
|
try {
|
|
808
|
+
const resolvedCatalog = await retryStartupDependency(
|
|
809
|
+
"model catalog",
|
|
810
|
+
() => resolveCatalogSettings(options.activityDependencies.db, settings),
|
|
811
|
+
{ ...retryOptions, onRetry },
|
|
812
|
+
);
|
|
813
|
+
observability.info("OpenGeni model catalog resolved", {
|
|
814
|
+
role: options.role,
|
|
815
|
+
catalogSource: resolvedCatalog.source,
|
|
816
|
+
catalogVersion: resolvedCatalog.version,
|
|
817
|
+
});
|
|
801
818
|
const needsSignaler =
|
|
802
819
|
options.role === "turn" ||
|
|
803
820
|
!options.activityDependencies.wakeSessionWorkflow ||
|
|
@@ -866,6 +883,11 @@ export async function createOpenGeniWorkerService(
|
|
|
866
883
|
observability,
|
|
867
884
|
read: signaler.getTurnTaskQueueStats,
|
|
868
885
|
});
|
|
886
|
+
} else {
|
|
887
|
+
sessionRecoveryMonitor = startSessionRecoveryMonitor({
|
|
888
|
+
observability,
|
|
889
|
+
read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db),
|
|
890
|
+
});
|
|
869
891
|
}
|
|
870
892
|
|
|
871
893
|
if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
|
|
@@ -900,14 +922,18 @@ export async function createOpenGeniWorkerService(
|
|
|
900
922
|
}
|
|
901
923
|
|
|
902
924
|
if (options.http !== false) {
|
|
925
|
+
const databaseReady = dbReadyCheck(
|
|
926
|
+
options.http?.readinessDb ?? options.activityDependencies.db,
|
|
927
|
+
options.databasePosture,
|
|
928
|
+
);
|
|
903
929
|
httpServer = startWorkerHttpServer({
|
|
904
930
|
settings,
|
|
905
931
|
observability,
|
|
906
932
|
checks: {
|
|
907
|
-
db:
|
|
908
|
-
|
|
909
|
-
options.
|
|
910
|
-
|
|
933
|
+
db: async () => {
|
|
934
|
+
await databaseReady();
|
|
935
|
+
await resolveCatalogSettings(options.activityDependencies.db, settings);
|
|
936
|
+
},
|
|
911
937
|
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
912
938
|
temporal: temporalReadyCheck(workerBundle.connection),
|
|
913
939
|
},
|
|
@@ -919,6 +945,7 @@ export async function createOpenGeniWorkerService(
|
|
|
919
945
|
httpServer?.stop(true);
|
|
920
946
|
await Promise.allSettled([
|
|
921
947
|
turnCapacityMonitor?.close(),
|
|
948
|
+
sessionRecoveryMonitor?.close(),
|
|
922
949
|
workerBundle?.connection.close(),
|
|
923
950
|
signaler?.close(),
|
|
924
951
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -941,6 +968,7 @@ export async function createOpenGeniWorkerService(
|
|
|
941
968
|
httpServer?.stop(true);
|
|
942
969
|
await Promise.allSettled([
|
|
943
970
|
turnCapacityMonitor?.close(),
|
|
971
|
+
sessionRecoveryMonitor?.close(),
|
|
944
972
|
activeWorkerBundle.connection.close(),
|
|
945
973
|
activeSignaler?.close(),
|
|
946
974
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -1032,6 +1060,11 @@ export async function startWorker() {
|
|
|
1032
1060
|
}
|
|
1033
1061
|
const settings = getSettings();
|
|
1034
1062
|
const observability = createObservability(settings, { component: `worker-${role}` });
|
|
1063
|
+
// The Agents SDK also listens for unhandled rejections and exits the process.
|
|
1064
|
+
// Keep detached SDK/transport promises observational here: durable turn
|
|
1065
|
+
// activity boundaries own failures, while process exit causes cross-session
|
|
1066
|
+
// lease loss for every turn assigned to this shared worker pod.
|
|
1067
|
+
const disposeUnhandledRejectionBoundary = installWorkerUnhandledRejectionBoundary();
|
|
1035
1068
|
const retryOptions = startupRetryOptions(settings);
|
|
1036
1069
|
const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
|
|
1037
1070
|
logStartupDependencyRetry(observability, event);
|
|
@@ -1090,6 +1123,7 @@ export async function startWorker() {
|
|
|
1090
1123
|
});
|
|
1091
1124
|
} finally {
|
|
1092
1125
|
await Promise.allSettled([bus?.close(), readinessDbClient.close(), dbClient.close()]);
|
|
1126
|
+
disposeUnhandledRejectionBoundary();
|
|
1093
1127
|
}
|
|
1094
1128
|
}
|
|
1095
1129
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
3
3
|
import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
|
|
4
|
+
import type { SessionEventAppendPhaseObservation } from "@opengeni/db";
|
|
4
5
|
import type { EventLogger } from "@opengeni/events";
|
|
5
6
|
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
6
7
|
import type { CompanyBrainContributionReceipt } from "./model-context-contributions";
|
|
@@ -18,6 +19,7 @@ import {
|
|
|
18
19
|
} from "@opengeni/runtime";
|
|
19
20
|
|
|
20
21
|
export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
|
|
22
|
+
export type WorkerDeathRecoveryOutcome = "recovering" | "exhausted";
|
|
21
23
|
export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
|
|
22
24
|
export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
|
|
23
25
|
export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
@@ -28,6 +30,10 @@ export type TurnTaskQueueStats = {
|
|
|
28
30
|
tasksAddRate: number;
|
|
29
31
|
tasksDispatchRate: number;
|
|
30
32
|
};
|
|
33
|
+
export type SessionRecoveryBacklog = {
|
|
34
|
+
quiescence_missing: number;
|
|
35
|
+
projection_stale: number;
|
|
36
|
+
};
|
|
31
37
|
|
|
32
38
|
export type TemporalTurnTaskQueueStats = {
|
|
33
39
|
approximateBacklogCount?: unknown;
|
|
@@ -49,6 +55,12 @@ const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>(
|
|
|
49
55
|
// before floating-point precision can degrade; a restart resets both the
|
|
50
56
|
// process-local Prometheus registry and this guard together.
|
|
51
57
|
const MAX_MODEL_CACHE_COUNTER_TOTAL = 1_000_000_000_000;
|
|
58
|
+
const TURN_OUTCOMES: readonly TurnOutcome[] = ["completed", "failed", "cancelled", "recovering"];
|
|
59
|
+
const WORKER_DEATH_RECOVERY_OUTCOMES: readonly WorkerDeathRecoveryOutcome[] = [
|
|
60
|
+
"recovering",
|
|
61
|
+
"exhausted",
|
|
62
|
+
];
|
|
63
|
+
const WORKER_DEATH_TIMEOUT_TYPES = ["heartbeat", "schedule_to_start"] as const;
|
|
52
64
|
|
|
53
65
|
export function observabilityEventLogger(observability: Observability): EventLogger {
|
|
54
66
|
return {
|
|
@@ -163,6 +175,19 @@ export function runtimeMetricsHooksForObservability(
|
|
|
163
175
|
value: durationSeconds,
|
|
164
176
|
});
|
|
165
177
|
},
|
|
178
|
+
onMcpLifecycle: ({ phase, policy, outcome, durationSeconds }) => {
|
|
179
|
+
observability.incrementCounter({
|
|
180
|
+
name: "opengeni_mcp_lifecycle_operations_total",
|
|
181
|
+
help: "Total physical MCP lifecycle operations by bounded phase, policy, and outcome.",
|
|
182
|
+
labels: { phase, policy, outcome },
|
|
183
|
+
});
|
|
184
|
+
observability.observeHistogram({
|
|
185
|
+
name: "opengeni_mcp_lifecycle_operation_duration_seconds",
|
|
186
|
+
help: "MCP lifecycle operation duration in seconds by bounded phase, policy, and outcome.",
|
|
187
|
+
labels: { phase, policy, outcome },
|
|
188
|
+
value: durationSeconds,
|
|
189
|
+
});
|
|
190
|
+
},
|
|
166
191
|
onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
|
|
167
192
|
observability.incrementCounter({
|
|
168
193
|
name: "opengeni_machine_op_total",
|
|
@@ -300,6 +325,53 @@ export function turnLifecycleMetricsFor(observability: Observability): TurnLifec
|
|
|
300
325
|
return tracker;
|
|
301
326
|
}
|
|
302
327
|
|
|
328
|
+
export function initializeWorkerOutcomeMetrics(observability: Observability): void {
|
|
329
|
+
for (const outcome of TURN_OUTCOMES) {
|
|
330
|
+
observability.incrementCounter({
|
|
331
|
+
name: "opengeni_turns_total",
|
|
332
|
+
help: "Total agent turns by terminal outcome.",
|
|
333
|
+
labels: { outcome },
|
|
334
|
+
amount: 0,
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
for (const outcome of WORKER_DEATH_RECOVERY_OUTCOMES) {
|
|
338
|
+
for (const timeoutType of WORKER_DEATH_TIMEOUT_TYPES) {
|
|
339
|
+
observability.incrementCounter({
|
|
340
|
+
name: "opengeni_turn_worker_death_recoveries_total",
|
|
341
|
+
help: "Total durable same-turn worker-death recovery outcomes.",
|
|
342
|
+
labels: { outcome, timeout_type: timeoutType },
|
|
343
|
+
amount: 0,
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
export function recordWorkerDeathRecoveryMetrics(
|
|
350
|
+
observability: Observability,
|
|
351
|
+
input: {
|
|
352
|
+
outcome: WorkerDeathRecoveryOutcome;
|
|
353
|
+
timeoutType: "HEARTBEAT" | "SCHEDULE_TO_START";
|
|
354
|
+
},
|
|
355
|
+
): void {
|
|
356
|
+
const turnOutcome: TurnOutcome = input.outcome === "exhausted" ? "failed" : "recovering";
|
|
357
|
+
observability.incrementCounter({
|
|
358
|
+
name: "opengeni_turn_worker_death_recoveries_total",
|
|
359
|
+
help: "Total durable same-turn worker-death recovery outcomes.",
|
|
360
|
+
labels: {
|
|
361
|
+
outcome: input.outcome,
|
|
362
|
+
timeout_type: input.timeoutType.toLowerCase(),
|
|
363
|
+
},
|
|
364
|
+
});
|
|
365
|
+
// The turn process that owned this attempt is gone, so its process-local
|
|
366
|
+
// lifecycle tracker cannot publish the outcome. Record it from the fenced
|
|
367
|
+
// control activity only after the durable recovery transaction wins.
|
|
368
|
+
observability.incrementCounter({
|
|
369
|
+
name: "opengeni_turns_total",
|
|
370
|
+
help: "Total agent turns by terminal outcome.",
|
|
371
|
+
labels: { outcome: turnOutcome },
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
|
|
303
375
|
export class TurnLifecycleMetrics {
|
|
304
376
|
private readonly turns = new Map<string, { startedAt: number; lastProgressAt: number }>();
|
|
305
377
|
private timer: ReturnType<typeof setInterval> | null = null;
|
|
@@ -676,6 +748,88 @@ export function startTurnCapacityMonitor(input: {
|
|
|
676
748
|
};
|
|
677
749
|
}
|
|
678
750
|
|
|
751
|
+
export function recordSessionRecoveryBacklogGauges(
|
|
752
|
+
observability: Observability,
|
|
753
|
+
counts: SessionRecoveryBacklog,
|
|
754
|
+
): void {
|
|
755
|
+
for (const state of ["quiescence_missing", "projection_stale"] as const) {
|
|
756
|
+
observability.setGauge({
|
|
757
|
+
name: "opengeni_session_recovery_backlog",
|
|
758
|
+
help: "Current durable recovering-session obligations with no active attempt, by bounded reconciliation state.",
|
|
759
|
+
labels: { state },
|
|
760
|
+
value: nonnegativeFinite(counts[state]),
|
|
761
|
+
});
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
export function startSessionRecoveryMonitor(input: {
|
|
766
|
+
observability: Observability;
|
|
767
|
+
read: () => Promise<SessionRecoveryBacklog>;
|
|
768
|
+
intervalMs?: number;
|
|
769
|
+
now?: () => number;
|
|
770
|
+
}): { close: () => Promise<void> } {
|
|
771
|
+
const intervalMs = input.intervalMs ?? 60_000;
|
|
772
|
+
const now = input.now ?? Date.now;
|
|
773
|
+
const startedAt = now();
|
|
774
|
+
let lastSuccessAt: number | null = null;
|
|
775
|
+
let lastReadSucceeded = false;
|
|
776
|
+
let stopped = false;
|
|
777
|
+
let running: Promise<void> | null = null;
|
|
778
|
+
const recordStatus = () => {
|
|
779
|
+
const observedAt = now();
|
|
780
|
+
const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
|
|
781
|
+
const set = (name: string, help: string, value: number) =>
|
|
782
|
+
input.observability.setGauge({ name, help, value });
|
|
783
|
+
set(
|
|
784
|
+
"opengeni_session_recovery_monitor_last_read_success",
|
|
785
|
+
"Whether the latest durable session-recovery aggregate read completed successfully.",
|
|
786
|
+
lastReadSucceeded ? 1 : 0,
|
|
787
|
+
);
|
|
788
|
+
set(
|
|
789
|
+
"opengeni_session_recovery_monitor_last_success_timestamp_seconds",
|
|
790
|
+
"Unix timestamp of the latest successful durable session-recovery aggregate read, or zero before one succeeds.",
|
|
791
|
+
lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
|
|
792
|
+
);
|
|
793
|
+
set(
|
|
794
|
+
"opengeni_session_recovery_monitor_fresh",
|
|
795
|
+
"Whether durable session-recovery backlog gauges have a successful read within three monitor intervals.",
|
|
796
|
+
lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
|
|
797
|
+
);
|
|
798
|
+
};
|
|
799
|
+
const refresh = () => {
|
|
800
|
+
recordStatus();
|
|
801
|
+
if (stopped || running) return;
|
|
802
|
+
running = input
|
|
803
|
+
.read()
|
|
804
|
+
.then((counts) => {
|
|
805
|
+
recordSessionRecoveryBacklogGauges(input.observability, counts);
|
|
806
|
+
lastSuccessAt = now();
|
|
807
|
+
lastReadSucceeded = true;
|
|
808
|
+
recordStatus();
|
|
809
|
+
})
|
|
810
|
+
.catch((error) => {
|
|
811
|
+
lastReadSucceeded = false;
|
|
812
|
+
recordStatus();
|
|
813
|
+
input.observability.warn("session recovery monitor: durable aggregate read failed", {
|
|
814
|
+
error: error instanceof Error ? error.message : String(error),
|
|
815
|
+
});
|
|
816
|
+
})
|
|
817
|
+
.finally(() => {
|
|
818
|
+
running = null;
|
|
819
|
+
});
|
|
820
|
+
};
|
|
821
|
+
refresh();
|
|
822
|
+
const timer = setInterval(refresh, intervalMs);
|
|
823
|
+
timer.unref?.();
|
|
824
|
+
return {
|
|
825
|
+
close: async () => {
|
|
826
|
+
stopped = true;
|
|
827
|
+
clearInterval(timer);
|
|
828
|
+
await running;
|
|
829
|
+
},
|
|
830
|
+
};
|
|
831
|
+
}
|
|
832
|
+
|
|
679
833
|
export function recordSandboxLeaseGauges(
|
|
680
834
|
observability: Observability,
|
|
681
835
|
counts: Partial<Record<SandboxLeaseLiveness, number>>,
|
|
@@ -869,6 +1023,7 @@ export function recordSandboxRotationBacklogGauges(
|
|
|
869
1023
|
turnBlocked: number;
|
|
870
1024
|
directBlocked: number;
|
|
871
1025
|
processBlocked: number;
|
|
1026
|
+
interactionBlocked: number;
|
|
872
1027
|
},
|
|
873
1028
|
): void {
|
|
874
1029
|
const values = {
|
|
@@ -877,6 +1032,7 @@ export function recordSandboxRotationBacklogGauges(
|
|
|
877
1032
|
turn_blocked: backlog.turnBlocked,
|
|
878
1033
|
direct_blocked: backlog.directBlocked,
|
|
879
1034
|
process_blocked: backlog.processBlocked,
|
|
1035
|
+
interaction_blocked: backlog.interactionBlocked,
|
|
880
1036
|
} as const;
|
|
881
1037
|
for (const [kind, value] of Object.entries(values)) {
|
|
882
1038
|
observability.setGauge({
|
|
@@ -1349,6 +1505,22 @@ export function recordSandboxLogicalProvision(
|
|
|
1349
1505
|
}
|
|
1350
1506
|
}
|
|
1351
1507
|
|
|
1508
|
+
export function recordSandboxSharedPreparation(
|
|
1509
|
+
observability: Observability,
|
|
1510
|
+
measurement: {
|
|
1511
|
+
path: "owner" | "joined" | "reused";
|
|
1512
|
+
outcome: "completed" | "failed";
|
|
1513
|
+
durationSeconds: number;
|
|
1514
|
+
},
|
|
1515
|
+
): void {
|
|
1516
|
+
observability.observeHistogram({
|
|
1517
|
+
name: "opengeni_sandbox_shared_preparation_duration_seconds",
|
|
1518
|
+
help: "Duration of exact-lease immutable sandbox preparation by durable coordination path and outcome.",
|
|
1519
|
+
labels: { path: measurement.path, outcome: measurement.outcome },
|
|
1520
|
+
value: Math.max(0, measurement.durationSeconds),
|
|
1521
|
+
});
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1352
1524
|
export function recordTurnWorkerPreparationTotal(
|
|
1353
1525
|
observability: Observability,
|
|
1354
1526
|
input: {
|
|
@@ -1535,6 +1707,36 @@ export function recordSessionEventAppendLatency(
|
|
|
1535
1707
|
});
|
|
1536
1708
|
}
|
|
1537
1709
|
|
|
1710
|
+
export function sessionEventBatchSizeClass(eventCount: number): string {
|
|
1711
|
+
if (!Number.isFinite(eventCount) || eventCount <= 0) return "unknown";
|
|
1712
|
+
if (eventCount === 1) return "1";
|
|
1713
|
+
if (eventCount <= 5) return "2-5";
|
|
1714
|
+
if (eventCount <= 10) return "6-10";
|
|
1715
|
+
if (eventCount <= 25) return "11-25";
|
|
1716
|
+
if (eventCount <= 50) return "26-50";
|
|
1717
|
+
return "51+";
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
/** Bounded database-phase attribution for exact-attempt event appends. */
|
|
1721
|
+
export function recordSessionEventAppendPhase(
|
|
1722
|
+
observability: Observability,
|
|
1723
|
+
observation: SessionEventAppendPhaseObservation,
|
|
1724
|
+
): void {
|
|
1725
|
+
observability.observeHistogram({
|
|
1726
|
+
name: "opengeni_session_event_append_phase_seconds",
|
|
1727
|
+
help: "Duration of one bounded database phase in an exact-attempt session event append.",
|
|
1728
|
+
buckets: STREAM_IO_BUCKETS,
|
|
1729
|
+
labels: {
|
|
1730
|
+
path: "turn_attempt",
|
|
1731
|
+
event_class: observation.eventClass,
|
|
1732
|
+
batch_size_class: sessionEventBatchSizeClass(observation.eventCount),
|
|
1733
|
+
phase: observation.phase,
|
|
1734
|
+
outcome: observation.outcome,
|
|
1735
|
+
},
|
|
1736
|
+
value: Math.max(0, observation.durationSeconds),
|
|
1737
|
+
});
|
|
1738
|
+
}
|
|
1739
|
+
|
|
1538
1740
|
/** Latency of the best-effort NATS live fan-out — the delivery path. A p99 climb
|
|
1539
1741
|
* here (with append healthy) is delivery, not the write path. */
|
|
1540
1742
|
export function recordSessionEventPublishLatency(
|
|
@@ -24,6 +24,8 @@ import {
|
|
|
24
24
|
type PersonalGitHubGitBrokerRepositoryClaim,
|
|
25
25
|
} from "@opengeni/github";
|
|
26
26
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
27
|
+
import { execFileSync } from "node:child_process";
|
|
28
|
+
import { isIP } from "node:net";
|
|
27
29
|
import { isDeepStrictEqual } from "node:util";
|
|
28
30
|
|
|
29
31
|
const PERSONAL_GITHUB_SERVER_ID = "github:personal";
|
|
@@ -39,7 +41,7 @@ export function buildPersonalGitHubGitCredentials(
|
|
|
39
41
|
): NonNullable<ConnectionCredentialsPort["gitCredentials"]> | null {
|
|
40
42
|
if (!settings.githubPersonalOauthEnabled) return null;
|
|
41
43
|
const secret = settings.integrationsStateSecret?.trim();
|
|
42
|
-
const publicOrigin = personalGitBrokerOrigin(settings
|
|
44
|
+
const publicOrigin = personalGitBrokerOrigin(settings);
|
|
43
45
|
if (!secret || !publicOrigin) {
|
|
44
46
|
throw new Error("personal GitHub Git broker configuration is unavailable");
|
|
45
47
|
}
|
|
@@ -255,19 +257,58 @@ function personalGitHubCanWrite(permissions: {
|
|
|
255
257
|
return permissions.push || permissions.maintain || permissions.admin;
|
|
256
258
|
}
|
|
257
259
|
|
|
258
|
-
|
|
260
|
+
let cachedDockerBridgeGateway: string | null | undefined;
|
|
261
|
+
|
|
262
|
+
function dockerBridgeGateway(): string | null {
|
|
263
|
+
if (cachedDockerBridgeGateway !== undefined) return cachedDockerBridgeGateway;
|
|
264
|
+
try {
|
|
265
|
+
const value = execFileSync(
|
|
266
|
+
"docker",
|
|
267
|
+
["network", "inspect", "bridge", "--format", "{{(index .IPAM.Config 0).Gateway}}"],
|
|
268
|
+
{ encoding: "utf8", timeout: 2_000, maxBuffer: 16 * 1024 },
|
|
269
|
+
).trim();
|
|
270
|
+
cachedDockerBridgeGateway = isIP(value) ? value : null;
|
|
271
|
+
} catch {
|
|
272
|
+
cachedDockerBridgeGateway = null;
|
|
273
|
+
}
|
|
274
|
+
return cachedDockerBridgeGateway;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
export function personalGitBrokerOrigin(settings: Settings): string | null {
|
|
278
|
+
const value =
|
|
279
|
+
settings.opengeniMcpUrl ??
|
|
280
|
+
(settings.sandboxBackend === "docker"
|
|
281
|
+
? `http://host.docker.internal:${settings.apiPort}`
|
|
282
|
+
: settings.sandboxBackend === "local" || settings.sandboxBackend === "none"
|
|
283
|
+
? `http://127.0.0.1:${settings.apiPort}`
|
|
284
|
+
: settings.publicBaseUrl);
|
|
259
285
|
if (!value) return null;
|
|
260
286
|
const url = new URL(value);
|
|
261
287
|
if (
|
|
262
|
-
|
|
288
|
+
settings.sandboxBackend === "docker" &&
|
|
289
|
+
(url.hostname === "127.0.0.1" || url.hostname === "localhost")
|
|
290
|
+
) {
|
|
291
|
+
url.hostname = dockerBridgeGateway() ?? "host.docker.internal";
|
|
292
|
+
}
|
|
293
|
+
const dockerPrivateOrigin =
|
|
294
|
+
settings.sandboxBackend === "docker" &&
|
|
295
|
+
(url.hostname === "host.docker.internal" ||
|
|
296
|
+
url.hostname === cachedDockerBridgeGateway ||
|
|
297
|
+
/^[a-z][a-z0-9-]{0,62}$/iu.test(url.hostname));
|
|
298
|
+
const privateExecutionOrigin =
|
|
299
|
+
(url.protocol === "http:" &&
|
|
300
|
+
(url.hostname === "127.0.0.1" || url.hostname === "localhost" || dockerPrivateOrigin)) ||
|
|
301
|
+
["local", "test"].includes(settings.environment);
|
|
302
|
+
if (
|
|
303
|
+
(url.protocol !== "https:" && !privateExecutionOrigin) ||
|
|
263
304
|
url.username ||
|
|
264
305
|
url.password ||
|
|
265
|
-
url.pathname !== "/" ||
|
|
266
306
|
url.search ||
|
|
267
307
|
url.hash
|
|
268
308
|
) {
|
|
269
309
|
return null;
|
|
270
310
|
}
|
|
311
|
+
url.pathname = "/";
|
|
271
312
|
return url.origin;
|
|
272
313
|
}
|
|
273
314
|
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// re-enters admission instead of creating a rival.
|
|
12
12
|
// 3. (spawner) commitWarmingToWarm (the lease_epoch++ fence + folds the resume
|
|
13
13
|
// envelope onto the lease). Optional desktop work is deliberately absent:
|
|
14
|
-
//
|
|
14
|
+
// viewers and managed ComputerSession operations initialize it lazily.
|
|
15
15
|
// 4. the caller injects {client, session, sessionState} NON-OWNED into the run
|
|
16
16
|
// (the SDK never reaps it — the keystone), runs, then in `finally` calls the
|
|
17
17
|
// returned `release()` and drops the in-memory handle. NEVER provider-delete
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -672,7 +672,7 @@ function settleRetainedProcessForTurn(
|
|
|
672
672
|
) {
|
|
673
673
|
throw new Error("Retained-process settlement lost its exact durable backend identity");
|
|
674
674
|
}
|
|
675
|
-
await settleRetainedProcess(services.db, {
|
|
675
|
+
const settlement = await settleRetainedProcess(services.db, {
|
|
676
676
|
accountId: fence.accountId,
|
|
677
677
|
workspaceId: ids.workspaceId,
|
|
678
678
|
sessionId: ids.sessionId,
|
|
@@ -683,14 +683,17 @@ function settleRetainedProcessForTurn(
|
|
|
683
683
|
reason: proof.reason,
|
|
684
684
|
idleGraceMs: services.settings.sandboxIdleGraceMs,
|
|
685
685
|
});
|
|
686
|
+
if (settlement.process.state === "active") {
|
|
687
|
+
throw new Error("Retained-process settlement returned an active durable process");
|
|
688
|
+
}
|
|
686
689
|
await settleSessionBackgroundCommandForRetainedProcess(services.db, {
|
|
687
690
|
accountId: fence.accountId,
|
|
688
691
|
workspaceId: ids.workspaceId,
|
|
689
692
|
sessionId: ids.sessionId,
|
|
690
693
|
retainedProcessId: process.id,
|
|
691
|
-
outcome:
|
|
692
|
-
exitCode:
|
|
693
|
-
reason: proof.reason,
|
|
694
|
+
outcome: settlement.process.state,
|
|
695
|
+
exitCode: settlement.process.exitCode,
|
|
696
|
+
reason: settlement.process.settlementReason ?? proof.reason,
|
|
694
697
|
});
|
|
695
698
|
};
|
|
696
699
|
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
export type WorkerUnhandledRejectionProcess = {
|
|
2
|
+
on: (event: "unhandledRejection", listener: (reason: unknown) => void) => void;
|
|
3
|
+
off: (event: "unhandledRejection", listener: (reason: unknown) => void) => void;
|
|
4
|
+
};
|
|
5
|
+
|
|
6
|
+
export type WorkerUnhandledRejectionClassification =
|
|
7
|
+
| "detached_runtime_mcp_lifecycle"
|
|
8
|
+
| "detached_unknown";
|
|
9
|
+
|
|
10
|
+
type RuntimeMcpLifecycleError = {
|
|
11
|
+
name: "McpLifecycleError";
|
|
12
|
+
code: "mcp_connect_failed" | "mcp_close_failed";
|
|
13
|
+
origin: "runtime";
|
|
14
|
+
serverId: string;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The Agents SDK owns MCP connect promises and also installs a process-global
|
|
19
|
+
* unhandled-rejection listener. A best-effort lifecycle rejection can surface
|
|
20
|
+
* twice under concurrent teardown: once through the awaited MCP result and once
|
|
21
|
+
* through the SDK's detached worker promise. The first path already records and
|
|
22
|
+
* degrades the server; the second must not terminate the whole turn-worker pod.
|
|
23
|
+
*/
|
|
24
|
+
export function isDetachedRuntimeMcpLifecycleRejection(
|
|
25
|
+
reason: unknown,
|
|
26
|
+
): reason is RuntimeMcpLifecycleError {
|
|
27
|
+
try {
|
|
28
|
+
if (!reason || typeof reason !== "object") return false;
|
|
29
|
+
const candidate = reason as Partial<RuntimeMcpLifecycleError>;
|
|
30
|
+
return (
|
|
31
|
+
candidate.name === "McpLifecycleError" &&
|
|
32
|
+
(candidate.code === "mcp_connect_failed" || candidate.code === "mcp_close_failed") &&
|
|
33
|
+
candidate.origin === "runtime" &&
|
|
34
|
+
typeof candidate.serverId === "string" &&
|
|
35
|
+
candidate.serverId.length > 0
|
|
36
|
+
);
|
|
37
|
+
} catch {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function reportIsolatedUnhandledRejection(
|
|
43
|
+
classification: WorkerUnhandledRejectionClassification,
|
|
44
|
+
): void {
|
|
45
|
+
// Never render the rejection itself: provider and transport errors can carry
|
|
46
|
+
// request bodies or credentials. The durable activity path owns exact errors.
|
|
47
|
+
console.error("[worker] isolated detached unhandled rejection", { classification });
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Keep process-global promise rejection handling observational. Turn activities
|
|
52
|
+
* have their own durable failure/lease boundary; killing the shared worker here
|
|
53
|
+
* converts one detached SDK/transport promise into unrelated lease losses across
|
|
54
|
+
* every session assigned to the pod. The Agents SDK also checks for a consumer
|
|
55
|
+
* listener before applying its own process.exit(1), so this boundary must never
|
|
56
|
+
* reintroduce that fail-fast behavior for an unknown rejection shape.
|
|
57
|
+
*/
|
|
58
|
+
export function installWorkerUnhandledRejectionBoundary(
|
|
59
|
+
runtimeProcess: WorkerUnhandledRejectionProcess = process,
|
|
60
|
+
report: (
|
|
61
|
+
classification: WorkerUnhandledRejectionClassification,
|
|
62
|
+
) => void = reportIsolatedUnhandledRejection,
|
|
63
|
+
): () => void {
|
|
64
|
+
const onUnhandledRejection = (reason: unknown): void => {
|
|
65
|
+
report(
|
|
66
|
+
isDetachedRuntimeMcpLifecycleRejection(reason)
|
|
67
|
+
? "detached_runtime_mcp_lifecycle"
|
|
68
|
+
: "detached_unknown",
|
|
69
|
+
);
|
|
70
|
+
};
|
|
71
|
+
runtimeProcess.on("unhandledRejection", onUnhandledRejection);
|
|
72
|
+
return () => runtimeProcess.off("unhandledRejection", onUnhandledRejection);
|
|
73
|
+
}
|
|
@@ -68,6 +68,21 @@ export const automationActivity = proxyActivities<Pick<typeof activities, "dispa
|
|
|
68
68
|
},
|
|
69
69
|
);
|
|
70
70
|
|
|
71
|
+
/** Dispatch has a bounded retry window, but an accepted automation event must
|
|
72
|
+
* not remain durably `dispatching` after that window closes. This compensating
|
|
73
|
+
* database write is idempotent and retries without a cap until the run reaches
|
|
74
|
+
* a terminal state (or is observed as already dispatched). */
|
|
75
|
+
export const automationFailureActivity = proxyActivities<
|
|
76
|
+
Pick<typeof activities, "settleAutomationRunFailure">
|
|
77
|
+
>({
|
|
78
|
+
startToCloseTimeout: "2 minutes",
|
|
79
|
+
retry: {
|
|
80
|
+
initialInterval: "1 second",
|
|
81
|
+
backoffCoefficient: 2,
|
|
82
|
+
maximumInterval: "30 seconds",
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
|
|
71
86
|
/** Goal continuation evaluates a durable Postgres obligation at an idle
|
|
72
87
|
* boundary. A transient failure gets a short retry window, then records an
|
|
73
88
|
* explicit delayed outbox wake instead of relying on an unrelated mutation or
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { automationActivity } from "./activities";
|
|
1
|
+
import { automationActivity, automationFailureActivity } from "./activities";
|
|
2
2
|
|
|
3
3
|
export type AutomationRunWorkflowInput = {
|
|
4
4
|
accountId: string;
|
|
@@ -6,6 +6,25 @@ export type AutomationRunWorkflowInput = {
|
|
|
6
6
|
runId: string;
|
|
7
7
|
};
|
|
8
8
|
|
|
9
|
+
export type AutomationRunWorkflowActivities = {
|
|
10
|
+
dispatchAutomationRun: (input: AutomationRunWorkflowInput) => Promise<unknown>;
|
|
11
|
+
settleAutomationRunFailure: (input: AutomationRunWorkflowInput) => Promise<void>;
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export async function runAutomationRunWorkflow(
|
|
15
|
+
input: AutomationRunWorkflowInput,
|
|
16
|
+
activities: AutomationRunWorkflowActivities,
|
|
17
|
+
): Promise<void> {
|
|
18
|
+
try {
|
|
19
|
+
await activities.dispatchAutomationRun(input);
|
|
20
|
+
} catch {
|
|
21
|
+
await activities.settleAutomationRunFailure(input);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
9
25
|
export async function automationRunWorkflow(input: AutomationRunWorkflowInput): Promise<void> {
|
|
10
|
-
await
|
|
26
|
+
await runAutomationRunWorkflow(input, {
|
|
27
|
+
dispatchAutomationRun: automationActivity.dispatchAutomationRun,
|
|
28
|
+
settleAutomationRunFailure: automationFailureActivity.settleAutomationRunFailure,
|
|
29
|
+
});
|
|
11
30
|
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { ChannelASession } from "@opengeni/runtime/sandbox";
|
|
2
|
-
import type { SandboxRuntimeState } from "./turn-context.js";
|
|
3
|
-
type ImageReferenceSandboxState = Pick<SandboxRuntimeState, "resolvedSandbox" | "lazyOwnedSandbox" | "turnSandboxProvisioner">;
|
|
4
|
-
/**
|
|
5
|
-
* Join the turn's canonical lazy sandbox boundary before reading a
|
|
6
|
-
* model-supplied /workspace image reference. Durable File and artifact
|
|
7
|
-
* references never call this helper because their bytes come from object
|
|
8
|
-
* storage instead.
|
|
9
|
-
*/
|
|
10
|
-
export declare function resolveImageReferenceSandboxSession(sandboxState: ImageReferenceSandboxState, sdkOwnedSandboxSession: unknown): Promise<{
|
|
11
|
-
session: ChannelASession;
|
|
12
|
-
leaseEpoch: number;
|
|
13
|
-
}>;
|
|
14
|
-
export {};
|