@opengeni/worker-bundle 0.22.8 → 0.23.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +1 -4
- package/dist/activities/agent-turn/claim.d.ts +2 -4
- package/dist/activities/agent-turn/codex.d.ts +6 -5
- package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
- package/dist/activities/agent-turn/errors.d.ts +13 -0
- package/dist/activities/agent-turn/finalization.d.ts +1 -3
- package/dist/activities/agent-turn/quiescence.d.ts +39 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
- package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +3 -32
- package/dist/activities/agent-turn/turn-context.d.ts +1 -8
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
- package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
- package/dist/activities/codex-rotation.d.ts +2 -0
- package/dist/activities/session-state.d.ts +2 -1
- package/dist/{activities-control-VZIIV7RX.js → activities-control-52W5IDWM.js} +125 -13
- package/dist/activities-control-52W5IDWM.js.map +1 -0
- package/dist/{activities-turn-5RGQDBFV.js → activities-turn-A2G2KLSY.js} +534 -482
- package/dist/activities-turn-A2G2KLSY.js.map +1 -0
- package/dist/{chunk-NAVSAZ56.js → chunk-GFNXVZVQ.js} +160 -1
- package/dist/chunk-GFNXVZVQ.js.map +1 -0
- package/dist/{chunk-O4HDX5SC.js → chunk-IPVMCAW5.js} +144 -15
- package/dist/chunk-IPVMCAW5.js.map +1 -0
- package/dist/index.js +42 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +28 -0
- package/dist/personal-github-git-credentials.d.ts +1 -0
- package/dist/unhandled-rejection-boundary.d.ts +29 -0
- package/package.json +14 -14
- package/src/activities/agent-turn/agent-build.ts +15 -31
- package/src/activities/agent-turn/claim.ts +29 -68
- package/src/activities/agent-turn/codex.ts +19 -10
- package/src/activities/agent-turn/compaction-prep.ts +50 -1
- package/src/activities/agent-turn/errors.ts +78 -0
- package/src/activities/agent-turn/failure-settlement.ts +30 -19
- package/src/activities/agent-turn/finalization.ts +24 -21
- package/src/activities/agent-turn/history.ts +5 -1
- package/src/activities/agent-turn/quiescence.ts +86 -0
- package/src/activities/agent-turn/run-credentials.ts +12 -1
- package/src/activities/agent-turn/run.ts +14 -11
- package/src/activities/agent-turn/sandbox-establish.ts +46 -11
- package/src/activities/agent-turn/sandbox-route.ts +21 -25
- package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
- package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
- package/src/activities/agent-turn/stream-attempt.ts +98 -14
- package/src/activities/agent-turn/tool-environment.ts +10 -13
- package/src/activities/agent-turn/tool-policy.ts +3 -55
- package/src/activities/agent-turn/turn-context.ts +8 -14
- package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
- package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
- package/src/activities/codex-capacity.ts +13 -5
- package/src/activities/codex-rotation.ts +17 -0
- package/src/activities/goals.ts +10 -4
- package/src/activities/retained-screenshots.ts +104 -21
- package/src/activities/sandbox-lease.ts +7 -4
- package/src/activities/scheduled-tasks.ts +128 -4
- package/src/activities/session-state.ts +8 -1
- package/src/github-rest-mcp.ts +6 -3
- package/src/http.ts +3 -0
- package/src/index.ts +19 -0
- package/src/observability-metrics.ts +200 -0
- package/src/personal-github-git-credentials.ts +45 -4
- package/src/sandbox-resume.ts +1 -1
- package/src/sandbox-routing.ts +7 -4
- package/src/unhandled-rejection-boundary.ts +73 -0
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
- package/dist/activities/recording.d.ts +0 -90
- package/dist/activities-control-VZIIV7RX.js.map +0 -1
- package/dist/activities-turn-5RGQDBFV.js.map +0 -1
- package/dist/chunk-NAVSAZ56.js.map +0 -1
- package/dist/chunk-O4HDX5SC.js.map +0 -1
- package/src/activities/recording.ts +0 -279
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
3
3
|
import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
|
|
4
|
+
import type { SessionEventAppendPhaseObservation } from "@opengeni/db";
|
|
4
5
|
import type { EventLogger } from "@opengeni/events";
|
|
5
6
|
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
6
7
|
import type { CompanyBrainContributionReceipt } from "./model-context-contributions";
|
|
@@ -18,6 +19,7 @@ import {
|
|
|
18
19
|
} from "@opengeni/runtime";
|
|
19
20
|
|
|
20
21
|
export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
|
|
22
|
+
export type WorkerDeathRecoveryOutcome = "recovering" | "exhausted";
|
|
21
23
|
export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
|
|
22
24
|
export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
|
|
23
25
|
export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
@@ -28,6 +30,10 @@ export type TurnTaskQueueStats = {
|
|
|
28
30
|
tasksAddRate: number;
|
|
29
31
|
tasksDispatchRate: number;
|
|
30
32
|
};
|
|
33
|
+
export type SessionRecoveryBacklog = {
|
|
34
|
+
quiescence_missing: number;
|
|
35
|
+
projection_stale: number;
|
|
36
|
+
};
|
|
31
37
|
|
|
32
38
|
export type TemporalTurnTaskQueueStats = {
|
|
33
39
|
approximateBacklogCount?: unknown;
|
|
@@ -49,6 +55,12 @@ const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>(
|
|
|
49
55
|
// before floating-point precision can degrade; a restart resets both the
|
|
50
56
|
// process-local Prometheus registry and this guard together.
|
|
51
57
|
const MAX_MODEL_CACHE_COUNTER_TOTAL = 1_000_000_000_000;
|
|
58
|
+
const TURN_OUTCOMES: readonly TurnOutcome[] = ["completed", "failed", "cancelled", "recovering"];
|
|
59
|
+
const WORKER_DEATH_RECOVERY_OUTCOMES: readonly WorkerDeathRecoveryOutcome[] = [
|
|
60
|
+
"recovering",
|
|
61
|
+
"exhausted",
|
|
62
|
+
];
|
|
63
|
+
const WORKER_DEATH_TIMEOUT_TYPES = ["heartbeat", "schedule_to_start"] as const;
|
|
52
64
|
|
|
53
65
|
export function observabilityEventLogger(observability: Observability): EventLogger {
|
|
54
66
|
return {
|
|
@@ -163,6 +175,19 @@ export function runtimeMetricsHooksForObservability(
|
|
|
163
175
|
value: durationSeconds,
|
|
164
176
|
});
|
|
165
177
|
},
|
|
178
|
+
onMcpLifecycle: ({ phase, policy, outcome, durationSeconds }) => {
|
|
179
|
+
observability.incrementCounter({
|
|
180
|
+
name: "opengeni_mcp_lifecycle_operations_total",
|
|
181
|
+
help: "Total physical MCP lifecycle operations by bounded phase, policy, and outcome.",
|
|
182
|
+
labels: { phase, policy, outcome },
|
|
183
|
+
});
|
|
184
|
+
observability.observeHistogram({
|
|
185
|
+
name: "opengeni_mcp_lifecycle_operation_duration_seconds",
|
|
186
|
+
help: "MCP lifecycle operation duration in seconds by bounded phase, policy, and outcome.",
|
|
187
|
+
labels: { phase, policy, outcome },
|
|
188
|
+
value: durationSeconds,
|
|
189
|
+
});
|
|
190
|
+
},
|
|
166
191
|
onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
|
|
167
192
|
observability.incrementCounter({
|
|
168
193
|
name: "opengeni_machine_op_total",
|
|
@@ -300,6 +325,53 @@ export function turnLifecycleMetricsFor(observability: Observability): TurnLifec
|
|
|
300
325
|
return tracker;
|
|
301
326
|
}
|
|
302
327
|
|
|
328
|
+
export function initializeWorkerOutcomeMetrics(observability: Observability): void {
|
|
329
|
+
for (const outcome of TURN_OUTCOMES) {
|
|
330
|
+
observability.incrementCounter({
|
|
331
|
+
name: "opengeni_turns_total",
|
|
332
|
+
help: "Total agent turns by terminal outcome.",
|
|
333
|
+
labels: { outcome },
|
|
334
|
+
amount: 0,
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
for (const outcome of WORKER_DEATH_RECOVERY_OUTCOMES) {
|
|
338
|
+
for (const timeoutType of WORKER_DEATH_TIMEOUT_TYPES) {
|
|
339
|
+
observability.incrementCounter({
|
|
340
|
+
name: "opengeni_turn_worker_death_recoveries_total",
|
|
341
|
+
help: "Total durable same-turn worker-death recovery outcomes.",
|
|
342
|
+
labels: { outcome, timeout_type: timeoutType },
|
|
343
|
+
amount: 0,
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
export function recordWorkerDeathRecoveryMetrics(
|
|
350
|
+
observability: Observability,
|
|
351
|
+
input: {
|
|
352
|
+
outcome: WorkerDeathRecoveryOutcome;
|
|
353
|
+
timeoutType: "HEARTBEAT" | "SCHEDULE_TO_START";
|
|
354
|
+
},
|
|
355
|
+
): void {
|
|
356
|
+
const turnOutcome: TurnOutcome = input.outcome === "exhausted" ? "failed" : "recovering";
|
|
357
|
+
observability.incrementCounter({
|
|
358
|
+
name: "opengeni_turn_worker_death_recoveries_total",
|
|
359
|
+
help: "Total durable same-turn worker-death recovery outcomes.",
|
|
360
|
+
labels: {
|
|
361
|
+
outcome: input.outcome,
|
|
362
|
+
timeout_type: input.timeoutType.toLowerCase(),
|
|
363
|
+
},
|
|
364
|
+
});
|
|
365
|
+
// The turn process that owned this attempt is gone, so its process-local
|
|
366
|
+
// lifecycle tracker cannot publish the outcome. Record it from the fenced
|
|
367
|
+
// control activity only after the durable recovery transaction wins.
|
|
368
|
+
observability.incrementCounter({
|
|
369
|
+
name: "opengeni_turns_total",
|
|
370
|
+
help: "Total agent turns by terminal outcome.",
|
|
371
|
+
labels: { outcome: turnOutcome },
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
|
|
303
375
|
export class TurnLifecycleMetrics {
|
|
304
376
|
private readonly turns = new Map<string, { startedAt: number; lastProgressAt: number }>();
|
|
305
377
|
private timer: ReturnType<typeof setInterval> | null = null;
|
|
@@ -676,6 +748,88 @@ export function startTurnCapacityMonitor(input: {
|
|
|
676
748
|
};
|
|
677
749
|
}
|
|
678
750
|
|
|
751
|
+
export function recordSessionRecoveryBacklogGauges(
|
|
752
|
+
observability: Observability,
|
|
753
|
+
counts: SessionRecoveryBacklog,
|
|
754
|
+
): void {
|
|
755
|
+
for (const state of ["quiescence_missing", "projection_stale"] as const) {
|
|
756
|
+
observability.setGauge({
|
|
757
|
+
name: "opengeni_session_recovery_backlog",
|
|
758
|
+
help: "Current durable recovering-session obligations with no active attempt, by bounded reconciliation state.",
|
|
759
|
+
labels: { state },
|
|
760
|
+
value: nonnegativeFinite(counts[state]),
|
|
761
|
+
});
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
export function startSessionRecoveryMonitor(input: {
|
|
766
|
+
observability: Observability;
|
|
767
|
+
read: () => Promise<SessionRecoveryBacklog>;
|
|
768
|
+
intervalMs?: number;
|
|
769
|
+
now?: () => number;
|
|
770
|
+
}): { close: () => Promise<void> } {
|
|
771
|
+
const intervalMs = input.intervalMs ?? 60_000;
|
|
772
|
+
const now = input.now ?? Date.now;
|
|
773
|
+
const startedAt = now();
|
|
774
|
+
let lastSuccessAt: number | null = null;
|
|
775
|
+
let lastReadSucceeded = false;
|
|
776
|
+
let stopped = false;
|
|
777
|
+
let running: Promise<void> | null = null;
|
|
778
|
+
const recordStatus = () => {
|
|
779
|
+
const observedAt = now();
|
|
780
|
+
const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
|
|
781
|
+
const set = (name: string, help: string, value: number) =>
|
|
782
|
+
input.observability.setGauge({ name, help, value });
|
|
783
|
+
set(
|
|
784
|
+
"opengeni_session_recovery_monitor_last_read_success",
|
|
785
|
+
"Whether the latest durable session-recovery aggregate read completed successfully.",
|
|
786
|
+
lastReadSucceeded ? 1 : 0,
|
|
787
|
+
);
|
|
788
|
+
set(
|
|
789
|
+
"opengeni_session_recovery_monitor_last_success_timestamp_seconds",
|
|
790
|
+
"Unix timestamp of the latest successful durable session-recovery aggregate read, or zero before one succeeds.",
|
|
791
|
+
lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
|
|
792
|
+
);
|
|
793
|
+
set(
|
|
794
|
+
"opengeni_session_recovery_monitor_fresh",
|
|
795
|
+
"Whether durable session-recovery backlog gauges have a successful read within three monitor intervals.",
|
|
796
|
+
lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
|
|
797
|
+
);
|
|
798
|
+
};
|
|
799
|
+
const refresh = () => {
|
|
800
|
+
recordStatus();
|
|
801
|
+
if (stopped || running) return;
|
|
802
|
+
running = input
|
|
803
|
+
.read()
|
|
804
|
+
.then((counts) => {
|
|
805
|
+
recordSessionRecoveryBacklogGauges(input.observability, counts);
|
|
806
|
+
lastSuccessAt = now();
|
|
807
|
+
lastReadSucceeded = true;
|
|
808
|
+
recordStatus();
|
|
809
|
+
})
|
|
810
|
+
.catch((error) => {
|
|
811
|
+
lastReadSucceeded = false;
|
|
812
|
+
recordStatus();
|
|
813
|
+
input.observability.warn("session recovery monitor: durable aggregate read failed", {
|
|
814
|
+
error: error instanceof Error ? error.message : String(error),
|
|
815
|
+
});
|
|
816
|
+
})
|
|
817
|
+
.finally(() => {
|
|
818
|
+
running = null;
|
|
819
|
+
});
|
|
820
|
+
};
|
|
821
|
+
refresh();
|
|
822
|
+
const timer = setInterval(refresh, intervalMs);
|
|
823
|
+
timer.unref?.();
|
|
824
|
+
return {
|
|
825
|
+
close: async () => {
|
|
826
|
+
stopped = true;
|
|
827
|
+
clearInterval(timer);
|
|
828
|
+
await running;
|
|
829
|
+
},
|
|
830
|
+
};
|
|
831
|
+
}
|
|
832
|
+
|
|
679
833
|
export function recordSandboxLeaseGauges(
|
|
680
834
|
observability: Observability,
|
|
681
835
|
counts: Partial<Record<SandboxLeaseLiveness, number>>,
|
|
@@ -1349,6 +1503,22 @@ export function recordSandboxLogicalProvision(
|
|
|
1349
1503
|
}
|
|
1350
1504
|
}
|
|
1351
1505
|
|
|
1506
|
+
export function recordSandboxSharedPreparation(
|
|
1507
|
+
observability: Observability,
|
|
1508
|
+
measurement: {
|
|
1509
|
+
path: "owner" | "joined" | "reused";
|
|
1510
|
+
outcome: "completed" | "failed";
|
|
1511
|
+
durationSeconds: number;
|
|
1512
|
+
},
|
|
1513
|
+
): void {
|
|
1514
|
+
observability.observeHistogram({
|
|
1515
|
+
name: "opengeni_sandbox_shared_preparation_duration_seconds",
|
|
1516
|
+
help: "Duration of exact-lease immutable sandbox preparation by durable coordination path and outcome.",
|
|
1517
|
+
labels: { path: measurement.path, outcome: measurement.outcome },
|
|
1518
|
+
value: Math.max(0, measurement.durationSeconds),
|
|
1519
|
+
});
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1352
1522
|
export function recordTurnWorkerPreparationTotal(
|
|
1353
1523
|
observability: Observability,
|
|
1354
1524
|
input: {
|
|
@@ -1535,6 +1705,36 @@ export function recordSessionEventAppendLatency(
|
|
|
1535
1705
|
});
|
|
1536
1706
|
}
|
|
1537
1707
|
|
|
1708
|
+
export function sessionEventBatchSizeClass(eventCount: number): string {
|
|
1709
|
+
if (!Number.isFinite(eventCount) || eventCount <= 0) return "unknown";
|
|
1710
|
+
if (eventCount === 1) return "1";
|
|
1711
|
+
if (eventCount <= 5) return "2-5";
|
|
1712
|
+
if (eventCount <= 10) return "6-10";
|
|
1713
|
+
if (eventCount <= 25) return "11-25";
|
|
1714
|
+
if (eventCount <= 50) return "26-50";
|
|
1715
|
+
return "51+";
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
/** Bounded database-phase attribution for exact-attempt event appends. */
|
|
1719
|
+
export function recordSessionEventAppendPhase(
|
|
1720
|
+
observability: Observability,
|
|
1721
|
+
observation: SessionEventAppendPhaseObservation,
|
|
1722
|
+
): void {
|
|
1723
|
+
observability.observeHistogram({
|
|
1724
|
+
name: "opengeni_session_event_append_phase_seconds",
|
|
1725
|
+
help: "Duration of one bounded database phase in an exact-attempt session event append.",
|
|
1726
|
+
buckets: STREAM_IO_BUCKETS,
|
|
1727
|
+
labels: {
|
|
1728
|
+
path: "turn_attempt",
|
|
1729
|
+
event_class: observation.eventClass,
|
|
1730
|
+
batch_size_class: sessionEventBatchSizeClass(observation.eventCount),
|
|
1731
|
+
phase: observation.phase,
|
|
1732
|
+
outcome: observation.outcome,
|
|
1733
|
+
},
|
|
1734
|
+
value: Math.max(0, observation.durationSeconds),
|
|
1735
|
+
});
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1538
1738
|
/** Latency of the best-effort NATS live fan-out — the delivery path. A p99 climb
|
|
1539
1739
|
* here (with append healthy) is delivery, not the write path. */
|
|
1540
1740
|
export function recordSessionEventPublishLatency(
|
|
@@ -24,6 +24,8 @@ import {
|
|
|
24
24
|
type PersonalGitHubGitBrokerRepositoryClaim,
|
|
25
25
|
} from "@opengeni/github";
|
|
26
26
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
27
|
+
import { execFileSync } from "node:child_process";
|
|
28
|
+
import { isIP } from "node:net";
|
|
27
29
|
import { isDeepStrictEqual } from "node:util";
|
|
28
30
|
|
|
29
31
|
const PERSONAL_GITHUB_SERVER_ID = "github:personal";
|
|
@@ -39,7 +41,7 @@ export function buildPersonalGitHubGitCredentials(
|
|
|
39
41
|
): NonNullable<ConnectionCredentialsPort["gitCredentials"]> | null {
|
|
40
42
|
if (!settings.githubPersonalOauthEnabled) return null;
|
|
41
43
|
const secret = settings.integrationsStateSecret?.trim();
|
|
42
|
-
const publicOrigin = personalGitBrokerOrigin(settings
|
|
44
|
+
const publicOrigin = personalGitBrokerOrigin(settings);
|
|
43
45
|
if (!secret || !publicOrigin) {
|
|
44
46
|
throw new Error("personal GitHub Git broker configuration is unavailable");
|
|
45
47
|
}
|
|
@@ -255,19 +257,58 @@ function personalGitHubCanWrite(permissions: {
|
|
|
255
257
|
return permissions.push || permissions.maintain || permissions.admin;
|
|
256
258
|
}
|
|
257
259
|
|
|
258
|
-
|
|
260
|
+
let cachedDockerBridgeGateway: string | null | undefined;
|
|
261
|
+
|
|
262
|
+
function dockerBridgeGateway(): string | null {
|
|
263
|
+
if (cachedDockerBridgeGateway !== undefined) return cachedDockerBridgeGateway;
|
|
264
|
+
try {
|
|
265
|
+
const value = execFileSync(
|
|
266
|
+
"docker",
|
|
267
|
+
["network", "inspect", "bridge", "--format", "{{(index .IPAM.Config 0).Gateway}}"],
|
|
268
|
+
{ encoding: "utf8", timeout: 2_000, maxBuffer: 16 * 1024 },
|
|
269
|
+
).trim();
|
|
270
|
+
cachedDockerBridgeGateway = isIP(value) ? value : null;
|
|
271
|
+
} catch {
|
|
272
|
+
cachedDockerBridgeGateway = null;
|
|
273
|
+
}
|
|
274
|
+
return cachedDockerBridgeGateway;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
export function personalGitBrokerOrigin(settings: Settings): string | null {
|
|
278
|
+
const value =
|
|
279
|
+
settings.opengeniMcpUrl ??
|
|
280
|
+
(settings.sandboxBackend === "docker"
|
|
281
|
+
? `http://host.docker.internal:${settings.apiPort}`
|
|
282
|
+
: settings.sandboxBackend === "local" || settings.sandboxBackend === "none"
|
|
283
|
+
? `http://127.0.0.1:${settings.apiPort}`
|
|
284
|
+
: settings.publicBaseUrl);
|
|
259
285
|
if (!value) return null;
|
|
260
286
|
const url = new URL(value);
|
|
261
287
|
if (
|
|
262
|
-
|
|
288
|
+
settings.sandboxBackend === "docker" &&
|
|
289
|
+
(url.hostname === "127.0.0.1" || url.hostname === "localhost")
|
|
290
|
+
) {
|
|
291
|
+
url.hostname = dockerBridgeGateway() ?? "host.docker.internal";
|
|
292
|
+
}
|
|
293
|
+
const dockerPrivateOrigin =
|
|
294
|
+
settings.sandboxBackend === "docker" &&
|
|
295
|
+
(url.hostname === "host.docker.internal" ||
|
|
296
|
+
url.hostname === cachedDockerBridgeGateway ||
|
|
297
|
+
/^[a-z][a-z0-9-]{0,62}$/iu.test(url.hostname));
|
|
298
|
+
const privateExecutionOrigin =
|
|
299
|
+
(url.protocol === "http:" &&
|
|
300
|
+
(url.hostname === "127.0.0.1" || url.hostname === "localhost" || dockerPrivateOrigin)) ||
|
|
301
|
+
["local", "test"].includes(settings.environment);
|
|
302
|
+
if (
|
|
303
|
+
(url.protocol !== "https:" && !privateExecutionOrigin) ||
|
|
263
304
|
url.username ||
|
|
264
305
|
url.password ||
|
|
265
|
-
url.pathname !== "/" ||
|
|
266
306
|
url.search ||
|
|
267
307
|
url.hash
|
|
268
308
|
) {
|
|
269
309
|
return null;
|
|
270
310
|
}
|
|
311
|
+
url.pathname = "/";
|
|
271
312
|
return url.origin;
|
|
272
313
|
}
|
|
273
314
|
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// re-enters admission instead of creating a rival.
|
|
12
12
|
// 3. (spawner) commitWarmingToWarm (the lease_epoch++ fence + folds the resume
|
|
13
13
|
// envelope onto the lease). Optional desktop work is deliberately absent:
|
|
14
|
-
//
|
|
14
|
+
// viewers and managed ComputerSession operations initialize it lazily.
|
|
15
15
|
// 4. the caller injects {client, session, sessionState} NON-OWNED into the run
|
|
16
16
|
// (the SDK never reaps it — the keystone), runs, then in `finally` calls the
|
|
17
17
|
// returned `release()` and drops the in-memory handle. NEVER provider-delete
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -672,7 +672,7 @@ function settleRetainedProcessForTurn(
|
|
|
672
672
|
) {
|
|
673
673
|
throw new Error("Retained-process settlement lost its exact durable backend identity");
|
|
674
674
|
}
|
|
675
|
-
await settleRetainedProcess(services.db, {
|
|
675
|
+
const settlement = await settleRetainedProcess(services.db, {
|
|
676
676
|
accountId: fence.accountId,
|
|
677
677
|
workspaceId: ids.workspaceId,
|
|
678
678
|
sessionId: ids.sessionId,
|
|
@@ -683,14 +683,17 @@ function settleRetainedProcessForTurn(
|
|
|
683
683
|
reason: proof.reason,
|
|
684
684
|
idleGraceMs: services.settings.sandboxIdleGraceMs,
|
|
685
685
|
});
|
|
686
|
+
if (settlement.process.state === "active") {
|
|
687
|
+
throw new Error("Retained-process settlement returned an active durable process");
|
|
688
|
+
}
|
|
686
689
|
await settleSessionBackgroundCommandForRetainedProcess(services.db, {
|
|
687
690
|
accountId: fence.accountId,
|
|
688
691
|
workspaceId: ids.workspaceId,
|
|
689
692
|
sessionId: ids.sessionId,
|
|
690
693
|
retainedProcessId: process.id,
|
|
691
|
-
outcome:
|
|
692
|
-
exitCode:
|
|
693
|
-
reason: proof.reason,
|
|
694
|
+
outcome: settlement.process.state,
|
|
695
|
+
exitCode: settlement.process.exitCode,
|
|
696
|
+
reason: settlement.process.settlementReason ?? proof.reason,
|
|
694
697
|
});
|
|
695
698
|
};
|
|
696
699
|
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
export type WorkerUnhandledRejectionProcess = {
|
|
2
|
+
on: (event: "unhandledRejection", listener: (reason: unknown) => void) => void;
|
|
3
|
+
off: (event: "unhandledRejection", listener: (reason: unknown) => void) => void;
|
|
4
|
+
};
|
|
5
|
+
|
|
6
|
+
export type WorkerUnhandledRejectionClassification =
|
|
7
|
+
| "detached_runtime_mcp_lifecycle"
|
|
8
|
+
| "detached_unknown";
|
|
9
|
+
|
|
10
|
+
type RuntimeMcpLifecycleError = {
|
|
11
|
+
name: "McpLifecycleError";
|
|
12
|
+
code: "mcp_connect_failed" | "mcp_close_failed";
|
|
13
|
+
origin: "runtime";
|
|
14
|
+
serverId: string;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The Agents SDK owns MCP connect promises and also installs a process-global
|
|
19
|
+
* unhandled-rejection listener. A best-effort lifecycle rejection can surface
|
|
20
|
+
* twice under concurrent teardown: once through the awaited MCP result and once
|
|
21
|
+
* through the SDK's detached worker promise. The first path already records and
|
|
22
|
+
* degrades the server; the second must not terminate the whole turn-worker pod.
|
|
23
|
+
*/
|
|
24
|
+
export function isDetachedRuntimeMcpLifecycleRejection(
|
|
25
|
+
reason: unknown,
|
|
26
|
+
): reason is RuntimeMcpLifecycleError {
|
|
27
|
+
try {
|
|
28
|
+
if (!reason || typeof reason !== "object") return false;
|
|
29
|
+
const candidate = reason as Partial<RuntimeMcpLifecycleError>;
|
|
30
|
+
return (
|
|
31
|
+
candidate.name === "McpLifecycleError" &&
|
|
32
|
+
(candidate.code === "mcp_connect_failed" || candidate.code === "mcp_close_failed") &&
|
|
33
|
+
candidate.origin === "runtime" &&
|
|
34
|
+
typeof candidate.serverId === "string" &&
|
|
35
|
+
candidate.serverId.length > 0
|
|
36
|
+
);
|
|
37
|
+
} catch {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function reportIsolatedUnhandledRejection(
|
|
43
|
+
classification: WorkerUnhandledRejectionClassification,
|
|
44
|
+
): void {
|
|
45
|
+
// Never render the rejection itself: provider and transport errors can carry
|
|
46
|
+
// request bodies or credentials. The durable activity path owns exact errors.
|
|
47
|
+
console.error("[worker] isolated detached unhandled rejection", { classification });
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Keep process-global promise rejection handling observational. Turn activities
|
|
52
|
+
* have their own durable failure/lease boundary; killing the shared worker here
|
|
53
|
+
* converts one detached SDK/transport promise into unrelated lease losses across
|
|
54
|
+
* every session assigned to the pod. The Agents SDK also checks for a consumer
|
|
55
|
+
* listener before applying its own process.exit(1), so this boundary must never
|
|
56
|
+
* reintroduce that fail-fast behavior for an unknown rejection shape.
|
|
57
|
+
*/
|
|
58
|
+
export function installWorkerUnhandledRejectionBoundary(
|
|
59
|
+
runtimeProcess: WorkerUnhandledRejectionProcess = process,
|
|
60
|
+
report: (
|
|
61
|
+
classification: WorkerUnhandledRejectionClassification,
|
|
62
|
+
) => void = reportIsolatedUnhandledRejection,
|
|
63
|
+
): () => void {
|
|
64
|
+
const onUnhandledRejection = (reason: unknown): void => {
|
|
65
|
+
report(
|
|
66
|
+
isDetachedRuntimeMcpLifecycleRejection(reason)
|
|
67
|
+
? "detached_runtime_mcp_lifecycle"
|
|
68
|
+
: "detached_unknown",
|
|
69
|
+
);
|
|
70
|
+
};
|
|
71
|
+
runtimeProcess.on("unhandledRejection", onUnhandledRejection);
|
|
72
|
+
return () => runtimeProcess.off("unhandledRejection", onUnhandledRejection);
|
|
73
|
+
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { ChannelASession } from "@opengeni/runtime/sandbox";
|
|
2
|
-
import type { SandboxRuntimeState } from "./turn-context.js";
|
|
3
|
-
type ImageReferenceSandboxState = Pick<SandboxRuntimeState, "resolvedSandbox" | "lazyOwnedSandbox" | "turnSandboxProvisioner">;
|
|
4
|
-
/**
|
|
5
|
-
* Join the turn's canonical lazy sandbox boundary before reading a
|
|
6
|
-
* model-supplied /workspace image reference. Durable File and artifact
|
|
7
|
-
* references never call this helper because their bytes come from object
|
|
8
|
-
* storage instead.
|
|
9
|
-
*/
|
|
10
|
-
export declare function resolveImageReferenceSandboxSession(sandboxState: ImageReferenceSandboxState, sdkOwnedSandboxSession: unknown): Promise<{
|
|
11
|
-
session: ChannelASession;
|
|
12
|
-
leaseEpoch: number;
|
|
13
|
-
}>;
|
|
14
|
-
export {};
|
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
import type { Settings } from "@opengeni/config";
|
|
2
|
-
import type { Database } from "@opengeni/db";
|
|
3
|
-
import type { RecordingFailedReason, RecordingStartedPayload } from "@opengeni/contracts";
|
|
4
|
-
import { type RecordingProcess } from "@opengeni/runtime";
|
|
5
|
-
import type { ObjectStorage } from "@opengeni/storage";
|
|
6
|
-
import { DOWNLOAD_URL_TTL_SECONDS } from "@opengeni/storage";
|
|
7
|
-
export type RecordingMode = "manual" | "on-turn" | "on-verify";
|
|
8
|
-
export type ActiveRecording = {
|
|
9
|
-
recordingId: string;
|
|
10
|
-
turnId: string | null;
|
|
11
|
-
mode: RecordingMode;
|
|
12
|
-
proc: RecordingProcess;
|
|
13
|
-
dimensions: [number, number];
|
|
14
|
-
framerate: number;
|
|
15
|
-
};
|
|
16
|
-
export declare const RECORDING_SETTLEMENT_PREP_TIMEOUT_MS = 100000;
|
|
17
|
-
export declare function withRecordingPreparationTimeout(preparation: Promise<PreparedRecordingSettlement>, timeoutResult: PreparedRecordingSettlement, timeoutMs?: number): Promise<PreparedRecordingSettlement>;
|
|
18
|
-
/**
|
|
19
|
-
* Insert the recording row, launch ffmpeg on the box, and return the live handle.
|
|
20
|
-
* Emits the `recording.started` payload (the caller publishes it on the spine).
|
|
21
|
-
*/
|
|
22
|
-
export declare function beginRecording(args: {
|
|
23
|
-
settings: Settings;
|
|
24
|
-
db: Database;
|
|
25
|
-
accountId: string;
|
|
26
|
-
workspaceId: string;
|
|
27
|
-
sessionId: string;
|
|
28
|
-
turnId: string | null;
|
|
29
|
-
recordingId: string;
|
|
30
|
-
mode: RecordingMode;
|
|
31
|
-
session: unknown;
|
|
32
|
-
runAs?: string | undefined;
|
|
33
|
-
reason?: string | null;
|
|
34
|
-
}): Promise<{
|
|
35
|
-
active: ActiveRecording;
|
|
36
|
-
started: RecordingStartedPayload;
|
|
37
|
-
}>;
|
|
38
|
-
export type PreparedRecordingSettlement = {
|
|
39
|
-
mutation: {
|
|
40
|
-
action: "available";
|
|
41
|
-
recordingId: string;
|
|
42
|
-
storageKey: string;
|
|
43
|
-
sizeBytes: number;
|
|
44
|
-
durationSeconds: number;
|
|
45
|
-
};
|
|
46
|
-
deleteArtifactsAfterCommit: true;
|
|
47
|
-
} | {
|
|
48
|
-
mutation: {
|
|
49
|
-
action: "failed";
|
|
50
|
-
recordingId: string;
|
|
51
|
-
reason: RecordingFailedReason;
|
|
52
|
-
detail: string | null;
|
|
53
|
-
};
|
|
54
|
-
deleteArtifactsAfterCommit: false;
|
|
55
|
-
} | {
|
|
56
|
-
mutation: {
|
|
57
|
-
action: "discard";
|
|
58
|
-
recordingId: string;
|
|
59
|
-
};
|
|
60
|
-
deleteArtifactsAfterCommit: true;
|
|
61
|
-
};
|
|
62
|
-
/**
|
|
63
|
-
* Prepare an on-turn recording for the attempt-closing settlement. This performs
|
|
64
|
-
* only bounded box/storage work; it does not mutate canonical database state and
|
|
65
|
-
* never deletes the box artifact. The caller folds the returned mutation into the
|
|
66
|
-
* exact attempt-fenced turn settlement, then deletes artifacts only after commit.
|
|
67
|
-
* NEVER throws and never prevents turn truth from settling.
|
|
68
|
-
*/
|
|
69
|
-
export declare function prepareRecordingForSettlement(args: {
|
|
70
|
-
settings: Settings;
|
|
71
|
-
objectStorage: ObjectStorage | null;
|
|
72
|
-
workspaceId: string;
|
|
73
|
-
sessionId: string;
|
|
74
|
-
active: ActiveRecording;
|
|
75
|
-
session: unknown;
|
|
76
|
-
didComputerUse: boolean;
|
|
77
|
-
}): Promise<PreparedRecordingSettlement>;
|
|
78
|
-
/**
|
|
79
|
-
* Compensate a start whose fenced `recording.started` event was rejected. Since
|
|
80
|
-
* no canonical event was admitted, this exact recording id must leave no row,
|
|
81
|
-
* process, or box artifact behind.
|
|
82
|
-
*/
|
|
83
|
-
export declare function discardUnpublishedRecording(args: {
|
|
84
|
-
db: Database;
|
|
85
|
-
accountId: string;
|
|
86
|
-
workspaceId: string;
|
|
87
|
-
active: ActiveRecording;
|
|
88
|
-
session: unknown;
|
|
89
|
-
}): Promise<void>;
|
|
90
|
-
export { DOWNLOAD_URL_TTL_SECONDS };
|