@opengeni/worker-bundle 2.3.1-canary.37073905159001 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/claude-usage-observer.d.ts +6 -30
- package/dist/activities/agent-turn/credential-leases.d.ts +32 -5
- package/dist/activities/agent-turn/errors.d.ts +0 -3
- package/dist/activities/agent-turn/finalization-monitor.d.ts +1 -1
- package/dist/activities/agent-turn/finalization.d.ts +0 -2
- package/dist/activities/agent-turn/quiescence.d.ts +4 -3
- package/dist/activities/agent-turn/turn-context.d.ts +1 -1
- package/dist/activities/session-state.d.ts +2 -2
- package/dist/activities/types.d.ts +0 -7
- package/dist/{activities-control-TTGQGLCJ.js → activities-control-3HAH4FMG.js} +32 -83
- package/dist/activities-control-3HAH4FMG.js.map +1 -0
- package/dist/activities-control.d.ts +4 -4
- package/dist/{activities-turn-TTNAVGKN.js → activities-turn-KU25J3VQ.js} +308 -263
- package/dist/activities-turn-KU25J3VQ.js.map +1 -0
- package/dist/activities.d.ts +4 -4
- package/dist/{chunk-PA7Z55P4.js → chunk-3K3ULWEE.js} +17 -242
- package/dist/chunk-3K3ULWEE.js.map +1 -0
- package/dist/{chunk-YOI3Z3GF.js → chunk-YG27NEMX.js} +1 -9
- package/dist/chunk-YG27NEMX.js.map +1 -0
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +2 -5
- package/dist/index.js +16 -82
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -6
- package/dist/sandbox-resume.d.ts +0 -3
- package/dist/worker-service-lifecycle.d.ts +0 -12
- package/dist/workflow-bundle.js +2 -2
- package/package.json +21 -21
- package/src/activities/agent-turn/claim.ts +0 -13
- package/src/activities/agent-turn/claude-usage-observer.ts +8 -99
- package/src/activities/agent-turn/credential-leases.ts +213 -86
- package/src/activities/agent-turn/errors.ts +1 -61
- package/src/activities/agent-turn/failure-settlement.ts +5 -59
- package/src/activities/agent-turn/finalization-monitor.ts +4 -3
- package/src/activities/agent-turn/finalization.ts +3 -8
- package/src/activities/agent-turn/quiescence.ts +4 -3
- package/src/activities/agent-turn/run.ts +20 -24
- package/src/activities/agent-turn/sandbox-establish.ts +0 -2
- package/src/activities/agent-turn/sandbox-runtime.ts +1 -5
- package/src/activities/agent-turn/skill-selection.ts +0 -2
- package/src/activities/agent-turn/turn-context.ts +4 -1
- package/src/activities/codex-rotation.ts +12 -2
- package/src/activities/sandbox-lease.ts +8 -59
- package/src/activities/session-state.ts +4 -29
- package/src/activities/types.ts +0 -7
- package/src/activities-turn.ts +0 -5
- package/src/activities.ts +0 -6
- package/src/http.ts +2 -4
- package/src/index.ts +9 -39
- package/src/observability-metrics.ts +0 -27
- package/src/sandbox-resume.ts +0 -9
- package/src/sandbox-routing.ts +0 -77
- package/src/worker-service-lifecycle.ts +0 -53
- package/src/workflows/session.ts +1 -6
- package/dist/activities/agent-turn/subscription-lease.d.ts +0 -38
- package/dist/activities-control-TTGQGLCJ.js.map +0 -1
- package/dist/activities-turn-TTNAVGKN.js.map +0 -1
- package/dist/chunk-PA7Z55P4.js.map +0 -1
- package/dist/chunk-YOI3Z3GF.js.map +0 -1
- package/dist/session-event-fanout.d.ts +0 -6
- package/src/activities/agent-turn/subscription-lease.ts +0 -115
- package/src/session-event-fanout.ts +0 -23
|
@@ -1,17 +1,6 @@
|
|
|
1
1
|
import { type parseModelProvidersJson, type ClaudeUsageObservation } from "@opengeni/config";
|
|
2
2
|
type Scope = "workspace" | "organization";
|
|
3
|
-
type ClaudeRequestCredential = {
|
|
4
|
-
token: string;
|
|
5
|
-
connectionId: string;
|
|
6
|
-
credentialVersion: number;
|
|
7
|
-
};
|
|
8
|
-
export declare class ClaudeSubscriptionConnectionUnavailable extends Error {
|
|
9
|
-
readonly status = 409;
|
|
10
|
-
readonly code = "claude_subscription_connection_changed";
|
|
11
|
-
constructor();
|
|
12
|
-
}
|
|
13
3
|
export type CapturedClaudeUsage = {
|
|
14
|
-
scope: Scope;
|
|
15
4
|
token: string;
|
|
16
5
|
expectedConnectionId: string;
|
|
17
6
|
expectedCredentialVersion: number;
|
|
@@ -22,29 +11,16 @@ export type CapturedClaudeUsage = {
|
|
|
22
11
|
};
|
|
23
12
|
};
|
|
24
13
|
/** Capture the exact credential before requests; replacement fences late responses. */
|
|
25
|
-
export declare function createClaudeUsageObserver(providers: ReturnType<typeof parseModelProvidersJson>, latest: Map<
|
|
14
|
+
export declare function createClaudeUsageObserver(providers: ReturnType<typeof parseModelProvidersJson>, latest: Map<Scope, CapturedClaudeUsage>, readCredential: (scope: Scope) => Promise<{
|
|
26
15
|
token: string;
|
|
27
16
|
connectionId: string;
|
|
28
17
|
credentialVersion: number;
|
|
29
|
-
} | null>): Promise<((providerId: string, response: Response
|
|
30
|
-
prepareRequestWithObserver: (providerId: string, headers: Headers, resolve: (binding: {
|
|
31
|
-
scope: Scope;
|
|
32
|
-
expectedConnectionId: string;
|
|
33
|
-
expectedCredentialVersion: number;
|
|
34
|
-
}) => Promise<ClaudeRequestCredential | null>) => Promise<{
|
|
35
|
-
headers: Headers;
|
|
36
|
-
observe: (providerId: string, response: Response, upstreamModelId?: string, requestToken?: string | null) => void;
|
|
37
|
-
}>;
|
|
38
|
-
prepareRequest(providerId: string, headers: Headers, resolve: (binding: {
|
|
39
|
-
scope: Scope;
|
|
40
|
-
expectedConnectionId: string;
|
|
41
|
-
expectedCredentialVersion: number;
|
|
42
|
-
}) => Promise<ClaudeRequestCredential | null>): Promise<Headers>;
|
|
18
|
+
} | null>): Promise<((providerId: string, response: Response) => void) & {
|
|
43
19
|
binding(providerId: string): {
|
|
44
|
-
scope: Scope;
|
|
45
|
-
token: string;
|
|
46
|
-
expectedConnectionId: string;
|
|
47
|
-
expectedCredentialVersion: number;
|
|
20
|
+
readonly scope: Scope;
|
|
21
|
+
readonly token: string;
|
|
22
|
+
readonly expectedConnectionId: string;
|
|
23
|
+
readonly expectedCredentialVersion: number;
|
|
48
24
|
} | undefined;
|
|
49
25
|
renew(providerId: string, credential: {
|
|
50
26
|
token: string;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { SharedActivityServices } from "../types.js";
|
|
2
|
-
|
|
3
|
-
export type
|
|
2
|
+
export type LeaseRenewReason = "timer" | "runtime_event" | "model_usage";
|
|
3
|
+
export type LeaseLossReason = "deadline" | "not_found";
|
|
4
4
|
export declare class CodexCredentialLeaseLostError extends Error {
|
|
5
5
|
readonly reason: LeaseLossReason;
|
|
6
6
|
readonly code = "codex_credential_lease_lost";
|
|
@@ -19,13 +19,40 @@ export type TurnCredentialLeaseDeps = {
|
|
|
19
19
|
* cross-replica fairness primitive; the heartbeat here only extends its short
|
|
20
20
|
* TTL. A killed worker stops heartbeating and the holder self-expires.
|
|
21
21
|
*/
|
|
22
|
-
export declare class CodexTurnLease
|
|
22
|
+
export declare class CodexTurnLease {
|
|
23
|
+
private readonly deps;
|
|
24
|
+
held: boolean;
|
|
25
|
+
lost: boolean;
|
|
26
|
+
lossReason: LeaseLossReason | null;
|
|
27
|
+
holderId: string | null;
|
|
28
|
+
generation: number | null;
|
|
29
|
+
confirmedUntilMs: number | null;
|
|
30
|
+
private heartbeatInFlight;
|
|
31
|
+
private heartbeatTimer;
|
|
23
32
|
constructor(deps: TurnCredentialLeaseDeps);
|
|
33
|
+
markLost: (reason: LeaseLossReason) => void;
|
|
34
|
+
/** Fail closed at the last worker-confirmed lease deadline before dispatch. */
|
|
35
|
+
assertUsable: () => void;
|
|
36
|
+
renew: (reason: LeaseRenewReason) => Promise<void>;
|
|
37
|
+
startHeartbeat: () => void;
|
|
38
|
+
stopHeartbeat: () => void;
|
|
24
39
|
}
|
|
25
|
-
/** SuperGrok
|
|
26
|
-
export declare class XaiTurnLease
|
|
40
|
+
/** The xAI/SuperGrok credential holder for one running turn; same shape. */
|
|
41
|
+
export declare class XaiTurnLease {
|
|
42
|
+
private readonly deps;
|
|
43
|
+
held: boolean;
|
|
44
|
+
lost: boolean;
|
|
27
45
|
subjectId: string | null;
|
|
46
|
+
holderId: string | null;
|
|
47
|
+
generation: number | null;
|
|
48
|
+
confirmedUntilMs: number | null;
|
|
49
|
+
private heartbeatInFlight;
|
|
50
|
+
private heartbeatTimer;
|
|
28
51
|
constructor(deps: TurnCredentialLeaseDeps);
|
|
52
|
+
markLost: (reason: LeaseLossReason) => void;
|
|
53
|
+
renew: () => Promise<void>;
|
|
54
|
+
startHeartbeat: () => void;
|
|
55
|
+
stopHeartbeat: () => void;
|
|
29
56
|
}
|
|
30
57
|
/** Both serving-credential leases for one turn attempt plus their composite views. */
|
|
31
58
|
export type TurnCredentialLeases = {
|
|
@@ -90,7 +90,6 @@ export declare function postClaimDatabaseRecoveryFailure(input: {
|
|
|
90
90
|
triggerEventId: string;
|
|
91
91
|
executionGeneration: number;
|
|
92
92
|
sandboxSetupOutcomeUnknown?: true;
|
|
93
|
-
sandboxSetupRecoveryExhausted?: true;
|
|
94
93
|
providerRecovery?: {
|
|
95
94
|
failureCode: string;
|
|
96
95
|
providerRecoveryCount: number;
|
|
@@ -267,8 +266,6 @@ export declare function classifyXaiCredentialFailure(error: unknown): XaiCredent
|
|
|
267
266
|
export declare function agentRunFailurePayload(error: unknown, options?: {
|
|
268
267
|
isCodexTurn?: boolean;
|
|
269
268
|
}): ReturnType<typeof baseAgentRunFailurePayload>;
|
|
270
|
-
/** Keep Anthropic provider text on terminal failures, never recovery events. */
|
|
271
|
-
export declare function agentRunRecoveryFailurePayload(error: unknown, failure: ReturnType<typeof agentRunFailurePayload>): ReturnType<typeof agentRunFailurePayload>;
|
|
272
269
|
declare function baseAgentRunFailurePayload(error: unknown, options?: {
|
|
273
270
|
isCodexTurn?: boolean;
|
|
274
271
|
}): {
|
|
@@ -8,7 +8,7 @@ export declare function startTurnFinalizationMonitor(input: {
|
|
|
8
8
|
observability: Observability;
|
|
9
9
|
details: TurnHeartbeatDetails;
|
|
10
10
|
heartbeat: (details: TurnHeartbeatDetails) => void;
|
|
11
|
-
|
|
11
|
+
terminateWorker: () => void;
|
|
12
12
|
timeoutMs?: number;
|
|
13
13
|
slowAfterMs?: number;
|
|
14
14
|
}): {
|
|
@@ -16,8 +16,6 @@ export type TurnFinalizationDeps = {
|
|
|
16
16
|
wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
|
|
17
17
|
signalSessionAttemptQuiesced: ActivityServices["signalSessionAttemptQuiesced"];
|
|
18
18
|
signalCodexCapacityWorkflow: ActivityServices["signalCodexCapacityWorkflow"];
|
|
19
|
-
requestWorkerDrain: ActivityServices["requestWorkerDrain"];
|
|
20
|
-
turnFinalizationTimeoutMs: ActivityServices["turnFinalizationTimeoutMs"];
|
|
21
19
|
cancellationSignal: AbortSignal | undefined;
|
|
22
20
|
sandboxResumeController: AbortController;
|
|
23
21
|
activityContext: ReturnType<typeof currentActivityContext>;
|
|
@@ -49,9 +49,10 @@ export declare const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
|
|
|
49
49
|
export declare const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5000;
|
|
50
50
|
export declare const TURN_QUIESCENCE_WATCHDOG_MS: number;
|
|
51
51
|
export declare const TURN_FINALIZER_STEP_TIMEOUT_MS = 30000;
|
|
52
|
-
/** A fenced activity must never heartbeat forever while physical
|
|
53
|
-
* stuck. The
|
|
54
|
-
*
|
|
52
|
+
/** A fenced activity must never heartbeat forever while its physical writer
|
|
53
|
+
* drain is stuck. The last-resort worker exit stops every in-process writer;
|
|
54
|
+
* Kubernetes replaces the pod and the workflow's heartbeat-lease reconciler
|
|
55
|
+
* then commits the exact quiescence receipt. */
|
|
55
56
|
export declare function armTurnQuiescenceWatchdog(input: {
|
|
56
57
|
enabled: boolean;
|
|
57
58
|
timeoutMs?: number;
|
|
@@ -146,7 +146,7 @@ export type ProviderTurnState = {
|
|
|
146
146
|
xaiCredentialQuarantined: boolean;
|
|
147
147
|
priorSessionCodexCredentialId: string | null;
|
|
148
148
|
latestCodexUsage: CodexUsageHeaderSnapshot | null;
|
|
149
|
-
latestClaudeUsage: Map<
|
|
149
|
+
latestClaudeUsage: Map<"workspace" | "organization", import("./claude-usage-observer.js").CapturedClaudeUsage>;
|
|
150
150
|
lastCodexRequestOpaqueArtifacts: readonly string[];
|
|
151
151
|
};
|
|
152
152
|
export type TurnContext = {
|
|
@@ -53,8 +53,8 @@ export declare function createSessionStateActivities(services: () => Promise<Con
|
|
|
53
53
|
admissionFence?: import("@opengeni/db").SessionAdmissionFence;
|
|
54
54
|
} | {
|
|
55
55
|
kind: "admission-blocked";
|
|
56
|
-
reason?: "sandbox_setup_outcome_unknown"
|
|
57
|
-
ref?: import("@opengeni/db").SandboxSetupOutcomeUnknown
|
|
56
|
+
reason?: "sandbox_setup_outcome_unknown";
|
|
57
|
+
ref?: import("@opengeni/db").SandboxSetupOutcomeUnknown;
|
|
58
58
|
} | {
|
|
59
59
|
kind: "sandbox-lifecycle-wait";
|
|
60
60
|
ref: import("@opengeni/db").SandboxLifecycleWait;
|
|
@@ -87,11 +87,6 @@ export type SharedActivityServices = {
|
|
|
87
87
|
export type ControlActivityServices = SharedActivityServices;
|
|
88
88
|
/** Turn workers own the model loop and never construct document parsers. */
|
|
89
89
|
export type TurnActivityServices = SharedActivityServices & {
|
|
90
|
-
/** Host-owned containment: stop polling and checkpoint peer turns before exit.
|
|
91
|
-
* Bare activity embedders must provide their own worker lifecycle edge. */
|
|
92
|
-
requestWorkerDrain: () => void;
|
|
93
|
-
/** Deterministic test clock for the cleanup-only monitor. */
|
|
94
|
-
turnFinalizationTimeoutMs?: number | undefined;
|
|
95
90
|
runtime: OpenGeniRuntime;
|
|
96
91
|
/** Provider-free test/profiling seam; production injects the real runtime summarizer. */
|
|
97
92
|
summarizeContextForCompaction: typeof import("@opengeni/runtime").summarizeForCompaction;
|
|
@@ -242,8 +237,6 @@ export type PostClaimDatabaseRecoveryDetail = {
|
|
|
242
237
|
providerFailureCode?: string;
|
|
243
238
|
/** Persist the no-replay setup marker, not a new provider/setup attempt. */
|
|
244
239
|
sandboxSetupOutcomeUnknown?: true;
|
|
245
|
-
/** Preserve the exhausted budget for a command proven not dispatched. */
|
|
246
|
-
sandboxSetupRecoveryExhausted?: true;
|
|
247
240
|
};
|
|
248
241
|
export declare const POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE = "OpenGeniPostClaimDatabaseRecovery";
|
|
249
242
|
export declare const POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE = "Agent turn database recovery required after attempt claim.";
|
|
@@ -24,7 +24,6 @@ import {
|
|
|
24
24
|
notifyParentOfChildIdle,
|
|
25
25
|
persistWorkspaceArchiveCandidate,
|
|
26
26
|
providerRecoveryCountFromMetadata,
|
|
27
|
-
publishDurableSessionEvents,
|
|
28
27
|
putVersion1TarArchiveOrInline,
|
|
29
28
|
reconcileAutomaticSessionTitleFanout,
|
|
30
29
|
reconcilePendingParentSystemUpdates,
|
|
@@ -35,7 +34,7 @@ import {
|
|
|
35
34
|
warnDrainSnapshotFailure,
|
|
36
35
|
withFirstPartyTools,
|
|
37
36
|
workflowIdForSession
|
|
38
|
-
} from "./chunk-
|
|
37
|
+
} from "./chunk-3K3ULWEE.js";
|
|
39
38
|
import {
|
|
40
39
|
inspectOpenSandboxKubernetesInventory,
|
|
41
40
|
recordActiveUserGauges,
|
|
@@ -48,7 +47,6 @@ import {
|
|
|
48
47
|
recordRetainedProcessReconciliation,
|
|
49
48
|
recordSandboxCheckpointArtifactGauges,
|
|
50
49
|
recordSandboxCheckpointArtifactOutcome,
|
|
51
|
-
recordSandboxCommandContainment,
|
|
52
50
|
recordSandboxDeadlineRotationsRequested,
|
|
53
51
|
recordSandboxInventoryProjectionFailure,
|
|
54
52
|
recordSandboxInventoryProjectionSuccess,
|
|
@@ -62,7 +60,7 @@ import {
|
|
|
62
60
|
recordVerifiedSignupTrialSwitchGauge,
|
|
63
61
|
recordWorkerDeathRecoveryMetrics,
|
|
64
62
|
runtimeMetricsHooksForObservability
|
|
65
|
-
} from "./chunk-
|
|
63
|
+
} from "./chunk-YG27NEMX.js";
|
|
66
64
|
import "./chunk-PZ5AY32C.js";
|
|
67
65
|
|
|
68
66
|
// src/activities/knowledge-indexing.ts
|
|
@@ -2723,7 +2721,6 @@ import {
|
|
|
2723
2721
|
workspaceArchiveCaptureDeadlineElapsed,
|
|
2724
2722
|
retainedProcessReconciliationProof,
|
|
2725
2723
|
retainedProcessSettlementIdentity,
|
|
2726
|
-
recoverManagedSessionBackgroundCommand,
|
|
2727
2724
|
settleClaimedConnectedMachineBackgroundCommand,
|
|
2728
2725
|
settleSandboxCheckpointArtifactGc,
|
|
2729
2726
|
rlsContextForWorkspace,
|
|
@@ -2890,18 +2887,14 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2890
2887
|
try {
|
|
2891
2888
|
const { db, settings, observability } = await services();
|
|
2892
2889
|
const timing = sandboxDrainTiming(settings);
|
|
2893
|
-
const
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
observability.warn("sandbox reaper: retained command containment inspection failed", {
|
|
2898
|
-
error: error instanceof Error ? error.message : String(error)
|
|
2899
|
-
});
|
|
2900
|
-
}
|
|
2890
|
+
const onUnobservableCommandDrainError = (error) => {
|
|
2891
|
+
observability.warn("sandbox reaper: unobservable command drain inspection failed", {
|
|
2892
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2893
|
+
});
|
|
2901
2894
|
};
|
|
2902
2895
|
if (!settings.sandboxOwnershipEnabled) {
|
|
2903
2896
|
const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
|
|
2904
|
-
|
|
2897
|
+
onUnobservableCommandDrainError,
|
|
2905
2898
|
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
2906
2899
|
turnHolderTtlMs: settings.sandboxLeaseTtlMs,
|
|
2907
2900
|
interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
|
|
@@ -2928,7 +2921,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2928
2921
|
});
|
|
2929
2922
|
recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
|
|
2930
2923
|
const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
|
|
2931
|
-
|
|
2924
|
+
onUnobservableCommandDrainError,
|
|
2932
2925
|
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
2933
2926
|
// Dead-worker turn holders: a live holder is touched every 10s from the
|
|
2934
2927
|
// moment it is registered (resumeBoxForTurn's holder-liveness loop covers
|
|
@@ -2969,7 +2962,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2969
2962
|
instanceId: input.target.instanceId
|
|
2970
2963
|
});
|
|
2971
2964
|
try {
|
|
2972
|
-
const { db, settings, observability, objectStorage
|
|
2965
|
+
const { db, settings, observability, objectStorage } = await services();
|
|
2973
2966
|
assertSandboxDrainInputTiming(input);
|
|
2974
2967
|
const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
|
|
2975
2968
|
const row = input.target;
|
|
@@ -2989,8 +2982,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2989
2982
|
terminateBox,
|
|
2990
2983
|
probeDrainableProvider,
|
|
2991
2984
|
captureAttempt,
|
|
2992
|
-
objectStorage
|
|
2993
|
-
bus
|
|
2985
|
+
objectStorage
|
|
2994
2986
|
);
|
|
2995
2987
|
return { status: drainedCold ? "terminated" : "skipped" };
|
|
2996
2988
|
} catch (error) {
|
|
@@ -3625,24 +3617,6 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3625
3617
|
sessionId: process.sessionId,
|
|
3626
3618
|
processId: process.id
|
|
3627
3619
|
};
|
|
3628
|
-
if (process.ownerActorKind === "turn" && process.providerBackend === "modal" && !proof && !claim.ownerState.startsWith("background_")) {
|
|
3629
|
-
try {
|
|
3630
|
-
const command = await recoverManagedSessionBackgroundCommand(db, {
|
|
3631
|
-
...processScope,
|
|
3632
|
-
expected,
|
|
3633
|
-
reconciliationClaimId: claim.claimId
|
|
3634
|
-
});
|
|
3635
|
-
if (command) {
|
|
3636
|
-
claim.ownerState = `background_${command.state}`;
|
|
3637
|
-
recordRetainedProcessReconciliation(observability, "background_recovered");
|
|
3638
|
-
}
|
|
3639
|
-
} catch (error) {
|
|
3640
|
-
observability.warn("sandbox reaper: retained-command background recovery failed", {
|
|
3641
|
-
processId: process.id,
|
|
3642
|
-
error: error instanceof Error ? error.message : String(error)
|
|
3643
|
-
});
|
|
3644
|
-
}
|
|
3645
|
-
}
|
|
3646
3620
|
const storedCommandPersistence = retainedProviderCommandPersistence(
|
|
3647
3621
|
db,
|
|
3648
3622
|
processScope,
|
|
@@ -4634,7 +4608,7 @@ function sandboxDrainReason(lease) {
|
|
|
4634
4608
|
if (lease.rotationRequestedAt === null) return "idle";
|
|
4635
4609
|
return lease.rotationReason ?? "rotation_requested";
|
|
4636
4610
|
}
|
|
4637
|
-
async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage
|
|
4611
|
+
async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage) {
|
|
4638
4612
|
const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
|
|
4639
4613
|
const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
|
|
4640
4614
|
if (!lease) {
|
|
@@ -4948,31 +4922,18 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
4948
4922
|
}
|
|
4949
4923
|
}
|
|
4950
4924
|
const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
|
|
4951
|
-
const { wentCold
|
|
4925
|
+
const { wentCold } = await confirmDrainCold(db, {
|
|
4952
4926
|
accountId,
|
|
4953
4927
|
workspaceId: row.workspaceId,
|
|
4954
4928
|
sandboxGroupId: row.sandboxGroupId,
|
|
4955
4929
|
expectedEpoch: row.leaseEpoch,
|
|
4956
4930
|
...captureClaim ? { expectedCaptureId: captureClaim.id } : {},
|
|
4957
|
-
providerMissingBeforeCapture: providerMissing
|
|
4958
|
-
idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs
|
|
4959
|
-
});
|
|
4960
|
-
await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
|
|
4961
|
-
observability.warn("sandbox reaper: contained command event fanout failed", {
|
|
4962
|
-
sandboxGroupId: row.sandboxGroupId,
|
|
4963
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4964
|
-
});
|
|
4931
|
+
providerMissingBeforeCapture: providerMissing
|
|
4965
4932
|
});
|
|
4966
4933
|
if (wentCold) {
|
|
4967
4934
|
if (providerMissing) {
|
|
4968
4935
|
recordSandboxProviderMissingBeforeCapture(observability, backend);
|
|
4969
4936
|
}
|
|
4970
|
-
if (lease.unobservableCommandDrainIds?.length) {
|
|
4971
|
-
recordSandboxCommandContainment(
|
|
4972
|
-
observability,
|
|
4973
|
-
providerMissing ? "provider_missing" : "contained"
|
|
4974
|
-
);
|
|
4975
|
-
}
|
|
4976
4937
|
try {
|
|
4977
4938
|
await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
|
|
4978
4939
|
type: "sandbox.box.terminated",
|
|
@@ -5356,7 +5317,7 @@ import {
|
|
|
5356
5317
|
upsertScheduledSessionGoalForRun,
|
|
5357
5318
|
withSessionActivityRlsContext
|
|
5358
5319
|
} from "@opengeni/db";
|
|
5359
|
-
import { publishDurableSessionEvents
|
|
5320
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
5360
5321
|
import {
|
|
5361
5322
|
allowedFirstPartyMcpToolsForSession,
|
|
5362
5323
|
resolveFirstPartyMcpToolPolicy,
|
|
@@ -6497,7 +6458,7 @@ function createScheduledTaskActivities(services) {
|
|
|
6497
6458
|
events: started.events
|
|
6498
6459
|
});
|
|
6499
6460
|
} else {
|
|
6500
|
-
await
|
|
6461
|
+
await publishDurableSessionEvents(
|
|
6501
6462
|
bus,
|
|
6502
6463
|
task.workspaceId,
|
|
6503
6464
|
session.id,
|
|
@@ -6540,7 +6501,7 @@ function createScheduledTaskActivities(services) {
|
|
|
6540
6501
|
if (deferPublications) {
|
|
6541
6502
|
deferredEvents.push({ sessionId: session.id, events: started.events });
|
|
6542
6503
|
} else {
|
|
6543
|
-
await
|
|
6504
|
+
await publishDurableSessionEvents(
|
|
6544
6505
|
bus,
|
|
6545
6506
|
task.workspaceId,
|
|
6546
6507
|
session.id,
|
|
@@ -6569,7 +6530,7 @@ function createScheduledTaskActivities(services) {
|
|
|
6569
6530
|
if (deferPublications) {
|
|
6570
6531
|
deferredEvents.push({ sessionId: session.id, events: titleEvents });
|
|
6571
6532
|
} else {
|
|
6572
|
-
await
|
|
6533
|
+
await publishDurableSessionEvents(bus, task.workspaceId, session.id, titleEvents);
|
|
6573
6534
|
}
|
|
6574
6535
|
}
|
|
6575
6536
|
const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
|
|
@@ -6656,7 +6617,7 @@ function createScheduledTaskActivities(services) {
|
|
|
6656
6617
|
events: scheduledUpdate.events
|
|
6657
6618
|
});
|
|
6658
6619
|
} else {
|
|
6659
|
-
await
|
|
6620
|
+
await publishDurableSessionEvents(
|
|
6660
6621
|
bus,
|
|
6661
6622
|
task.workspaceId,
|
|
6662
6623
|
session.id,
|
|
@@ -6783,7 +6744,7 @@ function createScheduledTaskActivities(services) {
|
|
|
6783
6744
|
events: bundled.events
|
|
6784
6745
|
});
|
|
6785
6746
|
} else {
|
|
6786
|
-
await
|
|
6747
|
+
await publishDurableSessionEvents(
|
|
6787
6748
|
bus,
|
|
6788
6749
|
task.workspaceId,
|
|
6789
6750
|
session.id,
|
|
@@ -6865,7 +6826,7 @@ function createScheduledTaskActivities(services) {
|
|
|
6865
6826
|
});
|
|
6866
6827
|
}
|
|
6867
6828
|
for (const deferred of deferredEvents) {
|
|
6868
|
-
await
|
|
6829
|
+
await publishDurableSessionEvents(
|
|
6869
6830
|
bus,
|
|
6870
6831
|
task.workspaceId,
|
|
6871
6832
|
deferred.sessionId,
|
|
@@ -7097,7 +7058,7 @@ async function seedScheduledGeneratedSessionRoute(input) {
|
|
|
7097
7058
|
if (input.deferPublications) {
|
|
7098
7059
|
input.deferredEvents.push({ sessionId: input.session.id, events: failed.events });
|
|
7099
7060
|
} else {
|
|
7100
|
-
await
|
|
7061
|
+
await publishDurableSessionEvents(
|
|
7101
7062
|
input.bus,
|
|
7102
7063
|
input.task.workspaceId,
|
|
7103
7064
|
input.session.id,
|
|
@@ -7367,7 +7328,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
7367
7328
|
});
|
|
7368
7329
|
workflowId = started.temporalWorkflowId;
|
|
7369
7330
|
if (started.events.length > 0) {
|
|
7370
|
-
await
|
|
7331
|
+
await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, started.events);
|
|
7371
7332
|
}
|
|
7372
7333
|
if (task.runMode === "reusable_session") {
|
|
7373
7334
|
await materializeScheduledTaskReusableSessionFromRun(input.db, {
|
|
@@ -7386,7 +7347,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
7386
7347
|
taskName: task.name
|
|
7387
7348
|
});
|
|
7388
7349
|
if (titleEvents.length > 0) {
|
|
7389
|
-
await
|
|
7350
|
+
await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, titleEvents);
|
|
7390
7351
|
}
|
|
7391
7352
|
}
|
|
7392
7353
|
const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
|
|
@@ -7479,7 +7440,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
7479
7440
|
recordUserMessageAccepted(input.observability, { surface: "scheduled" });
|
|
7480
7441
|
}
|
|
7481
7442
|
if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
|
|
7482
|
-
await
|
|
7443
|
+
await publishDurableSessionEvents(
|
|
7483
7444
|
input.bus,
|
|
7484
7445
|
task.workspaceId,
|
|
7485
7446
|
session.id,
|
|
@@ -7835,10 +7796,9 @@ import {
|
|
|
7835
7796
|
expireSessionHumanInputRequest,
|
|
7836
7797
|
markSessionAttemptQuiesced,
|
|
7837
7798
|
requireSession as requireSession2,
|
|
7838
|
-
settleSessionIdleWithParentOutbox
|
|
7839
|
-
SANDBOX_SETUP_RECOVERY_LIMIT
|
|
7799
|
+
settleSessionIdleWithParentOutbox
|
|
7840
7800
|
} from "@opengeni/db";
|
|
7841
|
-
import { publishDurableSessionEvents as
|
|
7801
|
+
import { publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
|
|
7842
7802
|
import { CancelledFailure } from "@temporalio/activity";
|
|
7843
7803
|
var WORKER_DEATH_MAX_REDISPATCHES = 3;
|
|
7844
7804
|
function createSessionStateActivities(services, overrides = {}) {
|
|
@@ -7862,7 +7822,7 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7862
7822
|
const requireSessionFn = overrides.requireSession ?? requireSession2;
|
|
7863
7823
|
const settleSessionIdleWithParentOutboxFn = overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
|
|
7864
7824
|
const markSessionAttemptQuiescedFn = overrides.markSessionAttemptQuiesced ?? markSessionAttemptQuiesced;
|
|
7865
|
-
const publishDurableSessionEventsFn = overrides.publishDurableSessionEvents ??
|
|
7825
|
+
const publishDurableSessionEventsFn = overrides.publishDurableSessionEvents ?? publishDurableSessionEvents2;
|
|
7866
7826
|
const deliverFailedChildTurnToParentFn = overrides.deliverFailedChildTurnToParent ?? deliverFailedChildTurnToParent;
|
|
7867
7827
|
const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
|
|
7868
7828
|
const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
|
|
@@ -7951,16 +7911,12 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7951
7911
|
const postClaimIdentityMatches = Boolean(
|
|
7952
7912
|
postClaimRecovery && turn.id === postClaimRecovery.turnId && turn.triggerEventId === postClaimRecovery.triggerEventId && turn.executionGeneration === postClaimRecovery.executionGeneration
|
|
7953
7913
|
);
|
|
7954
|
-
if (postClaimRecovery?.sandboxSetupRecoveryExhausted && !postClaimIdentityMatches) {
|
|
7955
|
-
return { action: "stale" };
|
|
7956
|
-
}
|
|
7957
7914
|
const providerRecoveryCount = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryCount : void 0;
|
|
7958
7915
|
const providerFailureCode = postClaimIdentityMatches ? postClaimRecovery?.providerFailureCode : void 0;
|
|
7959
7916
|
const hasProviderRecoveryCount = providerRecoveryCount !== void 0;
|
|
7960
7917
|
const hasProviderFailureCode = providerFailureCode !== void 0;
|
|
7961
7918
|
const setupOutcomeUnknown = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupOutcomeUnknown === true;
|
|
7962
|
-
|
|
7963
|
-
if (setupOutcomeUnknown && setupRecoveryExhausted || (setupOutcomeUnknown || setupRecoveryExhausted) && (hasProviderRecoveryCount || hasProviderFailureCode) || setupRecoveryExhausted && turn.metadata?.providerRecoveryCount !== SANDBOX_SETUP_RECOVERY_LIMIT) {
|
|
7919
|
+
if (setupOutcomeUnknown && (hasProviderRecoveryCount || hasProviderFailureCode)) {
|
|
7964
7920
|
return { action: "stale" };
|
|
7965
7921
|
}
|
|
7966
7922
|
if (hasProviderRecoveryCount !== hasProviderFailureCode) {
|
|
@@ -7976,20 +7932,13 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7976
7932
|
turnId: turn.id,
|
|
7977
7933
|
triggerEventId: turn.triggerEventId,
|
|
7978
7934
|
attemptId: input.attemptId,
|
|
7979
|
-
reason: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" :
|
|
7935
|
+
reason: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : providerFailureCode ?? "claimed_attempt_database_failure",
|
|
7980
7936
|
...setupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {},
|
|
7981
|
-
...setupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
|
|
7982
7937
|
...providerRecoveryCount !== void 0 ? { providerRecoveryCount } : {},
|
|
7983
7938
|
detail: {
|
|
7984
|
-
code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" :
|
|
7985
|
-
retryable: !setupOutcomeUnknown
|
|
7939
|
+
code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : providerFailureCode ?? recoveredClaimCode,
|
|
7940
|
+
retryable: !setupOutcomeUnknown,
|
|
7986
7941
|
...setupOutcomeUnknown ? { setupOutcome: "unknown", replay: "blocked" } : {},
|
|
7987
|
-
...setupRecoveryExhausted ? {
|
|
7988
|
-
setupOutcome: "not_started",
|
|
7989
|
-
replay: "blocked",
|
|
7990
|
-
recoveryExhausted: true,
|
|
7991
|
-
providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT
|
|
7992
|
-
} : {},
|
|
7993
7942
|
...providerRecoveryCount !== void 0 ? {
|
|
7994
7943
|
databaseFailureCode: recoveredClaimCode,
|
|
7995
7944
|
providerRecoveryCount
|
|
@@ -8445,4 +8394,4 @@ export {
|
|
|
8445
8394
|
createControlActivities,
|
|
8446
8395
|
createControlActivitiesFromServices
|
|
8447
8396
|
};
|
|
8448
|
-
//# sourceMappingURL=activities-control-
|
|
8397
|
+
//# sourceMappingURL=activities-control-3HAH4FMG.js.map
|