@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +7 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +6 -5
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +13 -8
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/goals.d.ts +1 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +11 -1
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +94 -9
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +21 -26
- package/src/activities/agent-turn/claim.ts +26 -1
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +73 -12
- package/src/activities/agent-turn/failure-settlement.ts +472 -331
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +29 -17
- package/src/activities/agent-turn/run.ts +22 -5
- package/src/activities/agent-turn/stream-attempt.ts +35 -8
- package/src/activities/agent-turn/tool-environment.ts +42 -32
- package/src/activities/agent-turn/tool-policy.ts +14 -8
- package/src/activities/agent-turn/turn-context.ts +13 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +107 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/goals.ts +9 -9
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +246 -87
- package/src/activities/session-state.ts +20 -1
- package/src/activities/types.ts +16 -8
- package/src/activities/xai-quota.ts +35 -0
- package/src/activities.ts +1 -0
- package/src/github-rest-mcp.ts +0 -82
- package/src/observability-metrics.ts +4 -0
- package/src/sandbox-routing.ts +186 -16
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +114 -8
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
- package/dist/activities-turn-I4OPLQLS.js.map +0 -1
- package/dist/chunk-65JEWP2Q.js.map +0 -1
- package/dist/chunk-RQNH6CGR.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -151
package/src/github-rest-mcp.ts
CHANGED
|
@@ -11,23 +11,16 @@ import {
|
|
|
11
11
|
} from "@opengeni/contracts/personal-github";
|
|
12
12
|
import { personalGitHubRepositoryResources } from "@opengeni/core";
|
|
13
13
|
import {
|
|
14
|
-
beginConnectorActionExecution,
|
|
15
|
-
completeConnectorActionExecution,
|
|
16
14
|
getConnectionMetadata,
|
|
17
15
|
getPersonalGitHubRepositorySelectionState,
|
|
18
16
|
hasAuditableGitHubInstallationAuthority,
|
|
19
17
|
listGitHubInstallationAccessForWorkspace,
|
|
20
|
-
prepareConnectorActionApproval,
|
|
21
18
|
resolveAcceptedConnectionUse,
|
|
22
19
|
type Database,
|
|
23
20
|
type ResolveConnectionCredentialInput,
|
|
24
21
|
type ResolveConnectionCredentialResult,
|
|
25
22
|
type SessionTurnForExecution,
|
|
26
23
|
} from "@opengeni/db";
|
|
27
|
-
import {
|
|
28
|
-
AttemptToolApprovalRequiredError,
|
|
29
|
-
type AttemptToolAuthorization,
|
|
30
|
-
} from "@opengeni/codemode";
|
|
31
24
|
import { createGitHubAppInstallationTokenWithExpiry, githubAppBotIdentity } from "@opengeni/github";
|
|
32
25
|
import {
|
|
33
26
|
type AttemptConnectorActionBinding,
|
|
@@ -44,7 +37,6 @@ import {
|
|
|
44
37
|
GitHubRestAuthorityError,
|
|
45
38
|
githubRestConnectorActionOutcome,
|
|
46
39
|
GitHubRestMcpServer,
|
|
47
|
-
GitHubRestMutationNotExecutedError,
|
|
48
40
|
type GitHubRestRepository,
|
|
49
41
|
} from "@opengeni/runtime/github-rest-mcp";
|
|
50
42
|
|
|
@@ -60,7 +52,6 @@ export type GitHubRestMcpRuntime = {
|
|
|
60
52
|
tools: ToolRef[];
|
|
61
53
|
localMcpServers: LocalMcpServerRegistration[];
|
|
62
54
|
connectorBindings: AttemptConnectorActionBinding[];
|
|
63
|
-
authorizeCodemodeCall: AttemptToolAuthorization | null;
|
|
64
55
|
};
|
|
65
56
|
|
|
66
57
|
export async function buildGitHubRestMcpForTurn(input: {
|
|
@@ -81,7 +72,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
81
72
|
tools: input.tools,
|
|
82
73
|
localMcpServers: [],
|
|
83
74
|
connectorBindings: [],
|
|
84
|
-
authorizeCodemodeCall: null,
|
|
85
75
|
};
|
|
86
76
|
}
|
|
87
77
|
|
|
@@ -96,7 +86,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
96
86
|
tools: input.tools,
|
|
97
87
|
localMcpServers: [],
|
|
98
88
|
connectorBindings: [],
|
|
99
|
-
authorizeCodemodeCall: null,
|
|
100
89
|
};
|
|
101
90
|
}
|
|
102
91
|
const configuredIds = new Set(input.settings.mcpServers.map((server) => server.id));
|
|
@@ -112,7 +101,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
112
101
|
const localMcpServers: LocalMcpServerRegistration[] = [];
|
|
113
102
|
const toolRefs: ToolRef[] = [];
|
|
114
103
|
const connectorBindings: AttemptConnectorActionBinding[] = [];
|
|
115
|
-
const codemodeActions = createCodemodeActionLifecycle(input, connectorBindings);
|
|
116
104
|
|
|
117
105
|
if (appRepositories.length > 0) {
|
|
118
106
|
const appResolver = workspaceAppAuthorityResolver(input, appRepositories);
|
|
@@ -123,7 +111,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
123
111
|
authorityKind: "workspace_app",
|
|
124
112
|
repositories: appRepositories,
|
|
125
113
|
resolveAuthority: appResolver,
|
|
126
|
-
completeCodemodeAction: codemodeActions.complete,
|
|
127
114
|
}),
|
|
128
115
|
});
|
|
129
116
|
configs.push(internalMcpConfig(GITHUB_REST_MCP_APP_SERVER_ID, "GitHub — OpenGeni bot"));
|
|
@@ -141,7 +128,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
141
128
|
authorityKind: "personal_oauth",
|
|
142
129
|
repositories: personalRepositories,
|
|
143
130
|
resolveAuthority: personalAuthorityResolver(input),
|
|
144
|
-
completeCodemodeAction: codemodeActions.complete,
|
|
145
131
|
}),
|
|
146
132
|
});
|
|
147
133
|
configs.push(internalMcpConfig(GITHUB_REST_MCP_PERSONAL_SERVER_ID, "GitHub — My account"));
|
|
@@ -159,74 +145,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
159
145
|
tools: mergeToolRefs(input.tools, toolRefs),
|
|
160
146
|
localMcpServers,
|
|
161
147
|
connectorBindings,
|
|
162
|
-
authorizeCodemodeCall: codemodeActions.authorize,
|
|
163
|
-
};
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
function createCodemodeActionLifecycle(
|
|
167
|
-
input: {
|
|
168
|
-
db: Database;
|
|
169
|
-
accountId: string;
|
|
170
|
-
workspaceId: string;
|
|
171
|
-
sessionId: string;
|
|
172
|
-
attemptId: string;
|
|
173
|
-
turn: SessionTurnForExecution;
|
|
174
|
-
},
|
|
175
|
-
bindings: AttemptConnectorActionBinding[],
|
|
176
|
-
): {
|
|
177
|
-
authorize: AttemptToolAuthorization;
|
|
178
|
-
complete: NonNullable<
|
|
179
|
-
ConstructorParameters<typeof GitHubRestMcpServer>[0]["completeCodemodeAction"]
|
|
180
|
-
>;
|
|
181
|
-
} {
|
|
182
|
-
const identity = {
|
|
183
|
-
accountId: input.accountId,
|
|
184
|
-
workspaceId: input.workspaceId,
|
|
185
|
-
sessionId: input.sessionId,
|
|
186
|
-
turnId: input.turn.id,
|
|
187
|
-
attemptId: input.attemptId,
|
|
188
|
-
executionGeneration: input.turn.executionGeneration,
|
|
189
|
-
initiator: input.turn.initiator,
|
|
190
|
-
};
|
|
191
|
-
const executing = new Map<string, string>();
|
|
192
|
-
return {
|
|
193
|
-
authorize: async ({ call, entry }) => {
|
|
194
|
-
if (call.caller.kind !== "codemode") return;
|
|
195
|
-
const binding = bindings.find((candidate) => candidate.modelName === entry.modelName);
|
|
196
|
-
if (!binding) return;
|
|
197
|
-
const invocation = binding.call(call.operationId, call.arguments);
|
|
198
|
-
const policy = await prepareConnectorActionApproval(input.db, identity, invocation);
|
|
199
|
-
if (policy.managed && policy.decision === "ask") {
|
|
200
|
-
throw new AttemptToolApprovalRequiredError();
|
|
201
|
-
}
|
|
202
|
-
if (policy.managed && policy.decision === "block") {
|
|
203
|
-
throw new GitHubRestMutationNotExecutedError(
|
|
204
|
-
"GitHub connector action is blocked by policy",
|
|
205
|
-
);
|
|
206
|
-
}
|
|
207
|
-
const admission = await beginConnectorActionExecution(input.db, identity, invocation);
|
|
208
|
-
if (!admission.allowed) {
|
|
209
|
-
if (admission.reason === "approval_required") {
|
|
210
|
-
throw new AttemptToolApprovalRequiredError();
|
|
211
|
-
}
|
|
212
|
-
throw new GitHubRestMutationNotExecutedError(
|
|
213
|
-
`GitHub connector action was not executed: ${admission.reason}`,
|
|
214
|
-
);
|
|
215
|
-
}
|
|
216
|
-
if (admission.managed) executing.set(call.operationId, admission.requestId);
|
|
217
|
-
},
|
|
218
|
-
complete: async ({ operationId, outcome }) => {
|
|
219
|
-
const requestId = executing.get(operationId);
|
|
220
|
-
if (!requestId) return;
|
|
221
|
-
executing.delete(operationId);
|
|
222
|
-
await completeConnectorActionExecution(input.db, {
|
|
223
|
-
accountId: input.accountId,
|
|
224
|
-
workspaceId: input.workspaceId,
|
|
225
|
-
requestId,
|
|
226
|
-
attemptId: input.attemptId,
|
|
227
|
-
outcome,
|
|
228
|
-
});
|
|
229
|
-
},
|
|
230
148
|
};
|
|
231
149
|
}
|
|
232
150
|
|
|
@@ -1360,6 +1360,7 @@ export type TurnStartupPhase =
|
|
|
1360
1360
|
| "tool_optional_connect"
|
|
1361
1361
|
| "tool_attempt_catalog_build"
|
|
1362
1362
|
| "tool_attempt_catalog_persist"
|
|
1363
|
+
| "tool_workspace_gateway_catalog_build"
|
|
1363
1364
|
| "post_tool_preparation"
|
|
1364
1365
|
| "agent_construction"
|
|
1365
1366
|
| "post_agent_preparation"
|
|
@@ -1408,6 +1409,9 @@ export type TurnStartupPhase =
|
|
|
1408
1409
|
| "model_prepare_runner_before_mcp_tools"
|
|
1409
1410
|
| "model_prepare_mcp_tools_snapshot"
|
|
1410
1411
|
| "model_prepare_mcp_tools_before_input_filter"
|
|
1412
|
+
| "model_prepare_mcp_tools_before_repository_skill_discovery"
|
|
1413
|
+
| "model_prepare_repository_skill_discovery"
|
|
1414
|
+
| "model_prepare_repository_skill_discovery_before_input_filter"
|
|
1411
1415
|
| "model_prepare_input_filter_base"
|
|
1412
1416
|
| "model_prepare_input_filter_genesis"
|
|
1413
1417
|
| "model_prepare_input_filter_host"
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -17,11 +17,13 @@
|
|
|
17
17
|
import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
|
|
18
18
|
import {
|
|
19
19
|
adoptConnectedMachineSessionBackgroundCommand,
|
|
20
|
+
adoptManagedSessionBackgroundCommand,
|
|
20
21
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
21
22
|
advanceWorkspaceGeneration,
|
|
22
23
|
getRetainedProcess,
|
|
23
24
|
retainWorkspaceMutationProcess,
|
|
24
25
|
retainedProcessSettlementIdentity,
|
|
26
|
+
settleConnectedMachineSessionBackgroundCommand,
|
|
25
27
|
settleRetainedProcess,
|
|
26
28
|
verifyRetainedProcessMutationSettlement,
|
|
27
29
|
verifyWorkspaceMutationSettlement,
|
|
@@ -36,10 +38,9 @@ import {
|
|
|
36
38
|
type EnrollmentRecord,
|
|
37
39
|
type SandboxWorkspaceMutationAdmission,
|
|
38
40
|
} from "@opengeni/db";
|
|
39
|
-
import {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
} from "@opengeni/db/session-background-commands";
|
|
41
|
+
import { observeSessionBackgroundCommandCompletion } from "@opengeni/db/session-background-commands";
|
|
42
|
+
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
43
|
+
import type { OpStreamOutputFrame } from "@opengeni/runtime/sandbox";
|
|
43
44
|
import type { EventBus } from "@opengeni/events";
|
|
44
45
|
import {
|
|
45
46
|
buildSelfhostedBackendSession,
|
|
@@ -515,10 +516,6 @@ function afterPersistableHomeMutation(
|
|
|
515
516
|
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
516
517
|
operation: op,
|
|
517
518
|
providerBinding: boundAdmission.providerBinding ?? null,
|
|
518
|
-
backgroundCommand: {
|
|
519
|
-
commandId: retainedProcess.id,
|
|
520
|
-
command: op,
|
|
521
|
-
},
|
|
522
519
|
owner: {
|
|
523
520
|
kind: "turn",
|
|
524
521
|
turnId: fence.turnId,
|
|
@@ -642,6 +639,76 @@ function afterRetainedProcessMutation(
|
|
|
642
639
|
};
|
|
643
640
|
}
|
|
644
641
|
|
|
642
|
+
export function captureConnectedCommandOutput(
|
|
643
|
+
db: Database,
|
|
644
|
+
ids: { accountId: string; workspaceId: string; sessionId: string },
|
|
645
|
+
bus?: EventBus,
|
|
646
|
+
) {
|
|
647
|
+
return async (commandId: string, frames: OpStreamOutputFrame[]) => {
|
|
648
|
+
for (const frame of frames) {
|
|
649
|
+
const events = await appendSessionCommandOutput(db, {
|
|
650
|
+
...ids,
|
|
651
|
+
commandId,
|
|
652
|
+
chunkId: `op-frame:${frame.sequence}`,
|
|
653
|
+
stream: frame.stream,
|
|
654
|
+
chunk: frame.chunk,
|
|
655
|
+
});
|
|
656
|
+
if (events.length && bus)
|
|
657
|
+
await bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => undefined);
|
|
658
|
+
}
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
function captureRetainedProcessOutputForTurn(
|
|
663
|
+
services: RoutingWiringServices,
|
|
664
|
+
ids: RoutingWiringIds,
|
|
665
|
+
) {
|
|
666
|
+
const accountId = ids.workspaceMutationFence?.accountId;
|
|
667
|
+
if (!accountId) return undefined;
|
|
668
|
+
return async ({
|
|
669
|
+
process,
|
|
670
|
+
chunkId,
|
|
671
|
+
chunk,
|
|
672
|
+
stream,
|
|
673
|
+
streamFidelity,
|
|
674
|
+
}: {
|
|
675
|
+
process: RoutingRetainedProcess;
|
|
676
|
+
chunkId: string;
|
|
677
|
+
chunk: string;
|
|
678
|
+
stream: "stdout" | "stderr";
|
|
679
|
+
streamFidelity: "separate" | "merged";
|
|
680
|
+
}) => {
|
|
681
|
+
const events = await appendSessionCommandOutput(services.db, {
|
|
682
|
+
accountId,
|
|
683
|
+
workspaceId: ids.workspaceId,
|
|
684
|
+
sessionId: ids.sessionId,
|
|
685
|
+
commandId: process.id,
|
|
686
|
+
chunkId,
|
|
687
|
+
chunk,
|
|
688
|
+
stream,
|
|
689
|
+
streamFidelity,
|
|
690
|
+
});
|
|
691
|
+
if (events.length && services.bus)
|
|
692
|
+
await services.bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => undefined);
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
function observeRetainedProcessTerminalForTurn(
|
|
697
|
+
services: RoutingWiringServices,
|
|
698
|
+
ids: RoutingWiringIds,
|
|
699
|
+
) {
|
|
700
|
+
const fence = ids.workspaceMutationFence;
|
|
701
|
+
if (!fence) return undefined;
|
|
702
|
+
return async ({ process }: { process: RoutingRetainedProcess }) => {
|
|
703
|
+
await observeSessionBackgroundCommandCompletion(services.db, {
|
|
704
|
+
accountId: fence.accountId,
|
|
705
|
+
workspaceId: ids.workspaceId,
|
|
706
|
+
sessionId: ids.sessionId,
|
|
707
|
+
commandId: process.id,
|
|
708
|
+
});
|
|
709
|
+
};
|
|
710
|
+
}
|
|
711
|
+
|
|
645
712
|
function settleRetainedProcessForTurn(
|
|
646
713
|
services: RoutingWiringServices,
|
|
647
714
|
ids: RoutingWiringIds,
|
|
@@ -686,14 +753,50 @@ function settleRetainedProcessForTurn(
|
|
|
686
753
|
if (settlement.process.state === "active") {
|
|
687
754
|
throw new Error("Retained-process settlement returned an active durable process");
|
|
688
755
|
}
|
|
689
|
-
|
|
756
|
+
if (settlement.backgroundCommandEvents.length > 0 && services.bus) {
|
|
757
|
+
await services.bus
|
|
758
|
+
.publish(ids.workspaceId, ids.sessionId, settlement.backgroundCommandEvents)
|
|
759
|
+
.catch(() => undefined);
|
|
760
|
+
}
|
|
761
|
+
};
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
function adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
765
|
+
services: RoutingWiringServices,
|
|
766
|
+
ids: RoutingWiringIds,
|
|
767
|
+
):
|
|
768
|
+
| ((input: { backend: ResolvedActiveBackend; process: RoutingRetainedProcess }) => Promise<void>)
|
|
769
|
+
| undefined {
|
|
770
|
+
const fence = ids.workspaceMutationFence;
|
|
771
|
+
if (!fence) return undefined;
|
|
772
|
+
return async ({ backend, process }) => {
|
|
773
|
+
const durable = await getRetainedProcess(services.db, {
|
|
774
|
+
workspaceId: ids.workspaceId,
|
|
775
|
+
sessionId: ids.sessionId,
|
|
776
|
+
processId: process.id,
|
|
777
|
+
});
|
|
778
|
+
if (
|
|
779
|
+
!durable ||
|
|
780
|
+
durable.providerSessionId !== process.providerSessionId ||
|
|
781
|
+
durable.providerBackend !== backend.kind ||
|
|
782
|
+
durable.providerInstanceId !== backend.providerInstanceId ||
|
|
783
|
+
durable.leaseEpoch !== backend.leaseEpoch ||
|
|
784
|
+
durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
|
|
785
|
+
durable.routeTargetId !== backend.sandboxId ||
|
|
786
|
+
durable.routeEpoch !== backend.activeEpoch
|
|
787
|
+
) {
|
|
788
|
+
throw new Error("Background command adoption lost its exact retained-process identity");
|
|
789
|
+
}
|
|
790
|
+
await adoptManagedSessionBackgroundCommand(services.db, {
|
|
690
791
|
accountId: fence.accountId,
|
|
691
792
|
workspaceId: ids.workspaceId,
|
|
692
793
|
sessionId: ids.sessionId,
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
794
|
+
turnId: fence.turnId,
|
|
795
|
+
executionGeneration: fence.executionGeneration,
|
|
796
|
+
attemptId: fence.attemptId,
|
|
797
|
+
processId: process.id,
|
|
798
|
+
expected: retainedProcessSettlementIdentity(durable),
|
|
799
|
+
command: "execCommand",
|
|
697
800
|
});
|
|
698
801
|
};
|
|
699
802
|
}
|
|
@@ -736,6 +839,10 @@ export function wrapTurnBoxWithRouting(
|
|
|
736
839
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
737
840
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
738
841
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
842
|
+
const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
843
|
+
services,
|
|
844
|
+
ids,
|
|
845
|
+
);
|
|
739
846
|
const resolver = makeActiveBackendResolver({
|
|
740
847
|
workspaceId: ids.workspaceId,
|
|
741
848
|
defaultBackend: established.session as RoutableBackendSession,
|
|
@@ -794,7 +901,7 @@ export function wrapTurnBoxWithRouting(
|
|
|
794
901
|
return { commandId: adopted.id };
|
|
795
902
|
},
|
|
796
903
|
selfhostedSettleBackgroundCommand: async (command) => {
|
|
797
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
904
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
798
905
|
accountId: backgroundCommandAccountId,
|
|
799
906
|
workspaceId: ids.workspaceId,
|
|
800
907
|
sessionId: ids.sessionId,
|
|
@@ -806,8 +913,23 @@ export function wrapTurnBoxWithRouting(
|
|
|
806
913
|
outcome: command.outcome,
|
|
807
914
|
exitCode: command.exitCode,
|
|
808
915
|
reason: command.reason,
|
|
916
|
+
...(command.failure ? { failure: command.failure } : {}),
|
|
809
917
|
});
|
|
918
|
+
if (settlement && bus) {
|
|
919
|
+
await bus
|
|
920
|
+
.publish(ids.workspaceId, ids.sessionId, settlement.events)
|
|
921
|
+
.catch(() => undefined);
|
|
922
|
+
}
|
|
810
923
|
},
|
|
924
|
+
selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
|
|
925
|
+
db,
|
|
926
|
+
{
|
|
927
|
+
accountId: backgroundCommandAccountId,
|
|
928
|
+
workspaceId: ids.workspaceId,
|
|
929
|
+
sessionId: ids.sessionId,
|
|
930
|
+
},
|
|
931
|
+
bus,
|
|
932
|
+
),
|
|
811
933
|
}
|
|
812
934
|
: {}),
|
|
813
935
|
// The turn's declared environment → a selfhosted swap target's manifest, so the
|
|
@@ -892,6 +1014,13 @@ export function wrapTurnBoxWithRouting(
|
|
|
892
1014
|
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|
|
893
1015
|
...(afterProcessMutation ? { afterProcessMutation } : {}),
|
|
894
1016
|
...(settleProcess ? { settleProcess } : {}),
|
|
1017
|
+
...(ids.workspaceMutationFence
|
|
1018
|
+
? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
|
|
1019
|
+
: {}),
|
|
1020
|
+
...(ids.workspaceMutationFence
|
|
1021
|
+
? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids)! }
|
|
1022
|
+
: {}),
|
|
1023
|
+
...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
|
|
895
1024
|
...(ids.homeLease
|
|
896
1025
|
? {
|
|
897
1026
|
onDefaultBackendError: async ({
|
|
@@ -970,6 +1099,10 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
970
1099
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
971
1100
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
972
1101
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
1102
|
+
const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
1103
|
+
services,
|
|
1104
|
+
ids,
|
|
1105
|
+
);
|
|
973
1106
|
const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
|
|
974
1107
|
const defaultSupportsPty = defaultBackend
|
|
975
1108
|
? selectBackend(defaultBackend).capabilities.Terminal.pty
|
|
@@ -1038,7 +1171,7 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
1038
1171
|
return { commandId: adopted.id };
|
|
1039
1172
|
},
|
|
1040
1173
|
selfhostedSettleBackgroundCommand: async (command) => {
|
|
1041
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
1174
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
1042
1175
|
accountId: backgroundCommandAccountId,
|
|
1043
1176
|
workspaceId: ids.workspaceId,
|
|
1044
1177
|
sessionId: ids.sessionId,
|
|
@@ -1050,8 +1183,23 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
1050
1183
|
outcome: command.outcome,
|
|
1051
1184
|
exitCode: command.exitCode,
|
|
1052
1185
|
reason: command.reason,
|
|
1186
|
+
...(command.failure ? { failure: command.failure } : {}),
|
|
1053
1187
|
});
|
|
1188
|
+
if (settlement && bus) {
|
|
1189
|
+
await bus
|
|
1190
|
+
.publish(ids.workspaceId, ids.sessionId, settlement.events)
|
|
1191
|
+
.catch(() => undefined);
|
|
1192
|
+
}
|
|
1054
1193
|
},
|
|
1194
|
+
selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
|
|
1195
|
+
db,
|
|
1196
|
+
{
|
|
1197
|
+
accountId: backgroundCommandAccountId,
|
|
1198
|
+
workspaceId: ids.workspaceId,
|
|
1199
|
+
sessionId: ids.sessionId,
|
|
1200
|
+
},
|
|
1201
|
+
bus,
|
|
1202
|
+
),
|
|
1055
1203
|
}
|
|
1056
1204
|
: {}),
|
|
1057
1205
|
...(ids.environment !== undefined ? { environment: ids.environment } : {}),
|
|
@@ -1105,6 +1253,13 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
1105
1253
|
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|
|
1106
1254
|
...(afterProcessMutation ? { afterProcessMutation } : {}),
|
|
1107
1255
|
...(settleProcess ? { settleProcess } : {}),
|
|
1256
|
+
...(ids.workspaceMutationFence
|
|
1257
|
+
? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
|
|
1258
|
+
: {}),
|
|
1259
|
+
...(ids.workspaceMutationFence
|
|
1260
|
+
? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids)! }
|
|
1261
|
+
: {}),
|
|
1262
|
+
...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
|
|
1108
1263
|
...(args.homeLeaseIdentity
|
|
1109
1264
|
? {
|
|
1110
1265
|
onDefaultBackendError: async ({
|
|
@@ -1382,8 +1537,17 @@ export async function establishSelfhostedTurnSession(
|
|
|
1382
1537
|
},
|
|
1383
1538
|
}
|
|
1384
1539
|
: {}),
|
|
1540
|
+
captureBackgroundCommandOutput: captureConnectedCommandOutput(
|
|
1541
|
+
db,
|
|
1542
|
+
{
|
|
1543
|
+
accountId: args.accountId,
|
|
1544
|
+
workspaceId: args.workspaceId,
|
|
1545
|
+
sessionId: args.sessionId,
|
|
1546
|
+
},
|
|
1547
|
+
bus,
|
|
1548
|
+
),
|
|
1385
1549
|
settleBackgroundCommand: async (command) => {
|
|
1386
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
1550
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
1387
1551
|
accountId: args.accountId,
|
|
1388
1552
|
workspaceId: args.workspaceId,
|
|
1389
1553
|
sessionId: args.sessionId,
|
|
@@ -1395,7 +1559,13 @@ export async function establishSelfhostedTurnSession(
|
|
|
1395
1559
|
outcome: command.outcome,
|
|
1396
1560
|
exitCode: command.exitCode,
|
|
1397
1561
|
reason: command.reason,
|
|
1562
|
+
...(command.failure ? { failure: command.failure } : {}),
|
|
1398
1563
|
});
|
|
1564
|
+
if (settlement && bus) {
|
|
1565
|
+
await bus
|
|
1566
|
+
.publish(args.workspaceId, args.sessionId, settlement.events)
|
|
1567
|
+
.catch(() => undefined);
|
|
1568
|
+
}
|
|
1399
1569
|
},
|
|
1400
1570
|
// Meter every control op (out-of-band telemetry) — no-op when unwired.
|
|
1401
1571
|
...(onOp !== undefined ? { onOp } : {}),
|
package/src/workflows/session.ts
CHANGED
|
@@ -95,6 +95,14 @@ export function deferredResultMayContinue(entryWakeups: number, currentWakeups:
|
|
|
95
95
|
return currentWakeups !== entryWakeups;
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
+
/** A typed activity cancellation is recoverable unless its exact-attempt
|
|
99
|
+
* redispatch budget was atomically exhausted. */
|
|
100
|
+
export function cancelledAttemptRecoveryMayContinue(
|
|
101
|
+
action: activities.RecoverDispatchResult["action"],
|
|
102
|
+
): boolean {
|
|
103
|
+
return action !== "exceeded";
|
|
104
|
+
}
|
|
105
|
+
|
|
98
106
|
/**
|
|
99
107
|
* Bound repeated failures that happen before an attempt row exists. The
|
|
100
108
|
* activity mirrors this deterministic delay into the durable wake outbox, so
|
|
@@ -340,8 +348,13 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
340
348
|
// every new run records v2 and uses the activity-owned receipt contract.
|
|
341
349
|
const receiptGatedCancellation = patched("session-attempt-quiescence-v2");
|
|
342
350
|
const writerSetQuiescenceRecovery = patched("session-attempt-writer-set-quiescence-v1");
|
|
351
|
+
const preserveQuiescenceWake = patched("session-quiescence-reconciliation-wake-v1");
|
|
343
352
|
const staleControlSignalIsOnlyWakeHint = patched("session-control-stale-wake-v1");
|
|
344
353
|
const unclaimedAttemptRecovery = patched("session-unclaimed-attempt-recovery-v1");
|
|
354
|
+
// PR #2208 changed a typed-cancelled result from a plain re-peek into a
|
|
355
|
+
// recoverDispatch activity. Version that new command so histories which
|
|
356
|
+
// already recorded the legacy re-peek remain deterministic on replay.
|
|
357
|
+
const cancelledAttemptRecovery = patched("session-cancelled-attempt-recovery-v1");
|
|
345
358
|
const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue, receiptGatedCancellation);
|
|
346
359
|
let approvalWakeups = 0;
|
|
347
360
|
let interruptionWakeups = 0;
|
|
@@ -572,6 +585,8 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
572
585
|
continue;
|
|
573
586
|
}
|
|
574
587
|
if (peek.kind === "cancellation-wait") {
|
|
588
|
+
// The receipt wake can arrive while reconciliation returns an older pending result.
|
|
589
|
+
const beforeReconciliationSignalVersion = signalVersion;
|
|
575
590
|
if (writerSetQuiescenceRecovery) {
|
|
576
591
|
const reconciliation = await activity.reconcileSessionAttemptQuiescence({
|
|
577
592
|
accountId: input.accountId,
|
|
@@ -588,7 +603,9 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
588
603
|
// genuinely slow, close this workflow run rather than consuming a turn
|
|
589
604
|
// slot or churning control activities; the outbox uses signalWithStart to
|
|
590
605
|
// restart this exact workflow after the receipt commits.
|
|
591
|
-
const seenSignalVersion =
|
|
606
|
+
const seenSignalVersion = preserveQuiescenceWake
|
|
607
|
+
? beforeReconciliationSignalVersion
|
|
608
|
+
: signalVersion;
|
|
592
609
|
const woke = await condition(() => signalVersion !== seenSignalVersion, "5s");
|
|
593
610
|
if (woke) continue;
|
|
594
611
|
// Close only against the same signal snapshot. A proof signal accepted
|
|
@@ -601,6 +618,29 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
601
618
|
await waitForProviderCapacity(peek.ref);
|
|
602
619
|
continue;
|
|
603
620
|
}
|
|
621
|
+
if (peek.kind === "sandbox-lifecycle-wait") {
|
|
622
|
+
// The recovering turn carries an exact group/epoch marker. Do not reserve
|
|
623
|
+
// another turn-worker slot while the same draining lease still owns that
|
|
624
|
+
// transition. The draining->cold commit enqueues a durable workflow wake
|
|
625
|
+
// for this exact marker; keep only the standard five-second close-race
|
|
626
|
+
// window so a signal or an already-completed transition is not lost.
|
|
627
|
+
const seenSignalVersion = signalVersion;
|
|
628
|
+
const woke = await condition(() => signalVersion !== seenSignalVersion, "5s");
|
|
629
|
+
if (woke) continue;
|
|
630
|
+
const finalPeek = await activity.peekSessionWork({
|
|
631
|
+
workspaceId: input.workspaceId,
|
|
632
|
+
sessionId: input.sessionId,
|
|
633
|
+
});
|
|
634
|
+
if (
|
|
635
|
+
finalPeek.kind !== "sandbox-lifecycle-wait" ||
|
|
636
|
+
finalPeek.ref.sandboxGroupId !== peek.ref.sandboxGroupId ||
|
|
637
|
+
finalPeek.ref.leaseEpoch !== peek.ref.leaseEpoch
|
|
638
|
+
) {
|
|
639
|
+
continue;
|
|
640
|
+
}
|
|
641
|
+
if (signalVersion !== closeSignalVersion) continue;
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
604
644
|
if (peek.kind === "approval-wait") {
|
|
605
645
|
const seenApprovalWakeups = approvalWakeups;
|
|
606
646
|
const seenWakeups = wakeups;
|
|
@@ -644,6 +684,48 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
644
684
|
}
|
|
645
685
|
continue;
|
|
646
686
|
}
|
|
687
|
+
if (peek.kind === "input-wait") {
|
|
688
|
+
const settlement = await activity.settleSessionInputWait({
|
|
689
|
+
accountId: input.accountId,
|
|
690
|
+
workspaceId: input.workspaceId,
|
|
691
|
+
sessionId: input.sessionId,
|
|
692
|
+
waitTurnId: peek.waitTurnId,
|
|
693
|
+
disposition: peek.disposition,
|
|
694
|
+
});
|
|
695
|
+
if (settlement.action !== "held") continue;
|
|
696
|
+
|
|
697
|
+
// The durable outbox owns the long deadline. Keep this workflow run open
|
|
698
|
+
// only for the same bounded close-race window used by ordinary idle; any
|
|
699
|
+
// signal is a hint to re-peek PostgreSQL truth.
|
|
700
|
+
const seenWakeups = wakeups;
|
|
701
|
+
const seenApprovalWakeups = approvalWakeups;
|
|
702
|
+
const seenInterruptionWakeups = interruptionWakeups;
|
|
703
|
+
const woke = await condition(
|
|
704
|
+
() =>
|
|
705
|
+
interruptionWakeups !== seenInterruptionWakeups ||
|
|
706
|
+
wakeups !== seenWakeups ||
|
|
707
|
+
approvalWakeups !== seenApprovalWakeups,
|
|
708
|
+
"5s",
|
|
709
|
+
);
|
|
710
|
+
if (woke) continue;
|
|
711
|
+
const finalPeek = await activity.peekSessionWork({
|
|
712
|
+
workspaceId: input.workspaceId,
|
|
713
|
+
sessionId: input.sessionId,
|
|
714
|
+
});
|
|
715
|
+
if (
|
|
716
|
+
finalPeek.kind !== "input-wait" ||
|
|
717
|
+
finalPeek.disposition !== "held" ||
|
|
718
|
+
finalPeek.waitTurnId !== peek.waitTurnId
|
|
719
|
+
) {
|
|
720
|
+
continue;
|
|
721
|
+
}
|
|
722
|
+
await activity.markSessionIdle({
|
|
723
|
+
workspaceId: input.workspaceId,
|
|
724
|
+
sessionId: input.sessionId,
|
|
725
|
+
});
|
|
726
|
+
if (signalVersion !== closeSignalVersion) continue;
|
|
727
|
+
return;
|
|
728
|
+
}
|
|
647
729
|
if (peek.kind === "idle") {
|
|
648
730
|
let continuation: activities.MaybeContinueGoalResult = { action: "none" };
|
|
649
731
|
try {
|
|
@@ -663,11 +745,10 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
663
745
|
});
|
|
664
746
|
}
|
|
665
747
|
if (continuation.action === "continue" || continuation.action === "queue") continue;
|
|
666
|
-
// `none`, `paused`,
|
|
667
|
-
//
|
|
668
|
-
//
|
|
669
|
-
//
|
|
670
|
-
// restarts the workflow. No Temporal timer is used for pacing.
|
|
748
|
+
// `none`, `paused`, and `deferred` all close this run below. A deferred
|
|
749
|
+
// idle-backoff goal keeps its durable obligation armed; the delayed
|
|
750
|
+
// wake-outbox row at the pacing deadline or any producer signal restarts
|
|
751
|
+
// the workflow. No Temporal timer is used for pacing.
|
|
671
752
|
const seenWakeups = wakeups;
|
|
672
753
|
const seenApprovalWakeups = approvalWakeups;
|
|
673
754
|
const seenInterruptionWakeups = interruptionWakeups;
|
|
@@ -1052,8 +1133,33 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
1052
1133
|
return true;
|
|
1053
1134
|
}
|
|
1054
1135
|
|
|
1055
|
-
if (outcome.result.status === "failed"
|
|
1056
|
-
return
|
|
1136
|
+
if (outcome.result.status === "failed") {
|
|
1137
|
+
return false;
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
if (outcome.result.status === "cancelled") {
|
|
1141
|
+
// Histories created before session-cancelled-attempt-recovery-v1 must
|
|
1142
|
+
// preserve the old command sequence. Their durable state is still safe:
|
|
1143
|
+
// operator recovery or a later fresh workflow run can close a stranded
|
|
1144
|
+
// owner, while new histories use the bounded exact-attempt transaction.
|
|
1145
|
+
if (!cancelledAttemptRecovery) return true;
|
|
1146
|
+
// A typed cancellation is normally observed through the control-signal
|
|
1147
|
+
// branch above, after Pause/Steer has durably fenced the attempt. A
|
|
1148
|
+
// worker can nevertheless return the same shape after a shutdown or a
|
|
1149
|
+
// stale settlement race without closing its attempt row. Blindly
|
|
1150
|
+
// re-peeking then retries forever against `turn.status = running` and
|
|
1151
|
+
// strands every later prompt. Reconcile the exact attempt through the
|
|
1152
|
+
// same bounded, generation-fenced redispatch transaction used after an
|
|
1153
|
+
// activity heartbeat loss. If the attempt actually settled meanwhile,
|
|
1154
|
+
// the transaction is a stale no-op and the next peek observes truth.
|
|
1155
|
+
const recovery = await activity.recoverDispatch({
|
|
1156
|
+
accountId,
|
|
1157
|
+
workspaceId,
|
|
1158
|
+
sessionId,
|
|
1159
|
+
attemptId: outcome.result.attemptId,
|
|
1160
|
+
timeoutType: "HEARTBEAT",
|
|
1161
|
+
});
|
|
1162
|
+
return cancelledAttemptRecoveryMayContinue(recovery.action);
|
|
1057
1163
|
}
|
|
1058
1164
|
|
|
1059
1165
|
if (outcome.result.capacityWait) {
|