@opengeni/worker-bundle 2.2.0 → 2.2.1-canary.36919050519001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +3 -1
- package/dist/activities/agent-turn/admission.d.ts +24 -3
- package/dist/activities/agent-turn/agent-build.d.ts +1 -0
- package/dist/activities/agent-turn/claude-usage-observer.d.ts +13 -1
- package/dist/activities/agent-turn/compaction-prep.d.ts +20 -0
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/governance-model.d.ts +6 -0
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +2 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/session-title.d.ts +3 -1
- package/dist/activities/agent-turn/stream-attempt.d.ts +1 -0
- package/dist/activities/agent-turn/turn-context.d.ts +1 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +29 -0
- package/dist/activities/goals.d.ts +8 -0
- package/dist/activities/run-credentials.d.ts +4 -2
- package/dist/activities/types.d.ts +1 -1
- package/dist/{activities-control-MHNA3CYV.js → activities-control-55ECL47G.js} +403 -116
- package/dist/activities-control-55ECL47G.js.map +1 -0
- package/dist/activities-control.d.ts +30 -0
- package/dist/{activities-turn-FQ5OO2WW.js → activities-turn-T7Y3BWYO.js} +678 -246
- package/dist/activities-turn-T7Y3BWYO.js.map +1 -0
- package/dist/activities.d.ts +30 -0
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-RZAAGRFQ.js → chunk-OWNLATJL.js} +189 -80
- package/dist/chunk-OWNLATJL.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +45 -23
- package/dist/index.js.map +1 -1
- package/dist/sandbox-resume.d.ts +4 -1
- package/dist/workflow-bundle.js +72 -10
- package/dist/workflows/browser-deadline-checkpoint.d.ts +9 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +21 -20
- package/src/activities/agent-run-admission.ts +17 -1
- package/src/activities/agent-turn/admission.ts +64 -1
- package/src/activities/agent-turn/agent-build.ts +24 -3
- package/src/activities/agent-turn/claim.ts +56 -14
- package/src/activities/agent-turn/claude-usage-observer.ts +33 -1
- package/src/activities/agent-turn/compaction-prep.ts +67 -16
- package/src/activities/agent-turn/errors.ts +55 -0
- package/src/activities/agent-turn/failure-settlement.ts +26 -6
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/governance-model.ts +12 -0
- package/src/activities/agent-turn/model-call-admission.ts +94 -0
- package/src/activities/agent-turn/run-credentials.ts +15 -3
- package/src/activities/agent-turn/run.ts +43 -5
- package/src/activities/agent-turn/sandbox-establish.ts +9 -5
- package/src/activities/agent-turn/session-title.ts +8 -2
- package/src/activities/agent-turn/stream-attempt.ts +144 -38
- package/src/activities/agent-turn/tool-environment.ts +51 -39
- package/src/activities/agent-turn/turn-context.ts +1 -0
- package/src/activities/automations.ts +55 -3
- package/src/activities/browser-deadline-checkpoint.ts +279 -0
- package/src/activities/capabilities.ts +52 -14
- package/src/activities/goals.ts +67 -95
- package/src/activities/knowledge-indexing.ts +29 -0
- package/src/activities/open-suffix-resume.ts +2 -1
- package/src/activities/run-credentials.ts +22 -16
- package/src/activities/scheduled-tasks.ts +83 -13
- package/src/activities/types.ts +1 -0
- package/src/activities/workspace-credential-provider.ts +12 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/index.ts +53 -25
- package/src/sandbox-resume.ts +23 -8
- package/src/sandbox-routing.ts +4 -2
- package/src/workflows/browser-deadline-checkpoint.ts +50 -0
- package/src/workflows.ts +4 -0
- package/dist/activities-control-MHNA3CYV.js.map +0 -1
- package/dist/activities-turn-FQ5OO2WW.js.map +0 -1
- package/dist/chunk-RZAAGRFQ.js.map +0 -1
|
@@ -2,6 +2,7 @@ import type { Agent } from "@openai/agents";
|
|
|
2
2
|
import {
|
|
3
3
|
OPEN_SUFFIX_RUN_STATE_BLOB,
|
|
4
4
|
approvalIdentifier,
|
|
5
|
+
withPublicApprovalFields,
|
|
5
6
|
type HumanInputResponse,
|
|
6
7
|
type SessionEvent,
|
|
7
8
|
} from "@opengeni/contracts";
|
|
@@ -380,7 +381,7 @@ export async function settleOpenSuffixResumeIfNeeded(input: {
|
|
|
380
381
|
? [
|
|
381
382
|
{
|
|
382
383
|
type: "session.requiresAction" as const,
|
|
383
|
-
payload: { approvals: requiresActionApprovals },
|
|
384
|
+
payload: { approvals: requiresActionApprovals.map(withPublicApprovalFields) },
|
|
384
385
|
},
|
|
385
386
|
]
|
|
386
387
|
: []),
|
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
RunCredentialsRequest,
|
|
6
6
|
Session,
|
|
7
7
|
SessionTurn,
|
|
8
|
+
ToolRef,
|
|
8
9
|
} from "@opengeni/contracts";
|
|
9
10
|
import type { Settings } from "@opengeni/config";
|
|
10
11
|
import { getSessionRootId, type Database } from "@opengeni/db";
|
|
@@ -30,6 +31,8 @@ export type RunCredentialResolutionContext = {
|
|
|
30
31
|
initiatingHumanSubjectId?: string | null;
|
|
31
32
|
/** Installed in-process API routes must never enter a product callback. */
|
|
32
33
|
localMcpServerIds?: readonly string[];
|
|
34
|
+
/** Exact execution selection, resolved once at the turn policy boundary. */
|
|
35
|
+
effectiveTools: readonly ToolRef[];
|
|
33
36
|
};
|
|
34
37
|
|
|
35
38
|
export type BoundRunCredentialResolver = {
|
|
@@ -42,7 +45,7 @@ export type BoundRunCredentialResolver = {
|
|
|
42
45
|
export function buildRunCredentialsRequest(
|
|
43
46
|
input: Omit<
|
|
44
47
|
RunCredentialResolutionContext,
|
|
45
|
-
"db" | "connectionCredentials" | "settings" | "initiatingHumanSubjectId"
|
|
48
|
+
"db" | "connectionCredentials" | "settings" | "initiatingHumanSubjectId" | "effectiveTools"
|
|
46
49
|
> & {
|
|
47
50
|
rootSessionId: string;
|
|
48
51
|
purpose: "provision" | "renewal";
|
|
@@ -125,7 +128,7 @@ export async function bindRunCredentialResolver(
|
|
|
125
128
|
mcpServers: selectedSessionRemoteMcpTargets(
|
|
126
129
|
input.settings,
|
|
127
130
|
input.session.mcpServers ?? [],
|
|
128
|
-
input.
|
|
131
|
+
input.effectiveTools,
|
|
129
132
|
(input.localMcpServerIds ?? []).map((id) => ({ id })),
|
|
130
133
|
),
|
|
131
134
|
},
|
|
@@ -146,22 +149,25 @@ export async function bindRunCredentialResolver(
|
|
|
146
149
|
workspaceId: input.workspaceId,
|
|
147
150
|
sessionId: input.session.id,
|
|
148
151
|
};
|
|
152
|
+
// A renewable resolver belongs to one admitted turn, even if its caller
|
|
153
|
+
// subsequently replaces or mutates an in-memory session/turn projection.
|
|
154
|
+
const frozenRequest = structuredClone(
|
|
155
|
+
buildRunCredentialsRequest({
|
|
156
|
+
accountId: input.accountId,
|
|
157
|
+
workspaceId: input.workspaceId,
|
|
158
|
+
session: input.session,
|
|
159
|
+
turn: input.turn,
|
|
160
|
+
attemptId: input.attemptId,
|
|
161
|
+
effectiveSandboxBackend: input.effectiveSandboxBackend,
|
|
162
|
+
variableSet: input.variableSet,
|
|
163
|
+
rootSessionId,
|
|
164
|
+
purpose: "provision",
|
|
165
|
+
forceRefresh: false,
|
|
166
|
+
}),
|
|
167
|
+
);
|
|
149
168
|
return {
|
|
150
169
|
resolve: async ({ purpose, forceRefresh }) => {
|
|
151
|
-
const resolution = await resolver(
|
|
152
|
-
buildRunCredentialsRequest({
|
|
153
|
-
accountId: input.accountId,
|
|
154
|
-
workspaceId: input.workspaceId,
|
|
155
|
-
session: input.session,
|
|
156
|
-
turn: input.turn,
|
|
157
|
-
attemptId: input.attemptId,
|
|
158
|
-
effectiveSandboxBackend: input.effectiveSandboxBackend,
|
|
159
|
-
variableSet: input.variableSet,
|
|
160
|
-
rootSessionId,
|
|
161
|
-
purpose,
|
|
162
|
-
forceRefresh,
|
|
163
|
-
}),
|
|
164
|
-
);
|
|
170
|
+
const resolution = await resolver({ ...frozenRequest, purpose, forceRefresh });
|
|
165
171
|
return normalizeRunCredentialsResolution(resolution, scope);
|
|
166
172
|
},
|
|
167
173
|
};
|
|
@@ -15,6 +15,8 @@ import {
|
|
|
15
15
|
TurnExecutionPolicyV1,
|
|
16
16
|
type ScheduledTask,
|
|
17
17
|
type ScheduledTaskRun,
|
|
18
|
+
AgentConfigError,
|
|
19
|
+
type ResolvedAgentConfig,
|
|
18
20
|
} from "@opengeni/contracts";
|
|
19
21
|
import {
|
|
20
22
|
openGeniSlackBotMetadata,
|
|
@@ -33,6 +35,9 @@ import {
|
|
|
33
35
|
settingsWithEnabledCapabilityMcpServers,
|
|
34
36
|
settingsWithSessionMcpServerMetadata,
|
|
35
37
|
swapActiveSandbox,
|
|
38
|
+
agentConfigMayResolve,
|
|
39
|
+
applySessionAgentConfigWriteThrough,
|
|
40
|
+
resolveSessionAgentConfigForCreate,
|
|
36
41
|
} from "@opengeni/core";
|
|
37
42
|
import {
|
|
38
43
|
appendSessionEvents,
|
|
@@ -62,6 +67,7 @@ import {
|
|
|
62
67
|
getNestedAgentDepthDeploymentPolicy,
|
|
63
68
|
getSessionByCreateIdempotencyKey,
|
|
64
69
|
getSessionCreationExecutionPolicy,
|
|
70
|
+
metadataWithAgentConfigCreateIdentity,
|
|
65
71
|
getVariableSet,
|
|
66
72
|
isCodexBilledModel,
|
|
67
73
|
initializeSessionStartAtomically,
|
|
@@ -501,7 +507,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
501
507
|
settings.openaiReasoningEffort;
|
|
502
508
|
let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
503
509
|
let sandboxOs: "linux" | "macos" | "windows" = "linux";
|
|
504
|
-
|
|
510
|
+
let taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
505
511
|
// A task created by a live agent attempt froze its creator's effective
|
|
506
512
|
// first-party selection, permission set, and session access policy;
|
|
507
513
|
// its generated sessions inherit that boundary under today's
|
|
@@ -510,13 +516,55 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
510
516
|
const creatorPolicy = await getScheduledTaskCreatorPolicy(db, task.workspaceId, task.id);
|
|
511
517
|
// A person-chosen Slack channel adds only the two bot posting tools;
|
|
512
518
|
// they post nowhere else, whatever the creator's own selection was.
|
|
513
|
-
|
|
519
|
+
let firstPartyMcpTools = withScheduledSlackBotPostingTools(
|
|
514
520
|
creatorPolicy?.firstPartyMcpTools
|
|
515
521
|
? allowedFirstPartyMcpToolsForSession(settings, creatorPolicy.firstPartyMcpTools)
|
|
516
522
|
: resolveFirstPartyMcpToolPolicy(settings).default,
|
|
517
523
|
task.agentConfig,
|
|
518
524
|
resolveFirstPartyMcpToolPolicy(settings).allowed,
|
|
519
525
|
);
|
|
526
|
+
// Agent configuration for generated sessions (null = legacy). The
|
|
527
|
+
// scheduled baseline above (creator policy or deployment default) is
|
|
528
|
+
// what "all" reproduces; capabilities only narrow it. Frozen into the
|
|
529
|
+
// accepted execution so retries and recovery never resolve again.
|
|
530
|
+
let resolvedAgentConfig: ResolvedAgentConfig | null = null;
|
|
531
|
+
let resolvedAgentInstructions: string | undefined;
|
|
532
|
+
try {
|
|
533
|
+
const agentResolution = resolveSessionAgentConfigForCreate({
|
|
534
|
+
settings,
|
|
535
|
+
creator: "scheduled",
|
|
536
|
+
request: task.agentConfig.agent,
|
|
537
|
+
instructions: undefined,
|
|
538
|
+
workspaceSettings: agentConfigMayResolve(settings, task.agentConfig.agent)
|
|
539
|
+
? (await requireWorkspace(db, task.workspaceId)).settings
|
|
540
|
+
: {},
|
|
541
|
+
parent: null,
|
|
542
|
+
goal: task.agentConfig.goal !== undefined,
|
|
543
|
+
});
|
|
544
|
+
resolvedAgentConfig = agentResolution.config;
|
|
545
|
+
resolvedAgentInstructions = agentResolution.instructions;
|
|
546
|
+
const written = applySessionAgentConfigWriteThrough({
|
|
547
|
+
config: resolvedAgentConfig,
|
|
548
|
+
firstPartyMcpTools,
|
|
549
|
+
tools: taskTools,
|
|
550
|
+
toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
|
|
551
|
+
productServerIds: task.agentConfig.tools.map((tool) => tool.id),
|
|
552
|
+
});
|
|
553
|
+
firstPartyMcpTools = written.firstPartyMcpTools;
|
|
554
|
+
taskTools = written.tools;
|
|
555
|
+
} catch (error) {
|
|
556
|
+
const status =
|
|
557
|
+
error instanceof Error ? (error as Error & { status?: unknown }).status : undefined;
|
|
558
|
+
if (!(error instanceof Error) || status !== 422) throw error;
|
|
559
|
+
const notEnabled =
|
|
560
|
+
error.cause instanceof AgentConfigError &&
|
|
561
|
+
error.cause.code === "agent_config_not_enabled";
|
|
562
|
+
return await refuseAdmission(
|
|
563
|
+
"scheduled_authority_unavailable",
|
|
564
|
+
notEnabled,
|
|
565
|
+
`scheduled agent configuration is not admissible: ${error.message}`,
|
|
566
|
+
);
|
|
567
|
+
}
|
|
520
568
|
const firstPartyMcpPermissions = creatorPolicy?.firstPartyMcpPermissions
|
|
521
569
|
? [...creatorPolicy.firstPartyMcpPermissions]
|
|
522
570
|
: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS];
|
|
@@ -917,6 +965,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
917
965
|
workspaceId: task.workspaceId,
|
|
918
966
|
model: targetSessionExecution?.model ?? model,
|
|
919
967
|
requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1,
|
|
968
|
+
initiatingHumanSubjectId: causalHumanSubjectId,
|
|
920
969
|
},
|
|
921
970
|
);
|
|
922
971
|
if (admissionDenial) {
|
|
@@ -994,6 +1043,12 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
994
1043
|
resolvedTools: taskTools,
|
|
995
1044
|
resolvedFirstPartyMcpTools: firstPartyMcpTools,
|
|
996
1045
|
resolvedFirstPartyMcpPermissions: firstPartyMcpPermissions,
|
|
1046
|
+
...(resolvedAgentConfig
|
|
1047
|
+
? {
|
|
1048
|
+
resolvedAgentConfig,
|
|
1049
|
+
...(resolvedAgentInstructions !== undefined ? { resolvedAgentInstructions } : {}),
|
|
1050
|
+
}
|
|
1051
|
+
: {}),
|
|
997
1052
|
resolvedVariableSet: acceptedVariableSet
|
|
998
1053
|
? { id: acceptedVariableSet.id, generation: acceptedVariableSet.generation }
|
|
999
1054
|
: null,
|
|
@@ -1201,6 +1256,10 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
1201
1256
|
tools: taskTools,
|
|
1202
1257
|
firstPartyMcpTools,
|
|
1203
1258
|
firstPartyMcpPermissions,
|
|
1259
|
+
...(resolvedAgentConfig ? { agentConfig: resolvedAgentConfig } : {}),
|
|
1260
|
+
...(resolvedAgentInstructions !== undefined
|
|
1261
|
+
? { instructions: resolvedAgentInstructions }
|
|
1262
|
+
: {}),
|
|
1204
1263
|
...creatorSessionPolicy,
|
|
1205
1264
|
...(task.agentConfig.knowledgeSource?.destination.kind === "personal"
|
|
1206
1265
|
? {
|
|
@@ -2242,6 +2301,12 @@ async function recoverBoundScheduledTaskDispatch(input: {
|
|
|
2242
2301
|
tools: input.acceptedExecution.resolvedTools,
|
|
2243
2302
|
firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
|
|
2244
2303
|
firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
|
|
2304
|
+
...(input.acceptedExecution.resolvedAgentConfig
|
|
2305
|
+
? { agentConfig: input.acceptedExecution.resolvedAgentConfig }
|
|
2306
|
+
: {}),
|
|
2307
|
+
...(input.acceptedExecution.resolvedAgentInstructions !== undefined
|
|
2308
|
+
? { instructions: input.acceptedExecution.resolvedAgentInstructions }
|
|
2309
|
+
: {}),
|
|
2245
2310
|
...scheduledCreatorSessionPolicyInput(recoveredCreatorPolicy?.sessionPolicy ?? null),
|
|
2246
2311
|
...(task.agentConfig.knowledgeSource?.destination.kind === "personal"
|
|
2247
2312
|
? {
|
|
@@ -2379,20 +2444,23 @@ async function recoverBoundScheduledTaskDispatch(input: {
|
|
|
2379
2444
|
);
|
|
2380
2445
|
const expectedTaskMetadata = { ...task.agentConfig.metadata };
|
|
2381
2446
|
delete expectedTaskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
2382
|
-
const expectedMetadata =
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2447
|
+
const expectedMetadata = metadataWithAgentConfigCreateIdentity(
|
|
2448
|
+
{
|
|
2449
|
+
...expectedTaskMetadata,
|
|
2450
|
+
model: input.acceptedExecution.resolvedModel,
|
|
2451
|
+
reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
|
|
2452
|
+
scheduledTaskId: task.id,
|
|
2453
|
+
scheduledTaskRunMode: task.runMode,
|
|
2454
|
+
...(task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {}),
|
|
2455
|
+
scheduledTaskRunId: canonicalGeneratedRunId,
|
|
2456
|
+
...(frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}),
|
|
2457
|
+
},
|
|
2458
|
+
input.acceptedExecution.resolvedAgentConfig,
|
|
2459
|
+
);
|
|
2392
2460
|
if (
|
|
2393
2461
|
session.createIdempotencyKey !== expectedCreateKey ||
|
|
2394
2462
|
session.initialMessage !== task.agentConfig.prompt ||
|
|
2395
|
-
session.instructions !== null ||
|
|
2463
|
+
session.instructions !== (input.acceptedExecution.resolvedAgentInstructions ?? null) ||
|
|
2396
2464
|
session.policyRole !== null ||
|
|
2397
2465
|
stableJson(session.skills) !== "[]" ||
|
|
2398
2466
|
stableJson(session.toolPolicy) !==
|
|
@@ -2434,6 +2502,8 @@ async function recoverBoundScheduledTaskDispatch(input: {
|
|
|
2434
2502
|
stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) ||
|
|
2435
2503
|
stableJson(session.firstPartyMcpPermissions) !==
|
|
2436
2504
|
stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) ||
|
|
2505
|
+
stableJson(session.agent) !==
|
|
2506
|
+
stableJson(input.acceptedExecution.resolvedAgentConfig ?? null) ||
|
|
2437
2507
|
session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) ||
|
|
2438
2508
|
(session.variableSetId ?? null) !==
|
|
2439
2509
|
(input.acceptedExecution.resolvedVariableSet?.id ?? null) ||
|
package/src/activities/types.ts
CHANGED
|
@@ -458,6 +458,7 @@ export type DispatchScheduledTaskRunResult =
|
|
|
458
458
|
refusal?: import("@opengeni/contracts").ScheduledTaskAdmissionRefusal;
|
|
459
459
|
reason:
|
|
460
460
|
| "insufficient_credits"
|
|
461
|
+
| "allowance_exhausted"
|
|
461
462
|
| "monthly_model_cost_limit"
|
|
462
463
|
| "monthly_agent_run_limit"
|
|
463
464
|
| "malformed_manual_trigger"
|
|
@@ -78,6 +78,8 @@ export function credentialProviderRequestBody(
|
|
|
78
78
|
mcpServers: [],
|
|
79
79
|
},
|
|
80
80
|
): CredentialProviderRequest {
|
|
81
|
+
const via = input.initiatorContext.via;
|
|
82
|
+
const latestHop = Array.isArray(via) ? via.at(-1) : null;
|
|
81
83
|
return {
|
|
82
84
|
type: "credentials.request",
|
|
83
85
|
lane: selection.lane,
|
|
@@ -92,6 +94,16 @@ export function credentialProviderRequestBody(
|
|
|
92
94
|
turnId: input.turnId,
|
|
93
95
|
attemptId: input.attemptId,
|
|
94
96
|
initiator: { kind: input.initiator.kind, subjectId: input.initiator.subjectId },
|
|
97
|
+
initiatorContext: {
|
|
98
|
+
kind:
|
|
99
|
+
latestHop?.kind === "agent"
|
|
100
|
+
? "agent"
|
|
101
|
+
: input.initiator.kind === "subject"
|
|
102
|
+
? "human"
|
|
103
|
+
: "service",
|
|
104
|
+
initiator: input.initiator,
|
|
105
|
+
context: input.initiatorContext,
|
|
106
|
+
},
|
|
95
107
|
initiatingHumanSubjectId,
|
|
96
108
|
initiatingHuman,
|
|
97
109
|
sandboxBackend: input.effectiveSandboxBackend,
|
|
@@ -2,6 +2,7 @@ import { createKnowledgeIndexingActivities } from "./activities/knowledge-indexi
|
|
|
2
2
|
import { createSharedActivityServices } from "./activity-services";
|
|
3
3
|
import { createCodexCapacityActivities } from "./activities/codex-capacity";
|
|
4
4
|
import { createBrowserStateArtifactMaintenanceActivities } from "./activities/browser-state-artifact-reaper";
|
|
5
|
+
import { createBrowserDeadlineCheckpointActivities } from "./activities/browser-deadline-checkpoint";
|
|
5
6
|
import { createAutomationActivities } from "./activities/automations";
|
|
6
7
|
import { createDocumentActivities } from "./activities/documents";
|
|
7
8
|
import { createFileUploadReaperActivities } from "./activities/file-upload-reaper";
|
|
@@ -39,6 +40,7 @@ export function createControlActivitiesFromServices(
|
|
|
39
40
|
...createCodexCapacityActivities(services),
|
|
40
41
|
...createRigVerificationActivities(services),
|
|
41
42
|
...createBrowserStateArtifactMaintenanceActivities(services),
|
|
43
|
+
...createBrowserDeadlineCheckpointActivities(services),
|
|
42
44
|
...createFileUploadReaperActivities(services),
|
|
43
45
|
...createRetainedScreenshotMaintenanceActivities(services),
|
|
44
46
|
...createWorkflowWakeActivities(services),
|
package/src/activities.ts
CHANGED
|
@@ -117,6 +117,9 @@ export const maybeContinueGoal = defaultControlActivities.maybeContinueGoal;
|
|
|
117
117
|
export const getCodexCapacityWait = defaultControlActivities.getCodexCapacityWait;
|
|
118
118
|
export const reconcileCodexCapacityWait = defaultControlActivities.reconcileCodexCapacityWait;
|
|
119
119
|
export const prepareSandboxLeaseSweep = defaultControlActivities.prepareSandboxLeaseSweep;
|
|
120
|
+
export const listDueBrowserCheckpoints = defaultControlActivities.listDueBrowserCheckpoints;
|
|
121
|
+
export const checkpointBrowserBeforeDeadline =
|
|
122
|
+
defaultControlActivities.checkpointBrowserBeforeDeadline;
|
|
120
123
|
export const drainSandboxLease = defaultControlActivities.drainSandboxLease;
|
|
121
124
|
export const maintainSandboxLeaseSweep = defaultControlActivities.maintainSandboxLeaseSweep;
|
|
122
125
|
export const reapSandboxLeases = defaultControlActivities.reapSandboxLeases;
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export const BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID = "opengeni-browser-deadline-checkpoints-v1";
|
|
2
|
+
const QUEUE_SUFFIX = "-browser-checkpoint-v1";
|
|
3
|
+
export function browserDeadlineCheckpointTaskQueue(base: string): string {
|
|
4
|
+
return base.endsWith(QUEUE_SUFFIX) ? base : `${base}${QUEUE_SUFFIX}`;
|
|
5
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -99,6 +99,10 @@ import {
|
|
|
99
99
|
SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
100
100
|
sandboxLifecycleTaskQueue,
|
|
101
101
|
} from "./sandbox-reaper-contract";
|
|
102
|
+
import {
|
|
103
|
+
BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
|
|
104
|
+
browserDeadlineCheckpointTaskQueue,
|
|
105
|
+
} from "./browser-deadline-checkpoint-contract";
|
|
102
106
|
|
|
103
107
|
export {
|
|
104
108
|
createHostExportPump,
|
|
@@ -323,28 +327,39 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
323
327
|
return { worker, connection };
|
|
324
328
|
}
|
|
325
329
|
|
|
330
|
+
const ownedWorkers = [worker];
|
|
326
331
|
try {
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
332
|
+
// The new activity type has its own queue. Retain the V1 poller so old
|
|
333
|
+
// drain histories finish; old binaries never receive checkpoint work.
|
|
334
|
+
for (const taskQueue of [
|
|
335
|
+
sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
336
|
+
browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
|
|
337
|
+
]) {
|
|
338
|
+
const lifecycleWorker = await Worker.create({
|
|
339
|
+
...sharedWorkerOptions,
|
|
340
|
+
...workflowDefinition,
|
|
341
|
+
taskQueue,
|
|
342
|
+
reuseV8Context: true,
|
|
343
|
+
workflowThreadPoolSize: 1,
|
|
344
|
+
maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
345
|
+
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
346
|
+
maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
347
|
+
});
|
|
348
|
+
ownedWorkers.push(lifecycleWorker);
|
|
349
|
+
}
|
|
337
350
|
return {
|
|
338
|
-
worker: combineWorkerRunTargets(
|
|
351
|
+
worker: combineWorkerRunTargets(ownedWorkers),
|
|
339
352
|
connection,
|
|
340
353
|
};
|
|
341
354
|
} catch (error) {
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
355
|
+
for (const ownedWorker of ownedWorkers) {
|
|
356
|
+
try {
|
|
357
|
+
ownedWorker.shutdown();
|
|
358
|
+
} catch {
|
|
359
|
+
// Preserve the lifecycle worker construction failure. The base poller
|
|
360
|
+
// still received its shutdown request; a secondary synchronous
|
|
361
|
+
// shutdown error must not replace the startup root cause.
|
|
362
|
+
}
|
|
348
363
|
}
|
|
349
364
|
throw error;
|
|
350
365
|
}
|
|
@@ -478,20 +493,33 @@ export async function createWorkerWorkflowSignaler(
|
|
|
478
493
|
return normalizeTurnTaskQueueStats(response.stats);
|
|
479
494
|
},
|
|
480
495
|
startSandboxReaperWorkflow: async () => {
|
|
481
|
-
|
|
482
|
-
|
|
496
|
+
// Same backend tick, no additional schedule. A bounded independent
|
|
497
|
+
// inventory cannot prevent the existing per-box drain from starting.
|
|
498
|
+
const starts = await Promise.allSettled([
|
|
499
|
+
temporal.workflow.start("sandboxReaperWorkflowV2", {
|
|
483
500
|
taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
484
501
|
workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
485
502
|
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
486
503
|
args: [],
|
|
487
|
-
})
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
504
|
+
}),
|
|
505
|
+
temporal.workflow.start("browserDeadlineCheckpointSweepWorkflow", {
|
|
506
|
+
taskQueue: browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
|
|
507
|
+
workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
|
|
508
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
509
|
+
args: [],
|
|
510
|
+
}),
|
|
511
|
+
]);
|
|
512
|
+
for (const start of starts) {
|
|
513
|
+
if (
|
|
514
|
+
start.status === "rejected" &&
|
|
515
|
+
!(start.reason instanceof WorkflowExecutionAlreadyStartedError)
|
|
516
|
+
) {
|
|
517
|
+
// The other start already settled. Retry the router without losing
|
|
518
|
+
// ordinary draining or silently omitting the checkpoint sweep.
|
|
519
|
+
throw start.reason;
|
|
492
520
|
}
|
|
493
|
-
throw error;
|
|
494
521
|
}
|
|
522
|
+
return starts[0].status === "fulfilled" ? "started" : "already_running";
|
|
495
523
|
},
|
|
496
524
|
startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
|
|
497
525
|
try {
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -210,6 +210,9 @@ export type ResumeBoxIds = {
|
|
|
210
210
|
* never conflicts).
|
|
211
211
|
*/
|
|
212
212
|
image?: string;
|
|
213
|
+
/** Deployment/workspace image pins apply only to new creates; an existing
|
|
214
|
+
* group keeps its image. Omission preserves explicit image matching (B3). */
|
|
215
|
+
imagePolicy?: "require_match" | "new_creates_only";
|
|
213
216
|
/**
|
|
214
217
|
* RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
|
|
215
218
|
* acquireLease, which stamps it on the cold-create and conflicts on a live box
|
|
@@ -445,9 +448,10 @@ export async function waitForSandboxExecReadiness(
|
|
|
445
448
|
established: EstablishedSandboxSession,
|
|
446
449
|
timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
447
450
|
identity: { sandboxGroupId?: string | null } = {},
|
|
451
|
+
signal?: AbortSignal,
|
|
448
452
|
): Promise<void> {
|
|
449
453
|
try {
|
|
450
|
-
await verifySandboxExecReadiness(established, timeoutMs);
|
|
454
|
+
await verifySandboxExecReadiness(established, timeoutMs, signal);
|
|
451
455
|
} catch (error) {
|
|
452
456
|
if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
|
|
453
457
|
throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
|
|
@@ -1347,11 +1351,12 @@ async function resumeBoxForTurnOnce(
|
|
|
1347
1351
|
},
|
|
1348
1352
|
os,
|
|
1349
1353
|
// IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
|
|
1350
|
-
// conflicts on a live box already running a different image. A
|
|
1354
|
+
// conflicts on a live box already running a different required image. A
|
|
1351
1355
|
// SandboxImageConflictError propagates while another holder is active; a
|
|
1352
1356
|
// solo change requests a capture-and-drain rotation and this attempt retries
|
|
1353
1357
|
// after the cold successor can safely stamp the new image.
|
|
1354
1358
|
...(ids.image ? { image: ids.image } : {}),
|
|
1359
|
+
...(ids.imagePolicy ? { imagePolicy: ids.imagePolicy } : {}),
|
|
1355
1360
|
// RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
|
|
1356
1361
|
// + conflicts on a live box under a different rig. A SandboxRigConflictError
|
|
1357
1362
|
// propagates while another holder is active; a solo change uses the same
|
|
@@ -1917,9 +1922,14 @@ async function resumeBoxForTurnOnce(
|
|
|
1917
1922
|
sandboxGroupId: ids.sandboxGroupId,
|
|
1918
1923
|
});
|
|
1919
1924
|
} else {
|
|
1920
|
-
await waitForSandboxExecReadiness(
|
|
1921
|
-
|
|
1922
|
-
|
|
1925
|
+
await waitForSandboxExecReadiness(
|
|
1926
|
+
established,
|
|
1927
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
1928
|
+
{
|
|
1929
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1930
|
+
},
|
|
1931
|
+
cancellationSignal,
|
|
1932
|
+
);
|
|
1923
1933
|
}
|
|
1924
1934
|
await maybeRenewProviderExpiration(true);
|
|
1925
1935
|
throwIfReleasedOrCancelled();
|
|
@@ -2117,9 +2127,14 @@ async function resumeBoxForTurnOnce(
|
|
|
2117
2127
|
if (services.verifyAttachedSandboxReadiness) {
|
|
2118
2128
|
await services.verifyAttachedSandboxReadiness(established);
|
|
2119
2129
|
} else {
|
|
2120
|
-
await waitForSandboxExecReadiness(
|
|
2121
|
-
|
|
2122
|
-
|
|
2130
|
+
await waitForSandboxExecReadiness(
|
|
2131
|
+
established,
|
|
2132
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
2133
|
+
{
|
|
2134
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2135
|
+
},
|
|
2136
|
+
cancellationSignal,
|
|
2137
|
+
);
|
|
2123
2138
|
}
|
|
2124
2139
|
providerRenewalTarget = {
|
|
2125
2140
|
backend: ids.backend,
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -496,7 +496,7 @@ function afterPersistableHomeMutation(
|
|
|
496
496
|
op: string;
|
|
497
497
|
backend: ResolvedActiveBackend;
|
|
498
498
|
admission: unknown;
|
|
499
|
-
outcome: "resolved" | "rejected";
|
|
499
|
+
outcome: "resolved" | "rejected" | "outcome_unknown";
|
|
500
500
|
result?: unknown;
|
|
501
501
|
retainedProcess?: RoutingRetainedProcess;
|
|
502
502
|
}) => Promise<void | RoutingMutationSettlementResult>)
|
|
@@ -523,7 +523,7 @@ function afterPersistableHomeMutation(
|
|
|
523
523
|
) {
|
|
524
524
|
throw new Error("Persistable home mutation settlement lacked its bound admission");
|
|
525
525
|
}
|
|
526
|
-
if (outcome === "resolved" && retainedProcess) {
|
|
526
|
+
if ((outcome === "resolved" || outcome === "outcome_unknown") && retainedProcess) {
|
|
527
527
|
try {
|
|
528
528
|
await retainWorkspaceProviderCommand(services.db, {
|
|
529
529
|
accountId: fence.accountId,
|
|
@@ -582,6 +582,8 @@ function afterPersistableHomeMutation(
|
|
|
582
582
|
}
|
|
583
583
|
return;
|
|
584
584
|
}
|
|
585
|
+
if (outcome === "outcome_unknown")
|
|
586
|
+
throw new Error("Outcome-unknown command settlement requires its exact retained invocation");
|
|
585
587
|
await verifyWorkspaceMutationSettlement(services.db, {
|
|
586
588
|
accountId: fence.accountId,
|
|
587
589
|
workspaceId: ids.workspaceId,
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ParentClosePolicy,
|
|
3
|
+
WorkflowIdReusePolicy,
|
|
4
|
+
log,
|
|
5
|
+
proxyActivities,
|
|
6
|
+
startChild,
|
|
7
|
+
workflowInfo,
|
|
8
|
+
} from "@temporalio/workflow";
|
|
9
|
+
import type { BrowserDeadlineCheckpointTarget } from "@opengeni/db";
|
|
10
|
+
import type * as activities from "../activities";
|
|
11
|
+
|
|
12
|
+
const inventory = proxyActivities<Pick<typeof activities, "listDueBrowserCheckpoints">>({
|
|
13
|
+
startToCloseTimeout: "5 minutes",
|
|
14
|
+
heartbeatTimeout: "20 seconds",
|
|
15
|
+
});
|
|
16
|
+
const checkpoint = proxyActivities<Pick<typeof activities, "checkpointBrowserBeforeDeadline">>({
|
|
17
|
+
startToCloseTimeout: "25 minutes",
|
|
18
|
+
heartbeatTimeout: "20 seconds",
|
|
19
|
+
retry: { initialInterval: "1 second", maximumInterval: "30 seconds" },
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
export async function browserDeadlineCheckpointWorkflow(target: BrowserDeadlineCheckpointTarget) {
|
|
23
|
+
return await checkpoint.checkpointBrowserBeforeDeadline(target);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Runs from the existing lease-sweep dispatch. Inventory never performs provider
|
|
27
|
+
* I/O; each exact browser has an independent durable capture/cleanup child. */
|
|
28
|
+
export async function browserDeadlineCheckpointSweepWorkflow(): Promise<void> {
|
|
29
|
+
const targets = await inventory.listDueBrowserCheckpoints();
|
|
30
|
+
await Promise.all(
|
|
31
|
+
targets.map(async (target) => {
|
|
32
|
+
try {
|
|
33
|
+
await startChild(browserDeadlineCheckpointWorkflow, {
|
|
34
|
+
workflowId: `browser-deadline:${target.browserSessionId}:${target.leaseEpoch}:${target.controllerGeneration}`,
|
|
35
|
+
taskQueue: workflowInfo().taskQueue,
|
|
36
|
+
workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
|
|
37
|
+
parentClosePolicy: ParentClosePolicy.ABANDON,
|
|
38
|
+
args: [target],
|
|
39
|
+
});
|
|
40
|
+
} catch (error) {
|
|
41
|
+
if (error instanceof Error && error.name === "WorkflowExecutionAlreadyStartedError") return;
|
|
42
|
+
// Another sweep retries start; it cannot retire the browser's authority.
|
|
43
|
+
log.warn("browser deadline checkpoint child start deferred", {
|
|
44
|
+
browserSessionId: target.browserSessionId,
|
|
45
|
+
error: error instanceof Error ? error.name : "unknown",
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}),
|
|
49
|
+
);
|
|
50
|
+
}
|
package/src/workflows.ts
CHANGED
|
@@ -28,6 +28,10 @@ export {
|
|
|
28
28
|
sandboxReaperWorkflowV2,
|
|
29
29
|
} from "./workflows/sandbox-reaper";
|
|
30
30
|
export { fileUploadReaperWorkflow } from "./workflows/file-upload-reaper";
|
|
31
|
+
export {
|
|
32
|
+
browserDeadlineCheckpointWorkflow,
|
|
33
|
+
browserDeadlineCheckpointSweepWorkflow,
|
|
34
|
+
} from "./workflows/browser-deadline-checkpoint";
|
|
31
35
|
export { siteAuthMaintenanceWorkflow } from "./workflows/site-auth-maintenance";
|
|
32
36
|
export {
|
|
33
37
|
videoGenerationWorkflow,
|