@opengeni/worker-bundle 0.7.5 → 0.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BJUW5LCM.js +325 -0
- package/dist/chunk-BJUW5LCM.js.map +1 -0
- package/dist/host-export.d.ts +70 -0
- package/dist/host-export.js +7 -0
- package/dist/host-export.js.map +1 -0
- package/dist/index.d.ts +98 -6
- package/dist/index.js +4097 -1271
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +12360 -0
- package/package.json +18 -12
- package/src/activities/agent-turn.ts +2674 -828
- package/src/activities/capabilities.ts +11 -2
- package/src/activities/codex-capacity.ts +6 -11
- package/src/activities/codex-rotation.ts +75 -86
- package/src/activities/context-compaction.ts +104 -24
- package/src/activities/environment.ts +503 -97
- package/src/activities/expiring-material-renewal.ts +143 -0
- package/src/activities/goals.ts +4 -0
- package/src/activities/mcp-credentials.ts +38 -0
- package/src/activities/packs.ts +115 -2
- package/src/activities/rig-verification.ts +897 -244
- package/src/activities/run-credential-renewal.ts +70 -0
- package/src/activities/run-credentials.ts +132 -0
- package/src/activities/run-input.ts +23 -0
- package/src/activities/sandbox-lease.ts +33 -2
- package/src/activities/scheduled-tasks.ts +21 -0
- package/src/activities/session-state.ts +72 -0
- package/src/activities/streaming.ts +1 -0
- package/src/activities/toolspace-token-renewal.ts +57 -0
- package/src/activities/types.ts +44 -6
- package/src/activities/workspace-capture.ts +113 -93
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +416 -0
- package/src/host-export.ts +22 -0
- package/src/http.ts +77 -28
- package/src/index.ts +360 -156
- package/src/observability-metrics.ts +83 -15
- package/src/sandbox-resume.ts +47 -1
- package/src/sandbox-routing.ts +1 -1
- package/src/worker-service-lifecycle.ts +104 -0
- package/src/workflows/activities.ts +13 -7
- package/src/workflows/rig-verification.ts +3 -1
- package/src/workflows/session.ts +219 -23
- package/src/workflows.ts +1 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { NormalizedRunCredentialMaterial } from "@opengeni/runtime";
|
|
2
|
+
import {
|
|
3
|
+
nextExpiringMaterialRenewalDelay,
|
|
4
|
+
startExpiringMaterialRenewalLoop,
|
|
5
|
+
type ExpiringMaterialRenewalController,
|
|
6
|
+
} from "./expiring-material-renewal";
|
|
7
|
+
|
|
8
|
+
export const RUN_CREDENTIAL_DEFAULT_REFRESH_MS = 30 * 60_000;
|
|
9
|
+
export const RUN_CREDENTIAL_EXPIRY_LEAD_MS = 5 * 60_000;
|
|
10
|
+
export const RUN_CREDENTIAL_MIN_REFRESH_MS = 5_000;
|
|
11
|
+
export const RUN_CREDENTIAL_MAX_RETRY_MS = 5 * 60_000;
|
|
12
|
+
|
|
13
|
+
export type RunCredentialRenewalController = ExpiringMaterialRenewalController;
|
|
14
|
+
|
|
15
|
+
type RenewalTimer = unknown;
|
|
16
|
+
|
|
17
|
+
export type RunCredentialRenewalOptions = {
|
|
18
|
+
initialExpiresAt: Date | null;
|
|
19
|
+
resolve: () => Promise<NormalizedRunCredentialMaterial | null>;
|
|
20
|
+
write: (material: NormalizedRunCredentialMaterial | null) => Promise<void>;
|
|
21
|
+
now?: () => number;
|
|
22
|
+
schedule?: (callback: () => void, delayMs: number) => RenewalTimer;
|
|
23
|
+
clearSchedule?: (timer: RenewalTimer) => void;
|
|
24
|
+
onSuccess?: (result: { nextDelayMs: number; authNeeded: boolean }) => void;
|
|
25
|
+
onFailure?: (failure: { retryDelayMs: number; errorClass: string }) => void;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export function nextRunCredentialRenewalDelay(expiresAt: Date | null, nowMs = Date.now()): number {
|
|
29
|
+
return nextExpiringMaterialRenewalDelay(
|
|
30
|
+
expiresAt,
|
|
31
|
+
{
|
|
32
|
+
defaultRefreshMs: RUN_CREDENTIAL_DEFAULT_REFRESH_MS,
|
|
33
|
+
expiryLeadMs: RUN_CREDENTIAL_EXPIRY_LEAD_MS,
|
|
34
|
+
minRefreshMs: RUN_CREDENTIAL_MIN_REFRESH_MS,
|
|
35
|
+
maxRetryMs: RUN_CREDENTIAL_MAX_RETRY_MS,
|
|
36
|
+
},
|
|
37
|
+
nowMs,
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Single-flight renewal; late resolves cannot mutate a stopped attempt. */
|
|
42
|
+
export function startRunCredentialRenewalLoop(
|
|
43
|
+
options: RunCredentialRenewalOptions,
|
|
44
|
+
): RunCredentialRenewalController {
|
|
45
|
+
return startExpiringMaterialRenewalLoop<NormalizedRunCredentialMaterial | null>({
|
|
46
|
+
initialExpiresAt: options.initialExpiresAt,
|
|
47
|
+
resolve: options.resolve,
|
|
48
|
+
write: options.write,
|
|
49
|
+
expiresAt: (material) => material?.expiresAt ?? null,
|
|
50
|
+
policy: {
|
|
51
|
+
defaultRefreshMs: RUN_CREDENTIAL_DEFAULT_REFRESH_MS,
|
|
52
|
+
expiryLeadMs: RUN_CREDENTIAL_EXPIRY_LEAD_MS,
|
|
53
|
+
minRefreshMs: RUN_CREDENTIAL_MIN_REFRESH_MS,
|
|
54
|
+
maxRetryMs: RUN_CREDENTIAL_MAX_RETRY_MS,
|
|
55
|
+
},
|
|
56
|
+
...(options.now ? { now: options.now } : {}),
|
|
57
|
+
...(options.schedule ? { schedule: options.schedule } : {}),
|
|
58
|
+
...(options.clearSchedule ? { clearSchedule: options.clearSchedule } : {}),
|
|
59
|
+
...(options.onSuccess
|
|
60
|
+
? {
|
|
61
|
+
onSuccess: ({ material, nextDelayMs }) =>
|
|
62
|
+
options.onSuccess?.({
|
|
63
|
+
nextDelayMs,
|
|
64
|
+
authNeeded: (material?.authNeeded.length ?? 0) > 0,
|
|
65
|
+
}),
|
|
66
|
+
}
|
|
67
|
+
: {}),
|
|
68
|
+
...(options.onFailure ? { onFailure: options.onFailure } : {}),
|
|
69
|
+
});
|
|
70
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ConnectionCredentialsPort,
|
|
3
|
+
CredentialAuthNeededPayload,
|
|
4
|
+
SandboxBackend,
|
|
5
|
+
RunCredentialsRequest,
|
|
6
|
+
Session,
|
|
7
|
+
SessionTurn,
|
|
8
|
+
} from "@opengeni/contracts";
|
|
9
|
+
import { getSessionRootId, type Database } from "@opengeni/db";
|
|
10
|
+
import {
|
|
11
|
+
normalizeRunCredentialsResolution,
|
|
12
|
+
type NormalizedRunCredentialMaterial,
|
|
13
|
+
} from "@opengeni/runtime";
|
|
14
|
+
|
|
15
|
+
export type RunCredentialResolutionContext = {
|
|
16
|
+
db: Database;
|
|
17
|
+
connectionCredentials?: ConnectionCredentialsPort | null;
|
|
18
|
+
accountId: string;
|
|
19
|
+
workspaceId: string;
|
|
20
|
+
session: Session;
|
|
21
|
+
turn: SessionTurn;
|
|
22
|
+
attemptId: string;
|
|
23
|
+
effectiveSandboxBackend: SandboxBackend;
|
|
24
|
+
variableSet: { id: string; name: string } | null;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export type BoundRunCredentialResolver = {
|
|
28
|
+
resolve(input: {
|
|
29
|
+
purpose: "provision" | "renewal";
|
|
30
|
+
forceRefresh: boolean;
|
|
31
|
+
}): Promise<NormalizedRunCredentialMaterial | null>;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export function buildRunCredentialsRequest(
|
|
35
|
+
input: Omit<RunCredentialResolutionContext, "db" | "connectionCredentials"> & {
|
|
36
|
+
rootSessionId: string;
|
|
37
|
+
purpose: "provision" | "renewal";
|
|
38
|
+
forceRefresh: boolean;
|
|
39
|
+
},
|
|
40
|
+
): RunCredentialsRequest {
|
|
41
|
+
return {
|
|
42
|
+
accountId: input.accountId,
|
|
43
|
+
workspaceId: input.workspaceId,
|
|
44
|
+
sessionId: input.session.id,
|
|
45
|
+
parentSessionId: input.session.parentSessionId,
|
|
46
|
+
rootSessionId: input.rootSessionId,
|
|
47
|
+
sandboxGroupId: input.session.sandboxGroupId,
|
|
48
|
+
turnId: input.turn.id,
|
|
49
|
+
attemptId: input.attemptId,
|
|
50
|
+
executionGeneration: input.turn.executionGeneration,
|
|
51
|
+
initiator: input.turn.initiator,
|
|
52
|
+
initiatorContext: input.turn.initiatorContext,
|
|
53
|
+
effectiveSandboxBackend: input.effectiveSandboxBackend,
|
|
54
|
+
sandboxOs: input.turn.sandboxOs ?? input.session.sandboxOs,
|
|
55
|
+
purpose: input.purpose,
|
|
56
|
+
forceRefresh: input.forceRefresh,
|
|
57
|
+
variableSet: input.variableSet,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function runCredentialAuthNeededPayloads(
|
|
62
|
+
material: NormalizedRunCredentialMaterial,
|
|
63
|
+
): CredentialAuthNeededPayload[] {
|
|
64
|
+
return material.authNeeded.map((notice) => ({
|
|
65
|
+
credentialClass: "run",
|
|
66
|
+
reason: notice.reason,
|
|
67
|
+
...(notice.providerDomain ? { providerDomain: notice.providerDomain } : {}),
|
|
68
|
+
...(notice.connectionId ? { connectionId: notice.connectionId } : {}),
|
|
69
|
+
...(notice.scopes?.length ? { scopes: notice.scopes } : {}),
|
|
70
|
+
...(notice.resource ? { resource: notice.resource } : {}),
|
|
71
|
+
...(notice.authorizationUrl ? { authorizationUrl: notice.authorizationUrl } : {}),
|
|
72
|
+
...(notice.message ? { message: notice.message } : {}),
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function runCredentialModelNote(
|
|
77
|
+
material: NormalizedRunCredentialMaterial,
|
|
78
|
+
): string | undefined {
|
|
79
|
+
if (material.authNeeded.length === 0) return undefined;
|
|
80
|
+
return [
|
|
81
|
+
"[OpenGeni connected-service status]",
|
|
82
|
+
"One or more host-managed credentials need user attention. Continue with available capabilities, but do not claim the affected service is usable until it is reconnected.",
|
|
83
|
+
JSON.stringify({
|
|
84
|
+
credentials: material.authNeeded.map((notice) => ({
|
|
85
|
+
reason: notice.reason,
|
|
86
|
+
...(notice.providerDomain ? { providerDomain: notice.providerDomain } : {}),
|
|
87
|
+
...(notice.resource ? { resource: notice.resource } : {}),
|
|
88
|
+
...(notice.message ? { message: notice.message } : {}),
|
|
89
|
+
})),
|
|
90
|
+
}),
|
|
91
|
+
].join("\n");
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Freeze the provider-neutral host credential request to this exact admitted
|
|
96
|
+
* turn. The host selects connections and material; the worker never infers a
|
|
97
|
+
* provider from repositories, environment names, or an OpenGeni variable set.
|
|
98
|
+
*/
|
|
99
|
+
export async function bindRunCredentialResolver(
|
|
100
|
+
input: RunCredentialResolutionContext,
|
|
101
|
+
): Promise<BoundRunCredentialResolver | null> {
|
|
102
|
+
const resolver = input.connectionCredentials?.runCredentials;
|
|
103
|
+
if (!resolver) return null;
|
|
104
|
+
const rootSessionId = await getSessionRootId(input.db, input.workspaceId, input.session.id);
|
|
105
|
+
if (!rootSessionId) {
|
|
106
|
+
throw new Error(`cannot resolve run credentials for missing session ${input.session.id}`);
|
|
107
|
+
}
|
|
108
|
+
const scope = {
|
|
109
|
+
accountId: input.accountId,
|
|
110
|
+
workspaceId: input.workspaceId,
|
|
111
|
+
sessionId: input.session.id,
|
|
112
|
+
};
|
|
113
|
+
return {
|
|
114
|
+
resolve: async ({ purpose, forceRefresh }) => {
|
|
115
|
+
const resolution = await resolver(
|
|
116
|
+
buildRunCredentialsRequest({
|
|
117
|
+
accountId: input.accountId,
|
|
118
|
+
workspaceId: input.workspaceId,
|
|
119
|
+
session: input.session,
|
|
120
|
+
turn: input.turn,
|
|
121
|
+
attemptId: input.attemptId,
|
|
122
|
+
effectiveSandboxBackend: input.effectiveSandboxBackend,
|
|
123
|
+
variableSet: input.variableSet,
|
|
124
|
+
rootSessionId,
|
|
125
|
+
purpose,
|
|
126
|
+
forceRefresh,
|
|
127
|
+
}),
|
|
128
|
+
);
|
|
129
|
+
return normalizeRunCredentialsResolution(resolution, scope);
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
}
|
|
@@ -2,6 +2,7 @@ import type { FileAsset, ResourceRef, SessionSystemUpdate } from "@opengeni/cont
|
|
|
2
2
|
import {
|
|
3
3
|
getActiveSessionHistoryItems,
|
|
4
4
|
getLatestRunState,
|
|
5
|
+
getHumanInputResumeForEvent,
|
|
5
6
|
getSandboxSessionEnvelope,
|
|
6
7
|
getSessionEvent,
|
|
7
8
|
listSessionSystemUpdatesForTurn,
|
|
@@ -144,6 +145,7 @@ export type TurnInputOptions = {
|
|
|
144
145
|
turnId: string;
|
|
145
146
|
recovering?: boolean;
|
|
146
147
|
unavailableSandboxFilesNote?: string;
|
|
148
|
+
runCredentialsNote?: string;
|
|
147
149
|
};
|
|
148
150
|
|
|
149
151
|
export async function turnInput(
|
|
@@ -173,6 +175,7 @@ export async function turnInput(
|
|
|
173
175
|
: undefined,
|
|
174
176
|
updateContext,
|
|
175
177
|
options.unavailableSandboxFilesNote,
|
|
178
|
+
options.runCredentialsNote,
|
|
176
179
|
);
|
|
177
180
|
if (trigger.type === "user.message") {
|
|
178
181
|
const payload = trigger.payload as { text?: unknown; resources?: unknown };
|
|
@@ -228,6 +231,26 @@ export async function turnInput(
|
|
|
228
231
|
modelHistoryFromItems: false,
|
|
229
232
|
};
|
|
230
233
|
}
|
|
234
|
+
if (trigger.type === "user.humanInputResponse") {
|
|
235
|
+
const [state, resume] = await Promise.all([
|
|
236
|
+
getLatestRunState(db, trigger.workspaceId, trigger.sessionId),
|
|
237
|
+
getHumanInputResumeForEvent(db, trigger.workspaceId, trigger.sessionId, trigger),
|
|
238
|
+
]);
|
|
239
|
+
if (!state) {
|
|
240
|
+
throw new Error("No saved run state is available for human-input response");
|
|
241
|
+
}
|
|
242
|
+
if (!resume) {
|
|
243
|
+
throw new Error("Human-input response does not resolve to a durable request");
|
|
244
|
+
}
|
|
245
|
+
return {
|
|
246
|
+
input: await runtime.prepareInput(agent, {
|
|
247
|
+
kind: "human_input",
|
|
248
|
+
serializedRunState: resumeRunStateForCodexAccount(state, current),
|
|
249
|
+
toolCallId: resume.toolCallId,
|
|
250
|
+
}),
|
|
251
|
+
modelHistoryFromItems: false,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
231
254
|
throw new Error(`Unsupported trigger event type: ${trigger.type}`);
|
|
232
255
|
}
|
|
233
256
|
|
|
@@ -63,6 +63,7 @@ import {
|
|
|
63
63
|
isProviderSandboxNotFoundError,
|
|
64
64
|
sweepModalOrphanSandboxes,
|
|
65
65
|
terminateModalSandboxById,
|
|
66
|
+
type ModalOrphanSweepTermination,
|
|
66
67
|
} from "@opengeni/runtime";
|
|
67
68
|
import type { ActivityServices } from "./types";
|
|
68
69
|
import { reconcilePendingParentSystemUpdates } from "./parent-wake";
|
|
@@ -447,6 +448,31 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
447
448
|
return forceDrained;
|
|
448
449
|
}
|
|
449
450
|
|
|
451
|
+
export function modalOrphanTerminationStillEligible(
|
|
452
|
+
latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
|
|
453
|
+
candidate: ModalOrphanSweepTermination,
|
|
454
|
+
): boolean {
|
|
455
|
+
if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
|
|
456
|
+
return false;
|
|
457
|
+
}
|
|
458
|
+
const leaseId = candidate.tags.opengeni_lease_id;
|
|
459
|
+
const workspaceId = candidate.tags.opengeni_workspace_id;
|
|
460
|
+
const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;
|
|
461
|
+
if (!leaseId || !workspaceId || !sandboxGroupId) {
|
|
462
|
+
return true;
|
|
463
|
+
}
|
|
464
|
+
const activeAttribution = latest.find(
|
|
465
|
+
(lease) =>
|
|
466
|
+
lease.leaseId === leaseId &&
|
|
467
|
+
lease.workspaceId === workspaceId &&
|
|
468
|
+
lease.sandboxGroupId === sandboxGroupId,
|
|
469
|
+
);
|
|
470
|
+
return Boolean(
|
|
471
|
+
!activeAttribution ||
|
|
472
|
+
(activeAttribution.instanceId !== null && activeAttribution.instanceId !== candidate.sandboxId),
|
|
473
|
+
);
|
|
474
|
+
}
|
|
475
|
+
|
|
450
476
|
async function sweepModalOrphansForConfiguredBackend(
|
|
451
477
|
settings: ActivityServices["settings"],
|
|
452
478
|
db: ActivityServices["db"],
|
|
@@ -468,7 +494,12 @@ async function sweepModalOrphansForConfiguredBackend(
|
|
|
468
494
|
return 0;
|
|
469
495
|
}
|
|
470
496
|
|
|
471
|
-
const result = await sweepModalOrphanSandboxes(settings, liveLeases
|
|
497
|
+
const result = await sweepModalOrphanSandboxes(settings, liveLeases, {
|
|
498
|
+
revalidateTermination: async (candidate) => {
|
|
499
|
+
const latest = await listLiveModalSandboxLeaseAttributions(db);
|
|
500
|
+
return modalOrphanTerminationStillEligible(latest, candidate);
|
|
501
|
+
},
|
|
502
|
+
});
|
|
472
503
|
for (const terminated of result.terminated) {
|
|
473
504
|
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
474
505
|
sandboxId: terminated.sandboxId,
|
|
@@ -647,7 +678,7 @@ export async function terminateProviderBox(
|
|
|
647
678
|
return true;
|
|
648
679
|
}
|
|
649
680
|
|
|
650
|
-
// CRITICAL SAFETY (bring-your-own-compute
|
|
681
|
+
// CRITICAL SAFETY (bring-your-own-compute): NEVER provider-stop
|
|
651
682
|
// a selfhosted box. The "box" is a user's PHYSICAL machine — you cannot kill it,
|
|
652
683
|
// and a delete()/kill() reaching the agent would be catastrophic. The reaper
|
|
653
684
|
// DRAINS the lease to cold only (refcount→0 → draining → this returns true so
|
|
@@ -77,6 +77,9 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
77
77
|
unit: "run",
|
|
78
78
|
sourceResourceType: "scheduled_task_run",
|
|
79
79
|
sourceResourceId: run.id,
|
|
80
|
+
initiator: { kind: "service", subjectId: "scheduler" },
|
|
81
|
+
initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
82
|
+
origin: "scheduled_task",
|
|
80
83
|
idempotencyKey: `usage:scheduled_task.fired:${run.id}`,
|
|
81
84
|
});
|
|
82
85
|
if (run.status === "dispatched" && run.sessionId && run.triggerEventId) {
|
|
@@ -88,6 +91,10 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
88
91
|
unit: "run",
|
|
89
92
|
sourceResourceType: "scheduled_task_run",
|
|
90
93
|
sourceResourceId: run.id,
|
|
94
|
+
sessionId: run.sessionId,
|
|
95
|
+
initiator: { kind: "service", subjectId: "scheduler" },
|
|
96
|
+
initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
97
|
+
origin: "scheduled_task",
|
|
91
98
|
idempotencyKey:
|
|
92
99
|
input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`,
|
|
93
100
|
});
|
|
@@ -164,6 +171,15 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
164
171
|
scheduledTaskId: task.id,
|
|
165
172
|
scheduledTaskRunId: run.id,
|
|
166
173
|
},
|
|
174
|
+
createdBy: {
|
|
175
|
+
kind: "service",
|
|
176
|
+
subjectId: "scheduler",
|
|
177
|
+
label: "OpenGeni scheduler",
|
|
178
|
+
},
|
|
179
|
+
createdByContext: {
|
|
180
|
+
scheduledTaskId: task.id,
|
|
181
|
+
scheduledTaskRunId: run.id,
|
|
182
|
+
},
|
|
167
183
|
model,
|
|
168
184
|
sandboxBackend,
|
|
169
185
|
variableSetId: task.variableSetId ?? null,
|
|
@@ -193,6 +209,7 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
193
209
|
type: "session.created",
|
|
194
210
|
payload: {
|
|
195
211
|
status: session.status,
|
|
212
|
+
createdBy: session.createdBy,
|
|
196
213
|
scheduledTaskId: task.id,
|
|
197
214
|
scheduledTaskRunId: run.id,
|
|
198
215
|
// Names/ids only; never values.
|
|
@@ -390,6 +407,10 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
390
407
|
unit: "run",
|
|
391
408
|
sourceResourceType: "scheduled_task_run",
|
|
392
409
|
sourceResourceId: run.id,
|
|
410
|
+
sessionId: result.sessionId,
|
|
411
|
+
initiator: { kind: "service", subjectId: "scheduler" },
|
|
412
|
+
initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
413
|
+
origin: "scheduled_task",
|
|
393
414
|
idempotencyKey:
|
|
394
415
|
input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`,
|
|
395
416
|
});
|
|
@@ -6,6 +6,8 @@ import {
|
|
|
6
6
|
countQueuedTurns,
|
|
7
7
|
getSessionEvent,
|
|
8
8
|
getSessionTurnForAttempt,
|
|
9
|
+
expireSessionHumanInputRequest,
|
|
10
|
+
markSessionAttemptQuiesced,
|
|
9
11
|
requireSession,
|
|
10
12
|
settleSessionIdleWithParentOutbox,
|
|
11
13
|
} from "@opengeni/db";
|
|
@@ -14,10 +16,13 @@ import { deliverFailedChildTurnToParent, notifyParentOfChildIdle } from "./paren
|
|
|
14
16
|
import { recordTurnsQueuedGauge } from "../observability-metrics";
|
|
15
17
|
import type {
|
|
16
18
|
ActivityServices,
|
|
19
|
+
ExpireSessionHumanInputInput,
|
|
20
|
+
ExpireSessionHumanInputResult,
|
|
17
21
|
PeekSessionWorkInput,
|
|
18
22
|
FailSessionAttemptInput,
|
|
19
23
|
SettleSessionInterruptionsInput,
|
|
20
24
|
MarkSessionIdleInput,
|
|
25
|
+
PersistSessionAttemptQuiescenceInput,
|
|
21
26
|
RecoverDispatchInput,
|
|
22
27
|
RecoverDispatchResult,
|
|
23
28
|
} from "./types";
|
|
@@ -30,8 +35,10 @@ export type SessionStateActivityOverrides = Partial<{
|
|
|
30
35
|
countQueuedTurns: typeof countQueuedTurns;
|
|
31
36
|
getSessionEvent: typeof getSessionEvent;
|
|
32
37
|
getSessionTurnForAttempt: typeof getSessionTurnForAttempt;
|
|
38
|
+
expireSessionHumanInputRequest: typeof expireSessionHumanInputRequest;
|
|
33
39
|
requireSession: typeof requireSession;
|
|
34
40
|
settleSessionIdleWithParentOutbox: typeof settleSessionIdleWithParentOutbox;
|
|
41
|
+
markSessionAttemptQuiesced: typeof markSessionAttemptQuiesced;
|
|
35
42
|
publishDurableSessionEvents: typeof publishDurableSessionEvents;
|
|
36
43
|
deliverFailedChildTurnToParent: typeof deliverFailedChildTurnToParent;
|
|
37
44
|
notifyParentOfChildIdle: typeof notifyParentOfChildIdle;
|
|
@@ -57,9 +64,13 @@ export function createSessionStateActivities(
|
|
|
57
64
|
const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns;
|
|
58
65
|
const getSessionEventFn = overrides.getSessionEvent ?? getSessionEvent;
|
|
59
66
|
const getSessionTurnForAttemptFn = overrides.getSessionTurnForAttempt ?? getSessionTurnForAttempt;
|
|
67
|
+
const expireSessionHumanInputRequestFn =
|
|
68
|
+
overrides.expireSessionHumanInputRequest ?? expireSessionHumanInputRequest;
|
|
60
69
|
const requireSessionFn = overrides.requireSession ?? requireSession;
|
|
61
70
|
const settleSessionIdleWithParentOutboxFn =
|
|
62
71
|
overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
|
|
72
|
+
const markSessionAttemptQuiescedFn =
|
|
73
|
+
overrides.markSessionAttemptQuiesced ?? markSessionAttemptQuiesced;
|
|
63
74
|
const publishDurableSessionEventsFn =
|
|
64
75
|
overrides.publishDurableSessionEvents ?? publishDurableSessionEvents;
|
|
65
76
|
const deliverFailedChildTurnToParentFn =
|
|
@@ -117,6 +128,19 @@ export function createSessionStateActivities(
|
|
|
117
128
|
input: SettleSessionInterruptionsInput,
|
|
118
129
|
): Promise<{ action: "paused" | "continue" | "stale" }> {
|
|
119
130
|
const { db, bus, observability } = await services();
|
|
131
|
+
if (input.phase === "attempt_quiesced") {
|
|
132
|
+
// Replay compatibility only: v1 histories scheduled this idempotent
|
|
133
|
+
// fallback after WAIT_CANCELLATION_COMPLETED. Receipt-gated v2 workflows
|
|
134
|
+
// never call it; runAgentTurn writes immediately after its hard fence.
|
|
135
|
+
const events = await markSessionAttemptQuiescedFn(db, {
|
|
136
|
+
workspaceId: input.workspaceId,
|
|
137
|
+
sessionId: input.sessionId,
|
|
138
|
+
attemptId: input.attemptId,
|
|
139
|
+
temporalWorkflowId: input.workflowId,
|
|
140
|
+
});
|
|
141
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, events);
|
|
142
|
+
return { action: "stale" };
|
|
143
|
+
}
|
|
120
144
|
const applied = await settleSessionAttemptInterruptionsFn(
|
|
121
145
|
db,
|
|
122
146
|
input.workspaceId,
|
|
@@ -130,6 +154,38 @@ export function createSessionStateActivities(
|
|
|
130
154
|
return { action: applied.action };
|
|
131
155
|
}
|
|
132
156
|
|
|
157
|
+
/** Persist an exact activity-owned physical-quiescence proof through the
|
|
158
|
+
* workflow control-activity retry policy. The DB transaction remains the
|
|
159
|
+
* sole receipt/wake authority; duplicate signals and activity retries reuse
|
|
160
|
+
* its attempt-scoped idempotency key. */
|
|
161
|
+
async function persistSessionAttemptQuiescence(
|
|
162
|
+
input: PersistSessionAttemptQuiescenceInput,
|
|
163
|
+
): Promise<void> {
|
|
164
|
+
const { db, bus, observability } = await services();
|
|
165
|
+
const events = await markSessionAttemptQuiescedFn(db, {
|
|
166
|
+
accountId: input.accountId,
|
|
167
|
+
workspaceId: input.workspaceId,
|
|
168
|
+
sessionId: input.sessionId,
|
|
169
|
+
attemptId: input.attemptId,
|
|
170
|
+
temporalWorkflowId: input.workflowId,
|
|
171
|
+
temporalWorkflowRunId: input.workflowRunId,
|
|
172
|
+
temporalActivityId: input.activityId,
|
|
173
|
+
allowUninterrupted: true,
|
|
174
|
+
});
|
|
175
|
+
try {
|
|
176
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, events);
|
|
177
|
+
} catch (error) {
|
|
178
|
+
// The receipt and exact workflow wake already committed atomically in
|
|
179
|
+
// Postgres. NATS is best-effort live fanout and must not keep this
|
|
180
|
+
// control activity retrying or delay receipt-gated admission.
|
|
181
|
+
observability.error("session-attempt quiescence event fanout failed", {
|
|
182
|
+
"opengeni.session_id": input.sessionId,
|
|
183
|
+
"opengeni.attempt_id": input.attemptId,
|
|
184
|
+
error: error instanceof Error ? error.message : String(error),
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
133
189
|
/**
|
|
134
190
|
* Recover the same current inference when its worker dies without completing
|
|
135
191
|
* a graceful checkpoint (heartbeat timeout, SIGKILL, OOM, or node loss).
|
|
@@ -177,6 +233,20 @@ export function createSessionStateActivities(
|
|
|
177
233
|
return peek;
|
|
178
234
|
}
|
|
179
235
|
|
|
236
|
+
async function expireSessionHumanInput(
|
|
237
|
+
input: ExpireSessionHumanInputInput,
|
|
238
|
+
): Promise<ExpireSessionHumanInputResult> {
|
|
239
|
+
const { db, bus } = await services();
|
|
240
|
+
const result = await expireSessionHumanInputRequestFn(db, input);
|
|
241
|
+
if (result.action === "not_found") return { action: "not_found" };
|
|
242
|
+
if (result.events.length > 0) {
|
|
243
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
244
|
+
}
|
|
245
|
+
return {
|
|
246
|
+
action: result.request.status === "expired" ? "expired" : "stale",
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
180
250
|
async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
|
|
181
251
|
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
182
252
|
const settled = await settleSessionIdleWithParentOutboxFn(
|
|
@@ -207,8 +277,10 @@ export function createSessionStateActivities(
|
|
|
207
277
|
return {
|
|
208
278
|
failSessionAttempt,
|
|
209
279
|
settleSessionInterruptions,
|
|
280
|
+
persistSessionAttemptQuiescence,
|
|
210
281
|
recoverDispatch,
|
|
211
282
|
peekSessionWork,
|
|
283
|
+
expireSessionHumanInput,
|
|
212
284
|
markSessionIdle,
|
|
213
285
|
};
|
|
214
286
|
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import {
|
|
2
|
+
startExpiringMaterialRenewalLoop,
|
|
3
|
+
type ExpiringMaterialRenewalController,
|
|
4
|
+
} from "./expiring-material-renewal";
|
|
5
|
+
import type { MintedSandboxToolspaceToken } from "./environment";
|
|
6
|
+
|
|
7
|
+
export const TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS = 30 * 60_000;
|
|
8
|
+
export const TOOLSPACE_TOKEN_EXPIRY_LEAD_MS = 5 * 60_000;
|
|
9
|
+
export const TOOLSPACE_TOKEN_MIN_REFRESH_MS = 5_000;
|
|
10
|
+
export const TOOLSPACE_TOKEN_MAX_RETRY_MS = 5 * 60_000;
|
|
11
|
+
|
|
12
|
+
export type ToolspaceTokenRenewalController = ExpiringMaterialRenewalController;
|
|
13
|
+
|
|
14
|
+
export type ToolspaceTokenRenewalOptions = {
|
|
15
|
+
initialExpiresAt: Date;
|
|
16
|
+
mint: () => Promise<MintedSandboxToolspaceToken | undefined>;
|
|
17
|
+
write: (material: MintedSandboxToolspaceToken) => Promise<void>;
|
|
18
|
+
now?: () => number;
|
|
19
|
+
schedule?: (callback: () => void, delayMs: number) => unknown;
|
|
20
|
+
clearSchedule?: (timer: unknown) => void;
|
|
21
|
+
onSuccess?: (result: { nextDelayMs: number }) => void;
|
|
22
|
+
onFailure?: (failure: { retryDelayMs: number; errorClass: string }) => void;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
class ToolspaceTokenRenewalError extends Error {
|
|
26
|
+
override readonly name = "ToolspaceTokenRenewalError";
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function startToolspaceTokenRenewalLoop(
|
|
30
|
+
options: ToolspaceTokenRenewalOptions,
|
|
31
|
+
): ToolspaceTokenRenewalController {
|
|
32
|
+
return startExpiringMaterialRenewalLoop<MintedSandboxToolspaceToken>({
|
|
33
|
+
initialExpiresAt: options.initialExpiresAt,
|
|
34
|
+
resolve: async () => {
|
|
35
|
+
const minted = await options.mint();
|
|
36
|
+
if (!minted) {
|
|
37
|
+
throw new ToolspaceTokenRenewalError("Toolspace token mint is no longer available");
|
|
38
|
+
}
|
|
39
|
+
return minted;
|
|
40
|
+
},
|
|
41
|
+
write: options.write,
|
|
42
|
+
expiresAt: (material) => material.expiresAt,
|
|
43
|
+
policy: {
|
|
44
|
+
defaultRefreshMs: TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS,
|
|
45
|
+
expiryLeadMs: TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
|
|
46
|
+
minRefreshMs: TOOLSPACE_TOKEN_MIN_REFRESH_MS,
|
|
47
|
+
maxRetryMs: TOOLSPACE_TOKEN_MAX_RETRY_MS,
|
|
48
|
+
},
|
|
49
|
+
...(options.now ? { now: options.now } : {}),
|
|
50
|
+
...(options.schedule ? { schedule: options.schedule } : {}),
|
|
51
|
+
...(options.clearSchedule ? { clearSchedule: options.clearSchedule } : {}),
|
|
52
|
+
...(options.onSuccess
|
|
53
|
+
? { onSuccess: ({ nextDelayMs }) => options.onSuccess?.({ nextDelayMs }) }
|
|
54
|
+
: {}),
|
|
55
|
+
...(options.onFailure ? { onFailure: options.onFailure } : {}),
|
|
56
|
+
});
|
|
57
|
+
}
|
package/src/activities/types.ts
CHANGED
|
@@ -36,6 +36,21 @@ export type SignalCodexCapacityWorkflow = (input: {
|
|
|
36
36
|
wakeRevision: number;
|
|
37
37
|
}) => Promise<void>;
|
|
38
38
|
|
|
39
|
+
/** Exact activity-owned proof that the hard sandbox/tool fence physically
|
|
40
|
+
* drained. This is delivery evidence only: the workflow still validates the
|
|
41
|
+
* persisted attempt dispatch and commits the authoritative Postgres receipt. */
|
|
42
|
+
export type SessionAttemptQuiescenceProof = {
|
|
43
|
+
accountId: string;
|
|
44
|
+
workspaceId: string;
|
|
45
|
+
sessionId: string;
|
|
46
|
+
attemptId: string;
|
|
47
|
+
workflowId: string;
|
|
48
|
+
workflowRunId: string;
|
|
49
|
+
activityId: string;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
export type SignalSessionAttemptQuiesced = (input: SessionAttemptQuiescenceProof) => Promise<void>;
|
|
53
|
+
|
|
39
54
|
export type ActivityServices = {
|
|
40
55
|
settings: Settings;
|
|
41
56
|
db: Database;
|
|
@@ -45,6 +60,9 @@ export type ActivityServices = {
|
|
|
45
60
|
documentServices: DocumentServices;
|
|
46
61
|
observability: Observability;
|
|
47
62
|
wakeSessionWorkflow: WakeSessionWorkflowSignal | null;
|
|
63
|
+
/** Durable signalWithStart fallback used only after the activity's direct
|
|
64
|
+
* physical-quiescence receipt write exhausts its bounded DB retries. */
|
|
65
|
+
signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced | null;
|
|
48
66
|
/** Revision-carrying capacity nudge; generic outbox repair is also sufficient. */
|
|
49
67
|
signalCodexCapacityWorkflow?: SignalCodexCapacityWorkflow | null;
|
|
50
68
|
// §7.5 P3 — host-entitlements port, the WORKER half of the same seam the API
|
|
@@ -61,7 +79,7 @@ export type ActivityServices = {
|
|
|
61
79
|
// admission and metering are separate operations, and only metering carries
|
|
62
80
|
// the idempotency key.
|
|
63
81
|
entitlements?: EntitlementsPort | null;
|
|
64
|
-
// §7.6
|
|
82
|
+
// §7.6 connection-credential provider — host connection-credential provider, the WORKER half of the
|
|
65
83
|
// federated-connection boundary. When set, the run's per-run credential mint
|
|
66
84
|
// delegates to the host instead of self-minting from `settings`:
|
|
67
85
|
// - `gitCredentials` REPLACES `createGitHubAppInstallationToken(settings,…)`
|
|
@@ -72,7 +90,7 @@ export type ActivityServices = {
|
|
|
72
90
|
// self-mint for THAT leg. null/undefined (standalone default) → both legs
|
|
73
91
|
// self-mint byte-for-byte as today.
|
|
74
92
|
//
|
|
75
|
-
//
|
|
93
|
+
// workspace-scope cross-check CROSS-CHECK: a provider echoes the `workspaceId` it scoped the
|
|
76
94
|
// credential to; the consuming activity ASSERTS agreement with the run's
|
|
77
95
|
// workspace BEFORE injecting `GH_TOKEN` (or applying decrypted values). A host
|
|
78
96
|
// mapping bug returning tenant B's creds for a tenant-A run is caught here.
|
|
@@ -123,8 +141,16 @@ export type SettleSessionInterruptionsInput = {
|
|
|
123
141
|
sessionId: string;
|
|
124
142
|
attemptId: string;
|
|
125
143
|
workflowId: string;
|
|
144
|
+
/**
|
|
145
|
+
* Replay-only compatibility for session workflow histories created before
|
|
146
|
+
* the receipt-gated cancellation v2 patch. New histories never send this
|
|
147
|
+
* phase; the exact activity writes the authoritative receipt itself.
|
|
148
|
+
*/
|
|
149
|
+
phase?: "logical" | "attempt_quiesced";
|
|
126
150
|
};
|
|
127
151
|
|
|
152
|
+
export type PersistSessionAttemptQuiescenceInput = SessionAttemptQuiescenceProof;
|
|
153
|
+
|
|
128
154
|
export type FailSessionAttemptInput = {
|
|
129
155
|
accountId: string;
|
|
130
156
|
workspaceId: string;
|
|
@@ -160,6 +186,17 @@ export type PeekSessionWorkInput = {
|
|
|
160
186
|
sessionId: string;
|
|
161
187
|
};
|
|
162
188
|
|
|
189
|
+
export type ExpireSessionHumanInputInput = {
|
|
190
|
+
accountId: string;
|
|
191
|
+
workspaceId: string;
|
|
192
|
+
sessionId: string;
|
|
193
|
+
requestId: string;
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
export type ExpireSessionHumanInputResult = {
|
|
197
|
+
action: "expired" | "stale" | "not_found";
|
|
198
|
+
};
|
|
199
|
+
|
|
163
200
|
export type MarkSessionIdleInput = {
|
|
164
201
|
workspaceId: string;
|
|
165
202
|
sessionId: string;
|
|
@@ -220,10 +257,11 @@ type ClaimedRunAgentTurnResult = {
|
|
|
220
257
|
// persisted in Postgres and reconstructed after workflow/worker restart.
|
|
221
258
|
// The workflow must not call maybeContinueGoal while this waiter is active.
|
|
222
259
|
capacityWait?: CodexCapacityWaitRef;
|
|
223
|
-
//
|
|
224
|
-
//
|
|
225
|
-
//
|
|
226
|
-
// prompt
|
|
260
|
+
// This execution reached a durable terminal-for-now boundary (for example,
|
|
261
|
+
// maintenance could not run or same-turn context recovery failed). End this
|
|
262
|
+
// workflow run without synthesizing another goal continuation from unchanged
|
|
263
|
+
// state. A later prompt/control/new-update wake may retry through normal claim
|
|
264
|
+
// ordering.
|
|
227
265
|
deferredUntilWake?: boolean;
|
|
228
266
|
};
|
|
229
267
|
|