@opengeni/worker-bundle 0.7.4 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BJUW5LCM.js +325 -0
- package/dist/chunk-BJUW5LCM.js.map +1 -0
- package/dist/host-export.d.ts +70 -0
- package/dist/host-export.js +7 -0
- package/dist/host-export.js.map +1 -0
- package/dist/index.d.ts +98 -6
- package/dist/index.js +2853 -834
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +12358 -0
- package/package.json +18 -12
- package/src/activities/agent-turn.ts +2052 -554
- package/src/activities/capabilities.ts +9 -2
- package/src/activities/codex-capacity.ts +6 -11
- package/src/activities/codex-rotation.ts +75 -86
- package/src/activities/context-compaction.ts +104 -24
- package/src/activities/environment.ts +367 -92
- package/src/activities/expiring-material-renewal.ts +143 -0
- package/src/activities/goals.ts +4 -0
- package/src/activities/mcp-credentials.ts +38 -0
- package/src/activities/packs.ts +115 -2
- package/src/activities/parent-wake.ts +48 -38
- package/src/activities/run-credential-renewal.ts +70 -0
- package/src/activities/run-credentials.ts +132 -0
- package/src/activities/run-input.ts +23 -0
- package/src/activities/sandbox-lease.ts +1 -1
- package/src/activities/scheduled-tasks.ts +21 -0
- package/src/activities/session-state.ts +83 -12
- package/src/activities/streaming.ts +1 -0
- package/src/activities/toolspace-token-renewal.ts +57 -0
- package/src/activities/types.ts +44 -6
- package/src/activities/workspace-capture.ts +113 -93
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +416 -0
- package/src/host-export.ts +22 -0
- package/src/http.ts +77 -28
- package/src/index.ts +360 -156
- package/src/sandbox-resume.ts +47 -1
- package/src/sandbox-routing.ts +1 -1
- package/src/worker-service-lifecycle.ts +104 -0
- package/src/workflows/activities.ts +13 -7
- package/src/workflows/session.ts +219 -23
- package/src/workflows.ts +1 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
export type ExpiringMaterialRenewalPolicy = {
|
|
2
|
+
defaultRefreshMs: number;
|
|
3
|
+
expiryLeadMs: number;
|
|
4
|
+
minRefreshMs: number;
|
|
5
|
+
maxRetryMs: number;
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
type RenewalTimer = unknown;
|
|
9
|
+
|
|
10
|
+
export type ExpiringMaterialRenewalController = {
|
|
11
|
+
refreshNow(): Promise<void>;
|
|
12
|
+
stop(): Promise<void>;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export type ExpiringMaterialRenewalOptions<T> = {
|
|
16
|
+
initialExpiresAt: Date | null;
|
|
17
|
+
resolve: () => Promise<T>;
|
|
18
|
+
write: (material: T) => Promise<void>;
|
|
19
|
+
expiresAt: (material: T) => Date | null;
|
|
20
|
+
policy: ExpiringMaterialRenewalPolicy;
|
|
21
|
+
now?: () => number;
|
|
22
|
+
schedule?: (callback: () => void, delayMs: number) => RenewalTimer;
|
|
23
|
+
clearSchedule?: (timer: RenewalTimer) => void;
|
|
24
|
+
onSuccess?: (result: { material: T; nextDelayMs: number }) => void;
|
|
25
|
+
onFailure?: (failure: { retryDelayMs: number; errorClass: string }) => void;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export function nextExpiringMaterialRenewalDelay(
|
|
29
|
+
expiresAt: Date | null,
|
|
30
|
+
policy: ExpiringMaterialRenewalPolicy,
|
|
31
|
+
nowMs = Date.now(),
|
|
32
|
+
): number {
|
|
33
|
+
if (!expiresAt || !Number.isFinite(expiresAt.getTime())) {
|
|
34
|
+
return policy.defaultRefreshMs;
|
|
35
|
+
}
|
|
36
|
+
return Math.min(
|
|
37
|
+
policy.defaultRefreshMs,
|
|
38
|
+
Math.max(policy.minRefreshMs, expiresAt.getTime() - nowMs - policy.expiryLeadMs),
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Attempt-owned single-flight renewal for expiring sandbox material.
|
|
44
|
+
*
|
|
45
|
+
* A late resolver cannot write after stop. A physical sandbox write that already
|
|
46
|
+
* started is drained so turn settlement cannot race a stale mutation.
|
|
47
|
+
*/
|
|
48
|
+
export function startExpiringMaterialRenewalLoop<T>(
|
|
49
|
+
options: ExpiringMaterialRenewalOptions<T>,
|
|
50
|
+
): ExpiringMaterialRenewalController {
|
|
51
|
+
const now = options.now ?? Date.now;
|
|
52
|
+
const schedule =
|
|
53
|
+
options.schedule ??
|
|
54
|
+
((callback: () => void, delayMs: number): ReturnType<typeof setTimeout> => {
|
|
55
|
+
const timer = setTimeout(callback, delayMs);
|
|
56
|
+
timer.unref?.();
|
|
57
|
+
return timer;
|
|
58
|
+
});
|
|
59
|
+
const clearSchedule =
|
|
60
|
+
options.clearSchedule ??
|
|
61
|
+
((timer: RenewalTimer): void => clearTimeout(timer as ReturnType<typeof setTimeout>));
|
|
62
|
+
let stopped = false;
|
|
63
|
+
let timer: RenewalTimer | null = null;
|
|
64
|
+
let inFlight: Promise<void> | null = null;
|
|
65
|
+
let writeInFlight: Promise<void> | null = null;
|
|
66
|
+
let retryDelayMs = options.policy.minRefreshMs;
|
|
67
|
+
|
|
68
|
+
const clearTimer = (): void => {
|
|
69
|
+
if (timer === null) return;
|
|
70
|
+
clearSchedule(timer);
|
|
71
|
+
timer = null;
|
|
72
|
+
};
|
|
73
|
+
const scheduleRefresh = (delayMs: number): void => {
|
|
74
|
+
if (stopped) return;
|
|
75
|
+
clearTimer();
|
|
76
|
+
timer = schedule(() => {
|
|
77
|
+
timer = null;
|
|
78
|
+
void refreshNow();
|
|
79
|
+
}, delayMs);
|
|
80
|
+
};
|
|
81
|
+
const performRefresh = async (): Promise<void> => {
|
|
82
|
+
try {
|
|
83
|
+
const material = await options.resolve();
|
|
84
|
+
if (stopped) return;
|
|
85
|
+
const write = options.write(material);
|
|
86
|
+
writeInFlight = write;
|
|
87
|
+
try {
|
|
88
|
+
await write;
|
|
89
|
+
} finally {
|
|
90
|
+
if (writeInFlight === write) writeInFlight = null;
|
|
91
|
+
}
|
|
92
|
+
if (stopped) return;
|
|
93
|
+
retryDelayMs = options.policy.minRefreshMs;
|
|
94
|
+
const nextDelayMs = nextExpiringMaterialRenewalDelay(
|
|
95
|
+
options.expiresAt(material),
|
|
96
|
+
options.policy,
|
|
97
|
+
now(),
|
|
98
|
+
);
|
|
99
|
+
try {
|
|
100
|
+
options.onSuccess?.({ material, nextDelayMs });
|
|
101
|
+
} catch {
|
|
102
|
+
// Observability never owns credential liveness.
|
|
103
|
+
}
|
|
104
|
+
scheduleRefresh(nextDelayMs);
|
|
105
|
+
} catch (error) {
|
|
106
|
+
if (stopped) return;
|
|
107
|
+
const scheduledRetryMs = retryDelayMs;
|
|
108
|
+
retryDelayMs = Math.min(retryDelayMs * 2, options.policy.maxRetryMs);
|
|
109
|
+
try {
|
|
110
|
+
options.onFailure?.({
|
|
111
|
+
retryDelayMs: scheduledRetryMs,
|
|
112
|
+
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
113
|
+
});
|
|
114
|
+
} catch {
|
|
115
|
+
// Observability never owns credential liveness.
|
|
116
|
+
}
|
|
117
|
+
scheduleRefresh(scheduledRetryMs);
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
const refreshNow = (): Promise<void> => {
|
|
121
|
+
if (stopped) return Promise.resolve();
|
|
122
|
+
if (inFlight) return inFlight;
|
|
123
|
+
clearTimer();
|
|
124
|
+
const operation = performRefresh();
|
|
125
|
+
inFlight = operation;
|
|
126
|
+
void operation.finally(() => {
|
|
127
|
+
if (inFlight === operation) inFlight = null;
|
|
128
|
+
});
|
|
129
|
+
return operation;
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
scheduleRefresh(
|
|
133
|
+
nextExpiringMaterialRenewalDelay(options.initialExpiresAt, options.policy, now()),
|
|
134
|
+
);
|
|
135
|
+
return {
|
|
136
|
+
refreshNow,
|
|
137
|
+
stop: async () => {
|
|
138
|
+
stopped = true;
|
|
139
|
+
clearTimer();
|
|
140
|
+
await writeInFlight?.catch(() => undefined);
|
|
141
|
+
},
|
|
142
|
+
};
|
|
143
|
+
}
|
package/src/activities/goals.ts
CHANGED
|
@@ -190,6 +190,10 @@ export function createGoalActivities(services: () => Promise<ActivityServices>)
|
|
|
190
190
|
unit: "run",
|
|
191
191
|
sourceResourceType: "session_system_update",
|
|
192
192
|
sourceResourceId: update.update.id,
|
|
193
|
+
sessionId: input.sessionId,
|
|
194
|
+
initiator: { kind: "service", subjectId: "goal-continuation" },
|
|
195
|
+
initiatorContext: { goalId: decision.goal.id },
|
|
196
|
+
origin: "goal",
|
|
193
197
|
idempotencyKey: `agent_run.created:goal:${input.workspaceId}:${update.update.id}`,
|
|
194
198
|
});
|
|
195
199
|
return { action: "continue" };
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { Settings } from "@opengeni/config";
|
|
2
|
+
import type { ConnectionCredentialsPort, SessionTurn } from "@opengeni/contracts";
|
|
3
|
+
import {
|
|
4
|
+
buildConnectionTokenResolver,
|
|
5
|
+
buildHostConnectionTokenResolver,
|
|
6
|
+
type Database,
|
|
7
|
+
type ResolveConnectionCredentialInput,
|
|
8
|
+
type ResolveConnectionCredentialResult,
|
|
9
|
+
} from "@opengeni/db";
|
|
10
|
+
|
|
11
|
+
export function connectionTokenResolverForTurn(input: {
|
|
12
|
+
db: Database;
|
|
13
|
+
settings: Settings;
|
|
14
|
+
connectionCredentials?: ConnectionCredentialsPort | null;
|
|
15
|
+
accountId: string;
|
|
16
|
+
workspaceId: string;
|
|
17
|
+
sessionId: string;
|
|
18
|
+
rootSessionId: string;
|
|
19
|
+
attemptId: string;
|
|
20
|
+
turn: SessionTurn;
|
|
21
|
+
}): (request: ResolveConnectionCredentialInput) => Promise<ResolveConnectionCredentialResult> {
|
|
22
|
+
const hostResolver = input.connectionCredentials?.mcpCredentials;
|
|
23
|
+
if (!hostResolver) {
|
|
24
|
+
return buildConnectionTokenResolver(input.db, input.settings);
|
|
25
|
+
}
|
|
26
|
+
return buildHostConnectionTokenResolver(hostResolver, {
|
|
27
|
+
accountId: input.accountId,
|
|
28
|
+
workspaceId: input.workspaceId,
|
|
29
|
+
sessionId: input.sessionId,
|
|
30
|
+
rootSessionId: input.rootSessionId,
|
|
31
|
+
turnId: input.turn.id,
|
|
32
|
+
attemptId: input.attemptId,
|
|
33
|
+
executionGeneration: input.turn.executionGeneration,
|
|
34
|
+
initiator: input.turn.initiator,
|
|
35
|
+
initiatorContext: input.turn.initiatorContext,
|
|
36
|
+
surface: "model",
|
|
37
|
+
});
|
|
38
|
+
}
|
package/src/activities/packs.ts
CHANGED
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
import type { Settings } from "@opengeni/config";
|
|
2
2
|
import { CapabilityPack } from "@opengeni/contracts";
|
|
3
|
-
import {
|
|
4
|
-
|
|
3
|
+
import {
|
|
4
|
+
getWorkspace,
|
|
5
|
+
getWorkspacePack,
|
|
6
|
+
listCapabilityInstallations,
|
|
7
|
+
listPackInstallations,
|
|
8
|
+
type Database,
|
|
9
|
+
} from "@opengeni/db";
|
|
10
|
+
import {
|
|
11
|
+
isSkillLibraryEntryId,
|
|
12
|
+
loadSkillLibrarySkill,
|
|
13
|
+
type EffectiveSkillSelection,
|
|
14
|
+
type PackSkill,
|
|
15
|
+
} from "@opengeni/runtime";
|
|
5
16
|
|
|
6
17
|
/**
|
|
7
18
|
* The pack-scoped runtime for a workspace: the sandbox image its sessions run
|
|
@@ -15,6 +26,19 @@ export type WorkspacePackRuntime = {
|
|
|
15
26
|
|
|
16
27
|
const emptyPackRuntime: WorkspacePackRuntime = { sandboxImage: null, skills: [] };
|
|
17
28
|
|
|
29
|
+
export type WorkspaceSkillLibraryRuntime = {
|
|
30
|
+
skillLibrarySkills: PackSkill[];
|
|
31
|
+
skillLibrarySelections: EffectiveSkillSelection[];
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
type SkillLibraryInstallationMetadata = {
|
|
35
|
+
libraryId: string;
|
|
36
|
+
libraryVersion: string;
|
|
37
|
+
contentSha256: string;
|
|
38
|
+
sourceCommit: string;
|
|
39
|
+
provenance: string;
|
|
40
|
+
};
|
|
41
|
+
|
|
18
42
|
/**
|
|
19
43
|
* Resolves the pack-scoped runtime from the workspace's active pack
|
|
20
44
|
* installations. Only registered (manifest-backed) packs can contribute a
|
|
@@ -45,6 +69,95 @@ export async function resolveWorkspacePackRuntime(
|
|
|
45
69
|
return workspacePackRuntimeFromPacks(packs);
|
|
46
70
|
}
|
|
47
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Resolves active immutable curated skills separately from pack skills. The
|
|
74
|
+
* capability installation is the workspace selection boundary; it carries
|
|
75
|
+
* only secret-free exact identity metadata, and the runtime loader verifies
|
|
76
|
+
* the pinned artifact hash before any content is materialized.
|
|
77
|
+
*/
|
|
78
|
+
export async function resolveWorkspaceSkillLibraryRuntime(
|
|
79
|
+
db: Database,
|
|
80
|
+
workspaceId: string,
|
|
81
|
+
): Promise<WorkspaceSkillLibraryRuntime> {
|
|
82
|
+
const installations = await listCapabilityInstallations(db, workspaceId);
|
|
83
|
+
const activeSkills = installations
|
|
84
|
+
.filter(
|
|
85
|
+
(installation) =>
|
|
86
|
+
installation.status === "active" &&
|
|
87
|
+
installation.kind === "skill" &&
|
|
88
|
+
installation.capabilityId.startsWith("skill:") &&
|
|
89
|
+
isSkillLibraryEntryId(installation.capabilityId.slice("skill:".length)),
|
|
90
|
+
)
|
|
91
|
+
.sort((a, b) => a.capabilityId.localeCompare(b.capabilityId));
|
|
92
|
+
const skillLibrarySkills: PackSkill[] = [];
|
|
93
|
+
const skillLibrarySelections: EffectiveSkillSelection[] = [];
|
|
94
|
+
for (const installation of activeSkills) {
|
|
95
|
+
const metadata = parseSkillLibraryInstallationMetadata(
|
|
96
|
+
installation.metadata,
|
|
97
|
+
installation.capabilityId,
|
|
98
|
+
);
|
|
99
|
+
if (installation.capabilityId !== `skill:${metadata.libraryId}`) {
|
|
100
|
+
throw new Error(
|
|
101
|
+
`Skill installation id does not match its library entry: ${installation.capabilityId} -> ${metadata.libraryId}`,
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
const loaded = loadSkillLibrarySkill(metadata.libraryId, metadata.libraryVersion);
|
|
105
|
+
if (installation.config.version !== metadata.libraryVersion) {
|
|
106
|
+
throw new Error(
|
|
107
|
+
`Skill installation version mismatch for ${metadata.libraryId}@${metadata.libraryVersion}: config has ${String(installation.config.version)}`,
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
if (loaded.entry.contentSha256 !== metadata.contentSha256) {
|
|
111
|
+
throw new Error(
|
|
112
|
+
`Skill installation hash mismatch for ${metadata.libraryId}@${metadata.libraryVersion}: expected ${metadata.contentSha256}, catalog has ${loaded.entry.contentSha256}`,
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
if (
|
|
116
|
+
loaded.entry.sourceCommit !== metadata.sourceCommit ||
|
|
117
|
+
loaded.entry.provenance !== metadata.provenance
|
|
118
|
+
) {
|
|
119
|
+
throw new Error(
|
|
120
|
+
`Skill installation provenance mismatch for ${metadata.libraryId}@${metadata.libraryVersion}; re-enable the skill from the current catalog`,
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
skillLibrarySkills.push({
|
|
124
|
+
name: loaded.skill.name,
|
|
125
|
+
description: loaded.skill.description,
|
|
126
|
+
files: loaded.skill.files.map((file) => ({ path: file.path, content: file.content })),
|
|
127
|
+
});
|
|
128
|
+
skillLibrarySelections.push({
|
|
129
|
+
id: loaded.entry.id,
|
|
130
|
+
name: loaded.entry.name,
|
|
131
|
+
source: "library",
|
|
132
|
+
version: loaded.entry.version,
|
|
133
|
+
contentSha256: loaded.entry.contentSha256,
|
|
134
|
+
reason: "enabled workspace capability installation",
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
return { skillLibrarySkills, skillLibrarySelections };
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function parseSkillLibraryInstallationMetadata(
|
|
141
|
+
metadata: Record<string, unknown>,
|
|
142
|
+
capabilityId: string,
|
|
143
|
+
): SkillLibraryInstallationMetadata {
|
|
144
|
+
const libraryId = stringMetadata(metadata.libraryId);
|
|
145
|
+
const libraryVersion = stringMetadata(metadata.libraryVersion);
|
|
146
|
+
const contentSha256 = stringMetadata(metadata.contentSha256);
|
|
147
|
+
const sourceCommit = stringMetadata(metadata.sourceCommit);
|
|
148
|
+
const provenance = stringMetadata(metadata.provenance);
|
|
149
|
+
if (!libraryId || !libraryVersion || !contentSha256 || !sourceCommit || !provenance) {
|
|
150
|
+
throw new Error(
|
|
151
|
+
`Skill installation ${capabilityId} is missing immutable library metadata; re-enable the skill from the current catalog`,
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
return { libraryId, libraryVersion, contentSha256, sourceCommit, provenance };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function stringMetadata(value: unknown): string | null {
|
|
158
|
+
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
|
159
|
+
}
|
|
160
|
+
|
|
48
161
|
/**
|
|
49
162
|
* Pure composition rule for enabled pack manifests. v1 keeps this small by
|
|
50
163
|
* design: at most one enabled pack may declare a sandbox image (no image
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
addSessionSystemUpdateWithSourceMutation,
|
|
7
7
|
claimPendingSessionSystemUpdateOutbox,
|
|
8
8
|
claimPendingSessionWorkflowWakes,
|
|
9
|
+
getSessionSystemUpdateOutboxByDedupeKey,
|
|
9
10
|
getOrCreateSessionSystemUpdateOutbox,
|
|
10
11
|
markSessionWorkflowWakeFailed,
|
|
11
12
|
markSessionSystemUpdateOutboxDeliveredInTransaction,
|
|
@@ -33,36 +34,15 @@ export type ReconcileSessionWorkflowWakeOverrides = Partial<{
|
|
|
33
34
|
}>;
|
|
34
35
|
|
|
35
36
|
/**
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
* terminal episode: the idempotency key is the child's current lastSequence,
|
|
40
|
-
* which advances every time the child does work, so a retry of the same
|
|
41
|
-
* terminal transition (activity retry, the workflow's idle re-check, the
|
|
42
|
-
* runAgentTurn-failed path overlapping a workflow-level wake) is deduped while
|
|
43
|
-
* a genuinely new idle-after-work episode notifies again. The parent receives
|
|
44
|
-
* one typed internal update; signalling its workflow ensures an idle parent can
|
|
45
|
-
* coalesce and process all pending updates in one inference. Failures here never
|
|
46
|
-
* fail the child because the durable outbox remains retryable.
|
|
47
|
-
*
|
|
48
|
-
* Lives in its own module so both the session-state terminal activities
|
|
49
|
-
* (markSessionIdle / failSession) and runAgentTurn's in-turn failure path can
|
|
50
|
-
* call it without a circular import between those two activity modules.
|
|
37
|
+
* Enrich and deliver the durable idle-boundary row committed by
|
|
38
|
+
* settleSessionIdleWithParentOutbox. Idle has no single owning turn, so its
|
|
39
|
+
* stable episode identity is the newest non-status event sequence.
|
|
51
40
|
*/
|
|
52
|
-
export async function
|
|
41
|
+
export async function notifyParentOfChildIdle(
|
|
53
42
|
svc: NotifyServices,
|
|
54
43
|
workspaceId: string,
|
|
55
44
|
childSessionId: string,
|
|
56
|
-
|
|
57
|
-
// Stable identifier for the terminal episode, used as the idempotency key so
|
|
58
|
-
// the same completion never wakes the parent twice. A FAILURE passes the
|
|
59
|
-
// failed turn's id: both the in-turn wake (runAgentTurn) and the
|
|
60
|
-
// workflow-level wake (failSession, after it appends more events that would
|
|
61
|
-
// shift lastSequence) key on the same turn, so a finally-throw that turns one
|
|
62
|
-
// failure into both paths still dedupes. An idle episode has no single
|
|
63
|
-
// owning turn, so it falls back to the child's lastSequence — which advances
|
|
64
|
-
// per work batch and is stable across retries of that same idle transition.
|
|
65
|
-
episodeKey?: string | null,
|
|
45
|
+
episodeKey: string,
|
|
66
46
|
): Promise<void> {
|
|
67
47
|
try {
|
|
68
48
|
const child = await getSession(svc.db, workspaceId, childSessionId);
|
|
@@ -75,23 +55,18 @@ export async function notifyParentOfChildTerminal(
|
|
|
75
55
|
// what re-arms the loop the user just stopped. Suppress the resume nudge and
|
|
76
56
|
// tell the agent to stay paused. Paired with the goal_set reactivation
|
|
77
57
|
// guard so a nudge that slips through still cannot revive the goal.
|
|
78
|
-
const clientEventId = `child-completion:${childSessionId}:${episodeKey
|
|
79
|
-
const payload = childCompletionPayload(child, goal
|
|
58
|
+
const clientEventId = `child-completion:${childSessionId}:${episodeKey}`;
|
|
59
|
+
const payload = childCompletionPayload(child, goal);
|
|
80
60
|
const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
|
|
81
61
|
accountId: child.accountId,
|
|
82
62
|
workspaceId,
|
|
83
63
|
sourceSessionId: child.id,
|
|
84
64
|
targetSessionId: child.parentSessionId,
|
|
85
65
|
kind: "child_terminal_result",
|
|
86
|
-
classification:
|
|
87
|
-
terminalStatus === "failed"
|
|
88
|
-
? "failure"
|
|
89
|
-
: goal?.status === "paused"
|
|
90
|
-
? "action_required"
|
|
91
|
-
: "success",
|
|
66
|
+
classification: goal?.status === "paused" ? "action_required" : "success",
|
|
92
67
|
sourceId: child.id,
|
|
93
68
|
dedupeKey: clientEventId,
|
|
94
|
-
summary: childCompletionSummary(child, goal,
|
|
69
|
+
summary: childCompletionSummary(child, goal, "idle"),
|
|
95
70
|
payload,
|
|
96
71
|
lineage: { childSessionId: child.id, parentSessionId: child.parentSessionId },
|
|
97
72
|
});
|
|
@@ -102,8 +77,44 @@ export async function notifyParentOfChildTerminal(
|
|
|
102
77
|
} catch (error) {
|
|
103
78
|
// A durable pending outbox row survives this boundary. The global worker
|
|
104
79
|
// reaper retries it; child terminal settlement never depends on this turn.
|
|
105
|
-
svc.observability.error("Failed to wake parent session on worker
|
|
80
|
+
svc.observability.error("Failed to wake parent session on worker idle boundary", {
|
|
81
|
+
childSessionId,
|
|
82
|
+
error: error instanceof Error ? error.message : String(error),
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Deliver the exact failure row already committed by turn settlement. This
|
|
89
|
+
* layer deliberately cannot create or rewrite the row: the settlement
|
|
90
|
+
* transaction is the sole owner of the failed turn id, payload, and lineage.
|
|
91
|
+
*/
|
|
92
|
+
export async function deliverFailedChildTurnToParent(
|
|
93
|
+
svc: NotifyServices,
|
|
94
|
+
workspaceId: string,
|
|
95
|
+
childSessionId: string,
|
|
96
|
+
turnId: string,
|
|
97
|
+
): Promise<void> {
|
|
98
|
+
try {
|
|
99
|
+
const child = await getSession(svc.db, workspaceId, childSessionId);
|
|
100
|
+
if (!child || !child.parentSessionId) return;
|
|
101
|
+
const dedupeKey = `child-completion:${childSessionId}:turn:${turnId}`;
|
|
102
|
+
const outbox = await getSessionSystemUpdateOutboxByDedupeKey(svc.db, {
|
|
103
|
+
accountId: child.accountId,
|
|
104
|
+
workspaceId,
|
|
105
|
+
dedupeKey,
|
|
106
|
+
});
|
|
107
|
+
if (!outbox) {
|
|
108
|
+
throw new Error(`Committed failed-child outbox disappeared: ${dedupeKey}`);
|
|
109
|
+
}
|
|
110
|
+
if (outbox.status === "delivered") return;
|
|
111
|
+
await deliverParentSystemUpdateOutbox(svc, outbox);
|
|
112
|
+
} catch (error) {
|
|
113
|
+
// Settlement already committed the retryable outbox row. The global
|
|
114
|
+
// reconciler remains the recovery path if immediate delivery fails.
|
|
115
|
+
svc.observability.error("Failed to deliver committed child-turn failure", {
|
|
106
116
|
childSessionId,
|
|
117
|
+
turnId,
|
|
107
118
|
error: error instanceof Error ? error.message : String(error),
|
|
108
119
|
});
|
|
109
120
|
}
|
|
@@ -231,12 +242,11 @@ export async function reconcilePendingSessionWorkflowWakes(
|
|
|
231
242
|
function childCompletionPayload(
|
|
232
243
|
child: Session,
|
|
233
244
|
goal: SessionGoal | null,
|
|
234
|
-
terminalStatus: "idle" | "failed",
|
|
235
245
|
): Extract<SessionSystemUpdatePayload, { type: "child_terminal_result" }> {
|
|
236
246
|
return {
|
|
237
247
|
type: "child_terminal_result",
|
|
238
248
|
childSessionId: child.id,
|
|
239
|
-
status:
|
|
249
|
+
status: "idle",
|
|
240
250
|
...(goal
|
|
241
251
|
? {
|
|
242
252
|
goal: {
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { NormalizedRunCredentialMaterial } from "@opengeni/runtime";
|
|
2
|
+
import {
|
|
3
|
+
nextExpiringMaterialRenewalDelay,
|
|
4
|
+
startExpiringMaterialRenewalLoop,
|
|
5
|
+
type ExpiringMaterialRenewalController,
|
|
6
|
+
} from "./expiring-material-renewal";
|
|
7
|
+
|
|
8
|
+
export const RUN_CREDENTIAL_DEFAULT_REFRESH_MS = 30 * 60_000;
|
|
9
|
+
export const RUN_CREDENTIAL_EXPIRY_LEAD_MS = 5 * 60_000;
|
|
10
|
+
export const RUN_CREDENTIAL_MIN_REFRESH_MS = 5_000;
|
|
11
|
+
export const RUN_CREDENTIAL_MAX_RETRY_MS = 5 * 60_000;
|
|
12
|
+
|
|
13
|
+
export type RunCredentialRenewalController = ExpiringMaterialRenewalController;
|
|
14
|
+
|
|
15
|
+
type RenewalTimer = unknown;
|
|
16
|
+
|
|
17
|
+
export type RunCredentialRenewalOptions = {
|
|
18
|
+
initialExpiresAt: Date | null;
|
|
19
|
+
resolve: () => Promise<NormalizedRunCredentialMaterial | null>;
|
|
20
|
+
write: (material: NormalizedRunCredentialMaterial | null) => Promise<void>;
|
|
21
|
+
now?: () => number;
|
|
22
|
+
schedule?: (callback: () => void, delayMs: number) => RenewalTimer;
|
|
23
|
+
clearSchedule?: (timer: RenewalTimer) => void;
|
|
24
|
+
onSuccess?: (result: { nextDelayMs: number; authNeeded: boolean }) => void;
|
|
25
|
+
onFailure?: (failure: { retryDelayMs: number; errorClass: string }) => void;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export function nextRunCredentialRenewalDelay(expiresAt: Date | null, nowMs = Date.now()): number {
|
|
29
|
+
return nextExpiringMaterialRenewalDelay(
|
|
30
|
+
expiresAt,
|
|
31
|
+
{
|
|
32
|
+
defaultRefreshMs: RUN_CREDENTIAL_DEFAULT_REFRESH_MS,
|
|
33
|
+
expiryLeadMs: RUN_CREDENTIAL_EXPIRY_LEAD_MS,
|
|
34
|
+
minRefreshMs: RUN_CREDENTIAL_MIN_REFRESH_MS,
|
|
35
|
+
maxRetryMs: RUN_CREDENTIAL_MAX_RETRY_MS,
|
|
36
|
+
},
|
|
37
|
+
nowMs,
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Single-flight renewal; late resolves cannot mutate a stopped attempt. */
|
|
42
|
+
export function startRunCredentialRenewalLoop(
|
|
43
|
+
options: RunCredentialRenewalOptions,
|
|
44
|
+
): RunCredentialRenewalController {
|
|
45
|
+
return startExpiringMaterialRenewalLoop<NormalizedRunCredentialMaterial | null>({
|
|
46
|
+
initialExpiresAt: options.initialExpiresAt,
|
|
47
|
+
resolve: options.resolve,
|
|
48
|
+
write: options.write,
|
|
49
|
+
expiresAt: (material) => material?.expiresAt ?? null,
|
|
50
|
+
policy: {
|
|
51
|
+
defaultRefreshMs: RUN_CREDENTIAL_DEFAULT_REFRESH_MS,
|
|
52
|
+
expiryLeadMs: RUN_CREDENTIAL_EXPIRY_LEAD_MS,
|
|
53
|
+
minRefreshMs: RUN_CREDENTIAL_MIN_REFRESH_MS,
|
|
54
|
+
maxRetryMs: RUN_CREDENTIAL_MAX_RETRY_MS,
|
|
55
|
+
},
|
|
56
|
+
...(options.now ? { now: options.now } : {}),
|
|
57
|
+
...(options.schedule ? { schedule: options.schedule } : {}),
|
|
58
|
+
...(options.clearSchedule ? { clearSchedule: options.clearSchedule } : {}),
|
|
59
|
+
...(options.onSuccess
|
|
60
|
+
? {
|
|
61
|
+
onSuccess: ({ material, nextDelayMs }) =>
|
|
62
|
+
options.onSuccess?.({
|
|
63
|
+
nextDelayMs,
|
|
64
|
+
authNeeded: (material?.authNeeded.length ?? 0) > 0,
|
|
65
|
+
}),
|
|
66
|
+
}
|
|
67
|
+
: {}),
|
|
68
|
+
...(options.onFailure ? { onFailure: options.onFailure } : {}),
|
|
69
|
+
});
|
|
70
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ConnectionCredentialsPort,
|
|
3
|
+
CredentialAuthNeededPayload,
|
|
4
|
+
SandboxBackend,
|
|
5
|
+
RunCredentialsRequest,
|
|
6
|
+
Session,
|
|
7
|
+
SessionTurn,
|
|
8
|
+
} from "@opengeni/contracts";
|
|
9
|
+
import { getSessionRootId, type Database } from "@opengeni/db";
|
|
10
|
+
import {
|
|
11
|
+
normalizeRunCredentialsResolution,
|
|
12
|
+
type NormalizedRunCredentialMaterial,
|
|
13
|
+
} from "@opengeni/runtime";
|
|
14
|
+
|
|
15
|
+
export type RunCredentialResolutionContext = {
|
|
16
|
+
db: Database;
|
|
17
|
+
connectionCredentials?: ConnectionCredentialsPort | null;
|
|
18
|
+
accountId: string;
|
|
19
|
+
workspaceId: string;
|
|
20
|
+
session: Session;
|
|
21
|
+
turn: SessionTurn;
|
|
22
|
+
attemptId: string;
|
|
23
|
+
effectiveSandboxBackend: SandboxBackend;
|
|
24
|
+
variableSet: { id: string; name: string } | null;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export type BoundRunCredentialResolver = {
|
|
28
|
+
resolve(input: {
|
|
29
|
+
purpose: "provision" | "renewal";
|
|
30
|
+
forceRefresh: boolean;
|
|
31
|
+
}): Promise<NormalizedRunCredentialMaterial | null>;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export function buildRunCredentialsRequest(
|
|
35
|
+
input: Omit<RunCredentialResolutionContext, "db" | "connectionCredentials"> & {
|
|
36
|
+
rootSessionId: string;
|
|
37
|
+
purpose: "provision" | "renewal";
|
|
38
|
+
forceRefresh: boolean;
|
|
39
|
+
},
|
|
40
|
+
): RunCredentialsRequest {
|
|
41
|
+
return {
|
|
42
|
+
accountId: input.accountId,
|
|
43
|
+
workspaceId: input.workspaceId,
|
|
44
|
+
sessionId: input.session.id,
|
|
45
|
+
parentSessionId: input.session.parentSessionId,
|
|
46
|
+
rootSessionId: input.rootSessionId,
|
|
47
|
+
sandboxGroupId: input.session.sandboxGroupId,
|
|
48
|
+
turnId: input.turn.id,
|
|
49
|
+
attemptId: input.attemptId,
|
|
50
|
+
executionGeneration: input.turn.executionGeneration,
|
|
51
|
+
initiator: input.turn.initiator,
|
|
52
|
+
initiatorContext: input.turn.initiatorContext,
|
|
53
|
+
effectiveSandboxBackend: input.effectiveSandboxBackend,
|
|
54
|
+
sandboxOs: input.turn.sandboxOs ?? input.session.sandboxOs,
|
|
55
|
+
purpose: input.purpose,
|
|
56
|
+
forceRefresh: input.forceRefresh,
|
|
57
|
+
variableSet: input.variableSet,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function runCredentialAuthNeededPayloads(
|
|
62
|
+
material: NormalizedRunCredentialMaterial,
|
|
63
|
+
): CredentialAuthNeededPayload[] {
|
|
64
|
+
return material.authNeeded.map((notice) => ({
|
|
65
|
+
credentialClass: "run",
|
|
66
|
+
reason: notice.reason,
|
|
67
|
+
...(notice.providerDomain ? { providerDomain: notice.providerDomain } : {}),
|
|
68
|
+
...(notice.connectionId ? { connectionId: notice.connectionId } : {}),
|
|
69
|
+
...(notice.scopes?.length ? { scopes: notice.scopes } : {}),
|
|
70
|
+
...(notice.resource ? { resource: notice.resource } : {}),
|
|
71
|
+
...(notice.authorizationUrl ? { authorizationUrl: notice.authorizationUrl } : {}),
|
|
72
|
+
...(notice.message ? { message: notice.message } : {}),
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function runCredentialModelNote(
|
|
77
|
+
material: NormalizedRunCredentialMaterial,
|
|
78
|
+
): string | undefined {
|
|
79
|
+
if (material.authNeeded.length === 0) return undefined;
|
|
80
|
+
return [
|
|
81
|
+
"[OpenGeni connected-service status]",
|
|
82
|
+
"One or more host-managed credentials need user attention. Continue with available capabilities, but do not claim the affected service is usable until it is reconnected.",
|
|
83
|
+
JSON.stringify({
|
|
84
|
+
credentials: material.authNeeded.map((notice) => ({
|
|
85
|
+
reason: notice.reason,
|
|
86
|
+
...(notice.providerDomain ? { providerDomain: notice.providerDomain } : {}),
|
|
87
|
+
...(notice.resource ? { resource: notice.resource } : {}),
|
|
88
|
+
...(notice.message ? { message: notice.message } : {}),
|
|
89
|
+
})),
|
|
90
|
+
}),
|
|
91
|
+
].join("\n");
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Freeze the provider-neutral host credential request to this exact admitted
|
|
96
|
+
* turn. The host selects connections and material; the worker never infers a
|
|
97
|
+
* provider from repositories, environment names, or an OpenGeni variable set.
|
|
98
|
+
*/
|
|
99
|
+
export async function bindRunCredentialResolver(
|
|
100
|
+
input: RunCredentialResolutionContext,
|
|
101
|
+
): Promise<BoundRunCredentialResolver | null> {
|
|
102
|
+
const resolver = input.connectionCredentials?.runCredentials;
|
|
103
|
+
if (!resolver) return null;
|
|
104
|
+
const rootSessionId = await getSessionRootId(input.db, input.workspaceId, input.session.id);
|
|
105
|
+
if (!rootSessionId) {
|
|
106
|
+
throw new Error(`cannot resolve run credentials for missing session ${input.session.id}`);
|
|
107
|
+
}
|
|
108
|
+
const scope = {
|
|
109
|
+
accountId: input.accountId,
|
|
110
|
+
workspaceId: input.workspaceId,
|
|
111
|
+
sessionId: input.session.id,
|
|
112
|
+
};
|
|
113
|
+
return {
|
|
114
|
+
resolve: async ({ purpose, forceRefresh }) => {
|
|
115
|
+
const resolution = await resolver(
|
|
116
|
+
buildRunCredentialsRequest({
|
|
117
|
+
accountId: input.accountId,
|
|
118
|
+
workspaceId: input.workspaceId,
|
|
119
|
+
session: input.session,
|
|
120
|
+
turn: input.turn,
|
|
121
|
+
attemptId: input.attemptId,
|
|
122
|
+
effectiveSandboxBackend: input.effectiveSandboxBackend,
|
|
123
|
+
variableSet: input.variableSet,
|
|
124
|
+
rootSessionId,
|
|
125
|
+
purpose,
|
|
126
|
+
forceRefresh,
|
|
127
|
+
}),
|
|
128
|
+
);
|
|
129
|
+
return normalizeRunCredentialsResolution(resolution, scope);
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
}
|