@opengeni/worker-bundle 0.23.4 → 0.26.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +17 -2
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +8 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -2
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +1 -1
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +15 -1
- package/dist/activities/agent-turn/quiescence.d.ts +11 -6
- package/dist/activities/agent-turn/run.d.ts +9 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
- package/dist/activities/agent-turn/session-title.d.ts +21 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +4 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +40 -5
- package/dist/activities/agent-turn/turn-context.d.ts +13 -2
- package/dist/activities/agent-turn.d.ts +1 -1
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +3 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/goals.d.ts +12 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +19 -1
- package/dist/{activities-control-52W5IDWM.js → activities-control-CTEMLTHG.js} +339 -195
- package/dist/activities-control-CTEMLTHG.js.map +1 -0
- package/dist/activities-control.d.ts +4 -0
- package/dist/{activities-turn-A2G2KLSY.js → activities-turn-S5IU347A.js} +1846 -1136
- package/dist/activities-turn-S5IU347A.js.map +1 -0
- package/dist/activities.d.ts +4 -0
- package/dist/{chunk-GFNXVZVQ.js → chunk-D4ZEFOYH.js} +107 -7
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
- package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
- package/dist/{chunk-IPVMCAW5.js → chunk-TAUE3GYT.js} +225 -154
- package/dist/chunk-TAUE3GYT.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.js +43 -7
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +35 -8
- package/dist/sandbox-reaper-timeout.d.ts +2 -2
- package/dist/sandbox-resume.d.ts +6 -1
- package/dist/workflow-bundle.js +96 -10
- package/dist/workflows/activities.d.ts +6 -1
- package/dist/workflows/automations.d.ts +5 -0
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +47 -6
- package/src/activities/agent-turn/agent-build.ts +20 -26
- package/src/activities/agent-turn/claim.ts +59 -6
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/codex.ts +2 -2
- package/src/activities/agent-turn/compaction-prep.ts +8 -1
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +29 -1
- package/src/activities/agent-turn/failure-settlement.ts +451 -323
- package/src/activities/agent-turn/finalization.ts +31 -17
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +120 -45
- package/src/activities/agent-turn/quiescence.ts +32 -17
- package/src/activities/agent-turn/run.ts +98 -9
- package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
- package/src/activities/agent-turn/session-title.ts +55 -2
- package/src/activities/agent-turn/stream-attempt.ts +281 -136
- package/src/activities/agent-turn/tool-environment.ts +132 -39
- package/src/activities/agent-turn/tool-policy.ts +71 -12
- package/src/activities/agent-turn/turn-context.ts +21 -2
- package/src/activities/agent-turn.ts +5 -1
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +129 -2
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +89 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/context-compaction.ts +3 -0
- package/src/activities/goals.ts +116 -39
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/mcp-credentials.ts +53 -0
- package/src/activities/rig-verification.ts +5 -3
- package/src/activities/sandbox-lease.ts +29 -12
- package/src/activities/scheduled-tasks.ts +217 -110
- package/src/activities/session-state.ts +17 -0
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +22 -8
- package/src/activities.ts +2 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +2 -83
- package/src/index.ts +34 -4
- package/src/observability-metrics.ts +158 -14
- package/src/sandbox-reaper-timeout.ts +10 -4
- package/src/sandbox-resume.ts +58 -8
- package/src/sandbox-routing.ts +71 -16
- package/src/workflows/activities.ts +16 -0
- package/src/workflows/automations.ts +21 -2
- package/src/workflows/session.ts +85 -7
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-52W5IDWM.js.map +0 -1
- package/dist/activities-turn-A2G2KLSY.js.map +0 -1
- package/dist/chunk-GFNXVZVQ.js.map +0 -1
- package/dist/chunk-IPVMCAW5.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -141
package/src/activities/types.ts
CHANGED
|
@@ -75,6 +75,11 @@ export type InspectSessionAttemptActivity = (input: {
|
|
|
75
75
|
* agent runtime and document parser so each process can load only its role. */
|
|
76
76
|
export type SharedActivityServices = {
|
|
77
77
|
settings: Settings;
|
|
78
|
+
/**
|
|
79
|
+
* Original environment/host settings retained across live database-catalog
|
|
80
|
+
* refreshes. Older embedded test hosts may omit it and fall back to settings.
|
|
81
|
+
*/
|
|
82
|
+
catalogSourceSettings?: Settings;
|
|
78
83
|
db: Database;
|
|
79
84
|
bus: EventBus;
|
|
80
85
|
objectStorage: ObjectStorage | null;
|
|
@@ -338,6 +343,18 @@ export type PeekSessionWorkInput = {
|
|
|
338
343
|
sessionId: string;
|
|
339
344
|
};
|
|
340
345
|
|
|
346
|
+
export type SettleSessionInputWaitInput = {
|
|
347
|
+
accountId: string;
|
|
348
|
+
workspaceId: string;
|
|
349
|
+
sessionId: string;
|
|
350
|
+
waitTurnId: string;
|
|
351
|
+
disposition: "held" | "timeout" | "superseded";
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
export type SettleSessionInputWaitResult = {
|
|
355
|
+
action: "held" | "timeout" | "superseded" | "stale";
|
|
356
|
+
};
|
|
357
|
+
|
|
341
358
|
export type ExpireSessionHumanInputInput = {
|
|
342
359
|
accountId: string;
|
|
343
360
|
workspaceId: string;
|
|
@@ -373,14 +390,10 @@ export type MaybeContinueGoalInput = {
|
|
|
373
390
|
};
|
|
374
391
|
|
|
375
392
|
export type MaybeContinueGoalResult = {
|
|
376
|
-
// `
|
|
377
|
-
// materialized
|
|
378
|
-
//
|
|
379
|
-
|
|
380
|
-
// `deferred`: idle backoff between consecutive no-input continuations. Same
|
|
381
|
-
// shape as `held`: nothing materialized, obligation armed, a delayed outbox
|
|
382
|
-
// wake at the pacing deadline (pulled to now by any new input) restarts it.
|
|
383
|
-
action: "none" | "queue" | "continue" | "paused" | "held" | "deferred";
|
|
393
|
+
// `deferred`: idle backoff between consecutive no-input continuations.
|
|
394
|
+
// Nothing was materialized; the obligation remains armed and a delayed
|
|
395
|
+
// outbox wake at the pacing deadline restarts it.
|
|
396
|
+
action: "none" | "queue" | "continue" | "paused" | "deferred";
|
|
384
397
|
};
|
|
385
398
|
|
|
386
399
|
export type DispatchScheduledTaskRunInput = {
|
|
@@ -461,6 +474,7 @@ export type DispatchAutomationRunResult =
|
|
|
461
474
|
| { action: "started"; sessionId: string }
|
|
462
475
|
| { action: "already_dispatched"; sessionId: string }
|
|
463
476
|
| { action: "skipped"; reason: string }
|
|
477
|
+
| { action: "failed"; reason: string }
|
|
464
478
|
| { action: "not_found" };
|
|
465
479
|
|
|
466
480
|
export type RunKnowledgeSourceSyncBatchResult =
|
package/src/activities.ts
CHANGED
|
@@ -101,12 +101,14 @@ export const reconcileSessionAttemptQuiescence =
|
|
|
101
101
|
export const recoverDispatch = defaultControlActivities.recoverDispatch;
|
|
102
102
|
export const recoverEscapedMcpTimeout = defaultControlActivities.recoverEscapedMcpTimeout;
|
|
103
103
|
export const peekSessionWork = defaultControlActivities.peekSessionWork;
|
|
104
|
+
export const settleSessionInputWait = defaultControlActivities.settleSessionInputWait;
|
|
104
105
|
export const expireSessionHumanInput = defaultControlActivities.expireSessionHumanInput;
|
|
105
106
|
export const expireSessionInteractionIntervention =
|
|
106
107
|
defaultControlActivities.expireSessionInteractionIntervention;
|
|
107
108
|
export const markSessionIdle = defaultControlActivities.markSessionIdle;
|
|
108
109
|
export const dispatchScheduledTaskRun = defaultControlActivities.dispatchScheduledTaskRun;
|
|
109
110
|
export const dispatchAutomationRun = defaultControlActivities.dispatchAutomationRun;
|
|
111
|
+
export const settleAutomationRunFailure = defaultControlActivities.settleAutomationRunFailure;
|
|
110
112
|
export const runKnowledgeSourceSyncBatch = defaultControlActivities.runKnowledgeSourceSyncBatch;
|
|
111
113
|
export const enqueueGoalRetryWake = defaultControlActivities.enqueueGoalRetryWake;
|
|
112
114
|
export const maybeContinueGoal = defaultControlActivities.maybeContinueGoal;
|
package/src/activity-services.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengen
|
|
|
2
2
|
import { configureChildLifecycleNotices, createDb } from "@opengeni/db";
|
|
3
3
|
import { createNatsEventBus } from "@opengeni/events";
|
|
4
4
|
import { createObservability } from "@opengeni/observability";
|
|
5
|
+
import { resolveCatalogSettings } from "@opengeni/core";
|
|
5
6
|
import { createObjectStorage } from "@opengeni/storage";
|
|
6
7
|
import type { ActivityDependencies, SharedActivityServices } from "./activities/types";
|
|
7
8
|
import { observabilityEventLogger } from "./observability-metrics";
|
|
@@ -35,11 +36,13 @@ export function createSharedActivityServices(
|
|
|
35
36
|
});
|
|
36
37
|
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
37
38
|
const db = dependencies.db ?? dbClient!.db;
|
|
39
|
+
const resolvedSettings = (await resolveCatalogSettings(db, settings)).settings;
|
|
38
40
|
// Child lifecycle notice producers are process-global in @opengeni/db;
|
|
39
41
|
// install the boot-validated rollout flag once for this worker.
|
|
40
42
|
configureChildLifecycleNotices({ enabled: settings.childLifecycleNoticesEnabled });
|
|
41
43
|
return {
|
|
42
|
-
settings,
|
|
44
|
+
settings: resolvedSettings,
|
|
45
|
+
catalogSourceSettings: settings,
|
|
43
46
|
db,
|
|
44
47
|
bus:
|
|
45
48
|
dependencies.bus ??
|
package/src/github-rest-mcp.ts
CHANGED
|
@@ -11,26 +11,20 @@ import {
|
|
|
11
11
|
} from "@opengeni/contracts/personal-github";
|
|
12
12
|
import { personalGitHubRepositoryResources } from "@opengeni/core";
|
|
13
13
|
import {
|
|
14
|
-
beginConnectorActionExecution,
|
|
15
|
-
completeConnectorActionExecution,
|
|
16
14
|
getConnectionMetadata,
|
|
17
15
|
getPersonalGitHubRepositorySelectionState,
|
|
18
16
|
hasAuditableGitHubInstallationAuthority,
|
|
19
17
|
listGitHubInstallationAccessForWorkspace,
|
|
20
|
-
prepareConnectorActionApproval,
|
|
21
18
|
resolveAcceptedConnectionUse,
|
|
22
19
|
type Database,
|
|
23
20
|
type ResolveConnectionCredentialInput,
|
|
24
21
|
type ResolveConnectionCredentialResult,
|
|
25
22
|
type SessionTurnForExecution,
|
|
26
23
|
} from "@opengeni/db";
|
|
27
|
-
import {
|
|
28
|
-
AttemptToolApprovalRequiredError,
|
|
29
|
-
type AttemptToolAuthorization,
|
|
30
|
-
} from "@opengeni/codemode";
|
|
31
24
|
import { createGitHubAppInstallationTokenWithExpiry, githubAppBotIdentity } from "@opengeni/github";
|
|
32
25
|
import {
|
|
33
26
|
type AttemptConnectorActionBinding,
|
|
27
|
+
ConnectorActionBindingRejectedError,
|
|
34
28
|
type LocalMcpServerRegistration,
|
|
35
29
|
prefixedMcpToolName,
|
|
36
30
|
} from "@opengeni/runtime";
|
|
@@ -43,7 +37,6 @@ import {
|
|
|
43
37
|
GitHubRestAuthorityError,
|
|
44
38
|
githubRestConnectorActionOutcome,
|
|
45
39
|
GitHubRestMcpServer,
|
|
46
|
-
GitHubRestMutationNotExecutedError,
|
|
47
40
|
type GitHubRestRepository,
|
|
48
41
|
} from "@opengeni/runtime/github-rest-mcp";
|
|
49
42
|
|
|
@@ -59,7 +52,6 @@ export type GitHubRestMcpRuntime = {
|
|
|
59
52
|
tools: ToolRef[];
|
|
60
53
|
localMcpServers: LocalMcpServerRegistration[];
|
|
61
54
|
connectorBindings: AttemptConnectorActionBinding[];
|
|
62
|
-
authorizeCodemodeCall: AttemptToolAuthorization | null;
|
|
63
55
|
};
|
|
64
56
|
|
|
65
57
|
export async function buildGitHubRestMcpForTurn(input: {
|
|
@@ -80,7 +72,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
80
72
|
tools: input.tools,
|
|
81
73
|
localMcpServers: [],
|
|
82
74
|
connectorBindings: [],
|
|
83
|
-
authorizeCodemodeCall: null,
|
|
84
75
|
};
|
|
85
76
|
}
|
|
86
77
|
|
|
@@ -95,7 +86,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
95
86
|
tools: input.tools,
|
|
96
87
|
localMcpServers: [],
|
|
97
88
|
connectorBindings: [],
|
|
98
|
-
authorizeCodemodeCall: null,
|
|
99
89
|
};
|
|
100
90
|
}
|
|
101
91
|
const configuredIds = new Set(input.settings.mcpServers.map((server) => server.id));
|
|
@@ -111,7 +101,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
111
101
|
const localMcpServers: LocalMcpServerRegistration[] = [];
|
|
112
102
|
const toolRefs: ToolRef[] = [];
|
|
113
103
|
const connectorBindings: AttemptConnectorActionBinding[] = [];
|
|
114
|
-
const codemodeActions = createCodemodeActionLifecycle(input, connectorBindings);
|
|
115
104
|
|
|
116
105
|
if (appRepositories.length > 0) {
|
|
117
106
|
const appResolver = workspaceAppAuthorityResolver(input, appRepositories);
|
|
@@ -122,7 +111,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
122
111
|
authorityKind: "workspace_app",
|
|
123
112
|
repositories: appRepositories,
|
|
124
113
|
resolveAuthority: appResolver,
|
|
125
|
-
completeCodemodeAction: codemodeActions.complete,
|
|
126
114
|
}),
|
|
127
115
|
});
|
|
128
116
|
configs.push(internalMcpConfig(GITHUB_REST_MCP_APP_SERVER_ID, "GitHub — OpenGeni bot"));
|
|
@@ -140,7 +128,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
140
128
|
authorityKind: "personal_oauth",
|
|
141
129
|
repositories: personalRepositories,
|
|
142
130
|
resolveAuthority: personalAuthorityResolver(input),
|
|
143
|
-
completeCodemodeAction: codemodeActions.complete,
|
|
144
131
|
}),
|
|
145
132
|
});
|
|
146
133
|
configs.push(internalMcpConfig(GITHUB_REST_MCP_PERSONAL_SERVER_ID, "GitHub — My account"));
|
|
@@ -158,74 +145,6 @@ export async function buildGitHubRestMcpForTurn(input: {
|
|
|
158
145
|
tools: mergeToolRefs(input.tools, toolRefs),
|
|
159
146
|
localMcpServers,
|
|
160
147
|
connectorBindings,
|
|
161
|
-
authorizeCodemodeCall: codemodeActions.authorize,
|
|
162
|
-
};
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
function createCodemodeActionLifecycle(
|
|
166
|
-
input: {
|
|
167
|
-
db: Database;
|
|
168
|
-
accountId: string;
|
|
169
|
-
workspaceId: string;
|
|
170
|
-
sessionId: string;
|
|
171
|
-
attemptId: string;
|
|
172
|
-
turn: SessionTurnForExecution;
|
|
173
|
-
},
|
|
174
|
-
bindings: AttemptConnectorActionBinding[],
|
|
175
|
-
): {
|
|
176
|
-
authorize: AttemptToolAuthorization;
|
|
177
|
-
complete: NonNullable<
|
|
178
|
-
ConstructorParameters<typeof GitHubRestMcpServer>[0]["completeCodemodeAction"]
|
|
179
|
-
>;
|
|
180
|
-
} {
|
|
181
|
-
const identity = {
|
|
182
|
-
accountId: input.accountId,
|
|
183
|
-
workspaceId: input.workspaceId,
|
|
184
|
-
sessionId: input.sessionId,
|
|
185
|
-
turnId: input.turn.id,
|
|
186
|
-
attemptId: input.attemptId,
|
|
187
|
-
executionGeneration: input.turn.executionGeneration,
|
|
188
|
-
initiator: input.turn.initiator,
|
|
189
|
-
};
|
|
190
|
-
const executing = new Map<string, string>();
|
|
191
|
-
return {
|
|
192
|
-
authorize: async ({ call, entry }) => {
|
|
193
|
-
if (call.caller.kind !== "codemode") return;
|
|
194
|
-
const binding = bindings.find((candidate) => candidate.modelName === entry.modelName);
|
|
195
|
-
if (!binding) return;
|
|
196
|
-
const invocation = binding.call(call.operationId, call.arguments);
|
|
197
|
-
const policy = await prepareConnectorActionApproval(input.db, identity, invocation);
|
|
198
|
-
if (policy.managed && policy.decision === "ask") {
|
|
199
|
-
throw new AttemptToolApprovalRequiredError();
|
|
200
|
-
}
|
|
201
|
-
if (policy.managed && policy.decision === "block") {
|
|
202
|
-
throw new GitHubRestMutationNotExecutedError(
|
|
203
|
-
"GitHub connector action is blocked by policy",
|
|
204
|
-
);
|
|
205
|
-
}
|
|
206
|
-
const admission = await beginConnectorActionExecution(input.db, identity, invocation);
|
|
207
|
-
if (!admission.allowed) {
|
|
208
|
-
if (admission.reason === "approval_required") {
|
|
209
|
-
throw new AttemptToolApprovalRequiredError();
|
|
210
|
-
}
|
|
211
|
-
throw new GitHubRestMutationNotExecutedError(
|
|
212
|
-
`GitHub connector action was not executed: ${admission.reason}`,
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
if (admission.managed) executing.set(call.operationId, admission.requestId);
|
|
216
|
-
},
|
|
217
|
-
complete: async ({ operationId, outcome }) => {
|
|
218
|
-
const requestId = executing.get(operationId);
|
|
219
|
-
if (!requestId) return;
|
|
220
|
-
executing.delete(operationId);
|
|
221
|
-
await completeConnectorActionExecution(input.db, {
|
|
222
|
-
accountId: input.accountId,
|
|
223
|
-
workspaceId: input.workspaceId,
|
|
224
|
-
requestId,
|
|
225
|
-
attemptId: input.attemptId,
|
|
226
|
-
outcome,
|
|
227
|
-
});
|
|
228
|
-
},
|
|
229
148
|
};
|
|
230
149
|
}
|
|
231
150
|
|
|
@@ -624,7 +543,7 @@ function githubConnectorBindings(
|
|
|
624
543
|
const repository =
|
|
625
544
|
typeof args.repository === "string" ? byName.get(args.repository.toLowerCase()) : null;
|
|
626
545
|
if (!repository) {
|
|
627
|
-
throw new
|
|
546
|
+
throw new ConnectorActionBindingRejectedError(
|
|
628
547
|
"GitHub action repository does not match an accepted resource",
|
|
629
548
|
);
|
|
630
549
|
}
|
package/src/index.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
assertRuntimeDatabasePosture,
|
|
12
12
|
countSessionRecoveryBacklog,
|
|
13
13
|
createDb,
|
|
14
|
+
getContextCompactionPendingSummary,
|
|
14
15
|
markSessionWorkflowWakeDelivered,
|
|
15
16
|
type Database,
|
|
16
17
|
type RuntimeDatabasePostureOptions,
|
|
@@ -56,14 +57,17 @@ import {
|
|
|
56
57
|
type WorkerLifecycleState,
|
|
57
58
|
} from "./http";
|
|
58
59
|
import {
|
|
60
|
+
initializeContextCompactionMetrics,
|
|
59
61
|
initializeWorkerOutcomeMetrics,
|
|
60
62
|
normalizeTurnTaskQueueStats,
|
|
61
63
|
observabilityEventLogger,
|
|
64
|
+
startContextCompactionPendingMonitor,
|
|
62
65
|
startSessionRecoveryMonitor,
|
|
63
66
|
startTurnCapacityMonitor,
|
|
64
67
|
type TurnTaskQueueStats,
|
|
65
68
|
} from "./observability-metrics";
|
|
66
69
|
import {
|
|
70
|
+
resolveCatalogSettings,
|
|
67
71
|
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
68
72
|
SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
69
73
|
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
@@ -224,6 +228,9 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
224
228
|
options.activityDependencies?.observability ??
|
|
225
229
|
createObservability(settings, { component: `worker-${options.role}` });
|
|
226
230
|
initializeWorkerOutcomeMetrics(observability);
|
|
231
|
+
if (options.role === "turn") {
|
|
232
|
+
initializeContextCompactionMetrics(observability);
|
|
233
|
+
}
|
|
227
234
|
if (options.role === "turn" && options.workflowBundle) {
|
|
228
235
|
throw new Error("workflowBundle is valid only for the control worker role");
|
|
229
236
|
}
|
|
@@ -799,11 +806,24 @@ export async function createOpenGeniWorkerService(
|
|
|
799
806
|
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
800
807
|
let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
|
|
801
808
|
let sessionRecoveryMonitor: ReturnType<typeof startSessionRecoveryMonitor> | undefined;
|
|
809
|
+
let contextCompactionPendingMonitor:
|
|
810
|
+
| ReturnType<typeof startContextCompactionPendingMonitor>
|
|
811
|
+
| undefined;
|
|
802
812
|
const schedules: Array<{ close: () => Promise<void> }> = [];
|
|
803
813
|
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
804
814
|
let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
|
|
805
815
|
|
|
806
816
|
try {
|
|
817
|
+
const resolvedCatalog = await retryStartupDependency(
|
|
818
|
+
"model catalog",
|
|
819
|
+
() => resolveCatalogSettings(options.activityDependencies.db, settings),
|
|
820
|
+
{ ...retryOptions, onRetry },
|
|
821
|
+
);
|
|
822
|
+
observability.info("OpenGeni model catalog resolved", {
|
|
823
|
+
role: options.role,
|
|
824
|
+
catalogSource: resolvedCatalog.source,
|
|
825
|
+
catalogVersion: resolvedCatalog.version,
|
|
826
|
+
});
|
|
807
827
|
const needsSignaler =
|
|
808
828
|
options.role === "turn" ||
|
|
809
829
|
!options.activityDependencies.wakeSessionWorkflow ||
|
|
@@ -877,6 +897,10 @@ export async function createOpenGeniWorkerService(
|
|
|
877
897
|
observability,
|
|
878
898
|
read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db),
|
|
879
899
|
});
|
|
900
|
+
contextCompactionPendingMonitor = startContextCompactionPendingMonitor({
|
|
901
|
+
observability,
|
|
902
|
+
read: async () => await getContextCompactionPendingSummary(options.activityDependencies.db),
|
|
903
|
+
});
|
|
880
904
|
}
|
|
881
905
|
|
|
882
906
|
if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
|
|
@@ -911,14 +935,18 @@ export async function createOpenGeniWorkerService(
|
|
|
911
935
|
}
|
|
912
936
|
|
|
913
937
|
if (options.http !== false) {
|
|
938
|
+
const databaseReady = dbReadyCheck(
|
|
939
|
+
options.http?.readinessDb ?? options.activityDependencies.db,
|
|
940
|
+
options.databasePosture,
|
|
941
|
+
);
|
|
914
942
|
httpServer = startWorkerHttpServer({
|
|
915
943
|
settings,
|
|
916
944
|
observability,
|
|
917
945
|
checks: {
|
|
918
|
-
db:
|
|
919
|
-
|
|
920
|
-
options.
|
|
921
|
-
|
|
946
|
+
db: async () => {
|
|
947
|
+
await databaseReady();
|
|
948
|
+
await resolveCatalogSettings(options.activityDependencies.db, settings);
|
|
949
|
+
},
|
|
922
950
|
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
923
951
|
temporal: temporalReadyCheck(workerBundle.connection),
|
|
924
952
|
},
|
|
@@ -931,6 +959,7 @@ export async function createOpenGeniWorkerService(
|
|
|
931
959
|
await Promise.allSettled([
|
|
932
960
|
turnCapacityMonitor?.close(),
|
|
933
961
|
sessionRecoveryMonitor?.close(),
|
|
962
|
+
contextCompactionPendingMonitor?.close(),
|
|
934
963
|
workerBundle?.connection.close(),
|
|
935
964
|
signaler?.close(),
|
|
936
965
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -954,6 +983,7 @@ export async function createOpenGeniWorkerService(
|
|
|
954
983
|
await Promise.allSettled([
|
|
955
984
|
turnCapacityMonitor?.close(),
|
|
956
985
|
sessionRecoveryMonitor?.close(),
|
|
986
|
+
contextCompactionPendingMonitor?.close(),
|
|
957
987
|
activeWorkerBundle.connection.close(),
|
|
958
988
|
activeSignaler?.close(),
|
|
959
989
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -34,6 +34,10 @@ export type SessionRecoveryBacklog = {
|
|
|
34
34
|
quiescence_missing: number;
|
|
35
35
|
projection_stale: number;
|
|
36
36
|
};
|
|
37
|
+
export type ContextCompactionPendingSummary = {
|
|
38
|
+
pendingCount: number;
|
|
39
|
+
oldestStartedAt: Date | null;
|
|
40
|
+
};
|
|
37
41
|
|
|
38
42
|
export type TemporalTurnTaskQueueStats = {
|
|
39
43
|
approximateBacklogCount?: unknown;
|
|
@@ -49,6 +53,7 @@ const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
|
49
53
|
const modelRequestTrackers = new WeakMap<Observability, ModelRequestLifecycleMetrics>();
|
|
50
54
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
51
55
|
const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
|
|
56
|
+
const initializedContextCompactionMetrics = new WeakSet<Observability>();
|
|
52
57
|
|
|
53
58
|
// A worker process reaching one trillion tokens in one provider/cache counter is
|
|
54
59
|
// already far outside an ordinary scrape lifetime. Refuse further increments
|
|
@@ -61,6 +66,18 @@ const WORKER_DEATH_RECOVERY_OUTCOMES: readonly WorkerDeathRecoveryOutcome[] = [
|
|
|
61
66
|
"exhausted",
|
|
62
67
|
];
|
|
63
68
|
const WORKER_DEATH_TIMEOUT_TYPES = ["heartbeat", "schedule_to_start"] as const;
|
|
69
|
+
const CONTEXT_COMPACTION_TRIGGERS = ["auto", "operator", "proactive", "overflow"] as const;
|
|
70
|
+
|
|
71
|
+
export type ContextCompactionTrigger = (typeof CONTEXT_COMPACTION_TRIGGERS)[number];
|
|
72
|
+
|
|
73
|
+
const CONTEXT_COMPACTION_STARTS_METRIC = {
|
|
74
|
+
name: "opengeni_context_compaction_starts_total",
|
|
75
|
+
help: "Total durable context compaction starts, by trigger.",
|
|
76
|
+
} as const;
|
|
77
|
+
const CONTEXT_COMPACTIONS_METRIC = {
|
|
78
|
+
name: "opengeni_context_compactions_total",
|
|
79
|
+
help: "Total completed context compactions, by trigger.",
|
|
80
|
+
} as const;
|
|
64
81
|
|
|
65
82
|
export function observabilityEventLogger(observability: Observability): EventLogger {
|
|
66
83
|
return {
|
|
@@ -346,6 +363,29 @@ export function initializeWorkerOutcomeMetrics(observability: Observability): vo
|
|
|
346
363
|
}
|
|
347
364
|
}
|
|
348
365
|
|
|
366
|
+
/**
|
|
367
|
+
* Publish every bounded compaction lifecycle series before the first event.
|
|
368
|
+
* The zero counters make the closed trigger catalog explicit before a scrape.
|
|
369
|
+
* Alerting uses the separate durable control-worker projection below; these
|
|
370
|
+
* process-local counters are lifecycle-rate diagnostics only.
|
|
371
|
+
*/
|
|
372
|
+
export function initializeContextCompactionMetrics(observability: Observability): void {
|
|
373
|
+
if (initializedContextCompactionMetrics.has(observability)) return;
|
|
374
|
+
for (const trigger of CONTEXT_COMPACTION_TRIGGERS) {
|
|
375
|
+
observability.incrementCounter({
|
|
376
|
+
...CONTEXT_COMPACTION_STARTS_METRIC,
|
|
377
|
+
labels: { trigger },
|
|
378
|
+
amount: 0,
|
|
379
|
+
});
|
|
380
|
+
observability.incrementCounter({
|
|
381
|
+
...CONTEXT_COMPACTIONS_METRIC,
|
|
382
|
+
labels: { trigger },
|
|
383
|
+
amount: 0,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
initializedContextCompactionMetrics.add(observability);
|
|
387
|
+
}
|
|
388
|
+
|
|
349
389
|
export function recordWorkerDeathRecoveryMetrics(
|
|
350
390
|
observability: Observability,
|
|
351
391
|
input: {
|
|
@@ -830,6 +870,96 @@ export function startSessionRecoveryMonitor(input: {
|
|
|
830
870
|
};
|
|
831
871
|
}
|
|
832
872
|
|
|
873
|
+
export function recordContextCompactionPendingGauges(
|
|
874
|
+
observability: Observability,
|
|
875
|
+
summary: ContextCompactionPendingSummary,
|
|
876
|
+
nowMs = Date.now(),
|
|
877
|
+
): void {
|
|
878
|
+
const oldestStartedAtMs = summary.oldestStartedAt?.getTime();
|
|
879
|
+
const oldestPendingAgeSeconds =
|
|
880
|
+
oldestStartedAtMs === undefined || !Number.isFinite(oldestStartedAtMs)
|
|
881
|
+
? 0
|
|
882
|
+
: Math.max(0, (nowMs - oldestStartedAtMs) / 1_000);
|
|
883
|
+
observability.setGauge({
|
|
884
|
+
name: "opengeni_context_compaction_pending",
|
|
885
|
+
help: "Current exact active attempts whose latest automatic compaction landmark is still started.",
|
|
886
|
+
value: nonnegativeFinite(summary.pendingCount),
|
|
887
|
+
});
|
|
888
|
+
observability.setGauge({
|
|
889
|
+
name: "opengeni_context_compaction_oldest_pending_age_seconds",
|
|
890
|
+
help: "Age in seconds of the oldest durably pending automatic compaction, or zero when none are pending.",
|
|
891
|
+
value: oldestPendingAgeSeconds,
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
export function startContextCompactionPendingMonitor(input: {
|
|
896
|
+
observability: Observability;
|
|
897
|
+
read: () => Promise<ContextCompactionPendingSummary>;
|
|
898
|
+
intervalMs?: number;
|
|
899
|
+
now?: () => number;
|
|
900
|
+
}): { close: () => Promise<void> } {
|
|
901
|
+
const intervalMs = input.intervalMs ?? 60_000;
|
|
902
|
+
const now = input.now ?? Date.now;
|
|
903
|
+
const startedAt = now();
|
|
904
|
+
let lastSuccessAt: number | null = null;
|
|
905
|
+
let lastReadSucceeded = false;
|
|
906
|
+
let stopped = false;
|
|
907
|
+
let running: Promise<void> | null = null;
|
|
908
|
+
const recordStatus = () => {
|
|
909
|
+
const observedAt = now();
|
|
910
|
+
const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
|
|
911
|
+
const set = (name: string, help: string, value: number) =>
|
|
912
|
+
input.observability.setGauge({ name, help, value });
|
|
913
|
+
set(
|
|
914
|
+
"opengeni_context_compaction_monitor_last_read_success",
|
|
915
|
+
"Whether the latest durable context-compaction aggregate read completed successfully.",
|
|
916
|
+
lastReadSucceeded ? 1 : 0,
|
|
917
|
+
);
|
|
918
|
+
set(
|
|
919
|
+
"opengeni_context_compaction_monitor_last_success_timestamp_seconds",
|
|
920
|
+
"Unix timestamp of the latest successful durable context-compaction aggregate read, or zero before one succeeds.",
|
|
921
|
+
lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
|
|
922
|
+
);
|
|
923
|
+
set(
|
|
924
|
+
"opengeni_context_compaction_monitor_fresh",
|
|
925
|
+
"Whether durable context-compaction pending gauges have a successful read within three monitor intervals.",
|
|
926
|
+
lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
|
|
927
|
+
);
|
|
928
|
+
};
|
|
929
|
+
const refresh = () => {
|
|
930
|
+
recordStatus();
|
|
931
|
+
if (stopped || running) return;
|
|
932
|
+
running = input
|
|
933
|
+
.read()
|
|
934
|
+
.then((summary) => {
|
|
935
|
+
recordContextCompactionPendingGauges(input.observability, summary, now());
|
|
936
|
+
lastSuccessAt = now();
|
|
937
|
+
lastReadSucceeded = true;
|
|
938
|
+
recordStatus();
|
|
939
|
+
})
|
|
940
|
+
.catch((error) => {
|
|
941
|
+
lastReadSucceeded = false;
|
|
942
|
+
recordStatus();
|
|
943
|
+
input.observability.warn("context compaction monitor: durable aggregate read failed", {
|
|
944
|
+
error: error instanceof Error ? error.message : String(error),
|
|
945
|
+
});
|
|
946
|
+
})
|
|
947
|
+
.finally(() => {
|
|
948
|
+
running = null;
|
|
949
|
+
});
|
|
950
|
+
};
|
|
951
|
+
refresh();
|
|
952
|
+
const timer = setInterval(refresh, intervalMs);
|
|
953
|
+
timer.unref?.();
|
|
954
|
+
return {
|
|
955
|
+
close: async () => {
|
|
956
|
+
stopped = true;
|
|
957
|
+
clearInterval(timer);
|
|
958
|
+
await running;
|
|
959
|
+
},
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
|
|
833
963
|
export function recordSandboxLeaseGauges(
|
|
834
964
|
observability: Observability,
|
|
835
965
|
counts: Partial<Record<SandboxLeaseLiveness, number>>,
|
|
@@ -1023,6 +1153,7 @@ export function recordSandboxRotationBacklogGauges(
|
|
|
1023
1153
|
turnBlocked: number;
|
|
1024
1154
|
directBlocked: number;
|
|
1025
1155
|
processBlocked: number;
|
|
1156
|
+
interactionBlocked: number;
|
|
1026
1157
|
},
|
|
1027
1158
|
): void {
|
|
1028
1159
|
const values = {
|
|
@@ -1031,6 +1162,7 @@ export function recordSandboxRotationBacklogGauges(
|
|
|
1031
1162
|
turn_blocked: backlog.turnBlocked,
|
|
1032
1163
|
direct_blocked: backlog.directBlocked,
|
|
1033
1164
|
process_blocked: backlog.processBlocked,
|
|
1165
|
+
interaction_blocked: backlog.interactionBlocked,
|
|
1034
1166
|
} as const;
|
|
1035
1167
|
for (const [kind, value] of Object.entries(values)) {
|
|
1036
1168
|
observability.setGauge({
|
|
@@ -1228,6 +1360,7 @@ export type TurnStartupPhase =
|
|
|
1228
1360
|
| "tool_optional_connect"
|
|
1229
1361
|
| "tool_attempt_catalog_build"
|
|
1230
1362
|
| "tool_attempt_catalog_persist"
|
|
1363
|
+
| "tool_workspace_gateway_catalog_build"
|
|
1231
1364
|
| "post_tool_preparation"
|
|
1232
1365
|
| "agent_construction"
|
|
1233
1366
|
| "post_agent_preparation"
|
|
@@ -1276,6 +1409,9 @@ export type TurnStartupPhase =
|
|
|
1276
1409
|
| "model_prepare_runner_before_mcp_tools"
|
|
1277
1410
|
| "model_prepare_mcp_tools_snapshot"
|
|
1278
1411
|
| "model_prepare_mcp_tools_before_input_filter"
|
|
1412
|
+
| "model_prepare_mcp_tools_before_repository_skill_discovery"
|
|
1413
|
+
| "model_prepare_repository_skill_discovery"
|
|
1414
|
+
| "model_prepare_repository_skill_discovery_before_input_filter"
|
|
1279
1415
|
| "model_prepare_input_filter_base"
|
|
1280
1416
|
| "model_prepare_input_filter_genesis"
|
|
1281
1417
|
| "model_prepare_input_filter_host"
|
|
@@ -1749,15 +1885,14 @@ export function recordSessionEventPublishLatency(
|
|
|
1749
1885
|
});
|
|
1750
1886
|
}
|
|
1751
1887
|
|
|
1752
|
-
// Context tokens per response span a wide range; buckets
|
|
1753
|
-
//
|
|
1888
|
+
// Context tokens per response span a wide range; buckets retain context-pressure
|
|
1889
|
+
// diagnostics while compaction alerting follows the durable model-aware start.
|
|
1754
1890
|
const MODEL_INPUT_TOKENS_BUCKETS = [
|
|
1755
1891
|
1_000, 5_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 300_000, 500_000, 1_000_000,
|
|
1756
1892
|
];
|
|
1757
1893
|
|
|
1758
|
-
/** Observed input (context) tokens per model response
|
|
1759
|
-
*
|
|
1760
|
-
* "compaction never firing while contexts run hot" failure mode expressible. */
|
|
1894
|
+
/** Observed input (context) tokens per model response for context-pressure
|
|
1895
|
+
* dashboards and provider diagnostics. */
|
|
1761
1896
|
export function recordModelInputTokens(
|
|
1762
1897
|
observability: Observability,
|
|
1763
1898
|
provider: string,
|
|
@@ -1776,15 +1911,24 @@ export function recordModelInputTokens(
|
|
|
1776
1911
|
});
|
|
1777
1912
|
}
|
|
1778
1913
|
|
|
1779
|
-
/** A context compaction
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1914
|
+
/** A context compaction successfully completed, by bounded trigger. */
|
|
1915
|
+
export function recordContextCompaction(
|
|
1916
|
+
observability: Observability,
|
|
1917
|
+
trigger: ContextCompactionTrigger,
|
|
1918
|
+
): void {
|
|
1919
|
+
observability.incrementCounter({ ...CONTEXT_COMPACTIONS_METRIC, labels: { trigger } });
|
|
1920
|
+
}
|
|
1921
|
+
|
|
1922
|
+
/**
|
|
1923
|
+
* A durable context-compaction start, recorded only after the attempt-fenced
|
|
1924
|
+
* `compaction.started` transaction commits. This is the model-aware threshold
|
|
1925
|
+
* signal; it does not guess from a static token count shared by unlike models.
|
|
1926
|
+
*/
|
|
1927
|
+
export function recordContextCompactionStarted(
|
|
1928
|
+
observability: Observability,
|
|
1929
|
+
trigger: ContextCompactionTrigger,
|
|
1930
|
+
): void {
|
|
1931
|
+
observability.incrementCounter({ ...CONTEXT_COMPACTION_STARTS_METRIC, labels: { trigger } });
|
|
1788
1932
|
}
|
|
1789
1933
|
|
|
1790
1934
|
const MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2_048, 8_192];
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
|
|
3
3
|
SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS,
|
|
4
|
+
effectiveSandboxDrainSnapshotTimeoutMs,
|
|
4
5
|
sandboxArchiveCaptureTimeoutMs,
|
|
5
6
|
type Settings,
|
|
6
7
|
} from "@opengeni/config";
|
|
@@ -13,16 +14,21 @@ import {
|
|
|
13
14
|
|
|
14
15
|
const STRICT_TIMEOUT_FENCE_MS = 1;
|
|
15
16
|
|
|
16
|
-
export function sandboxDrainTiming(
|
|
17
|
+
export function sandboxDrainTiming(
|
|
18
|
+
settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
|
|
19
|
+
): {
|
|
17
20
|
snapshotTimeoutMs: number;
|
|
18
21
|
captureTimeoutMs: number;
|
|
19
22
|
timeoutClass: SandboxDrainTimeoutClass;
|
|
20
23
|
activityTimeoutMs: number;
|
|
21
24
|
} {
|
|
22
|
-
const
|
|
25
|
+
const snapshotTimeoutMs = effectiveSandboxDrainSnapshotTimeoutMs(settings);
|
|
26
|
+
const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs({
|
|
27
|
+
sandboxSnapshotTimeoutMs: snapshotTimeoutMs,
|
|
28
|
+
});
|
|
23
29
|
const timeoutClass = sandboxDrainTimeoutClass(captureTimeoutMs);
|
|
24
30
|
return {
|
|
25
|
-
snapshotTimeoutMs
|
|
31
|
+
snapshotTimeoutMs,
|
|
26
32
|
captureTimeoutMs,
|
|
27
33
|
timeoutClass,
|
|
28
34
|
activityTimeoutMs: sandboxDrainActivityTimeoutMs(timeoutClass),
|
|
@@ -30,7 +36,7 @@ export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTime
|
|
|
30
36
|
}
|
|
31
37
|
|
|
32
38
|
export function assertSandboxReaperActivityTimeout(
|
|
33
|
-
settings: Pick<Settings, "sandboxSnapshotTimeoutMs">,
|
|
39
|
+
settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
|
|
34
40
|
): void {
|
|
35
41
|
const timing = sandboxDrainTiming(settings);
|
|
36
42
|
if (timing.snapshotTimeoutMs >= timing.captureTimeoutMs) {
|