@opengeni/worker-bundle 0.23.4 → 0.26.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +17 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  3. package/dist/activities/agent-turn/claim.d.ts +8 -0
  4. package/dist/activities/agent-turn/codex.d.ts +2 -2
  5. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  6. package/dist/activities/agent-turn/errors.d.ts +1 -1
  7. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +15 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +11 -6
  11. package/dist/activities/agent-turn/run.d.ts +9 -0
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
  13. package/dist/activities/agent-turn/session-title.d.ts +21 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +4 -13
  16. package/dist/activities/agent-turn/tool-policy.d.ts +40 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +13 -2
  18. package/dist/activities/agent-turn.d.ts +1 -1
  19. package/dist/activities/automations.d.ts +1 -0
  20. package/dist/activities/capabilities.d.ts +3 -1
  21. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  22. package/dist/activities/codex-capacity.d.ts +32 -3
  23. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  24. package/dist/activities/codex-image-generation.d.ts +1 -1
  25. package/dist/activities/codex-rotation.d.ts +8 -67
  26. package/dist/activities/context-compaction.d.ts +2 -0
  27. package/dist/activities/goals.d.ts +12 -1
  28. package/dist/activities/google-drive-publication.d.ts +2 -13
  29. package/dist/activities/image-generation-operation.d.ts +13 -4
  30. package/dist/activities/image-generation-references.d.ts +2 -1
  31. package/dist/activities/scheduled-tasks.d.ts +1 -0
  32. package/dist/activities/session-state.d.ts +4 -2
  33. package/dist/activities/types.d.ts +19 -1
  34. package/dist/{activities-control-52W5IDWM.js → activities-control-CTEMLTHG.js} +339 -195
  35. package/dist/activities-control-CTEMLTHG.js.map +1 -0
  36. package/dist/activities-control.d.ts +4 -0
  37. package/dist/{activities-turn-A2G2KLSY.js → activities-turn-S5IU347A.js} +1846 -1136
  38. package/dist/activities-turn-S5IU347A.js.map +1 -0
  39. package/dist/activities.d.ts +4 -0
  40. package/dist/{chunk-GFNXVZVQ.js → chunk-D4ZEFOYH.js} +107 -7
  41. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  42. package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
  43. package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
  44. package/dist/{chunk-IPVMCAW5.js → chunk-TAUE3GYT.js} +225 -154
  45. package/dist/chunk-TAUE3GYT.js.map +1 -0
  46. package/dist/github-rest-mcp.d.ts +0 -2
  47. package/dist/index.d.ts +2 -0
  48. package/dist/index.js +43 -7
  49. package/dist/index.js.map +1 -1
  50. package/dist/observability-metrics.d.ts +35 -8
  51. package/dist/sandbox-reaper-timeout.d.ts +2 -2
  52. package/dist/sandbox-resume.d.ts +6 -1
  53. package/dist/workflow-bundle.js +96 -10
  54. package/dist/workflows/activities.d.ts +6 -1
  55. package/dist/workflows/automations.d.ts +5 -0
  56. package/dist/workflows/session.d.ts +3 -0
  57. package/package.json +17 -17
  58. package/src/activities/agent-run-admission.ts +7 -1
  59. package/src/activities/agent-turn/admission.ts +47 -6
  60. package/src/activities/agent-turn/agent-build.ts +20 -26
  61. package/src/activities/agent-turn/claim.ts +59 -6
  62. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  63. package/src/activities/agent-turn/codex.ts +2 -2
  64. package/src/activities/agent-turn/compaction-prep.ts +8 -1
  65. package/src/activities/agent-turn/credential-leases.ts +36 -0
  66. package/src/activities/agent-turn/errors.ts +29 -1
  67. package/src/activities/agent-turn/failure-settlement.ts +451 -323
  68. package/src/activities/agent-turn/finalization.ts +31 -17
  69. package/src/activities/agent-turn/list-models.ts +101 -0
  70. package/src/activities/agent-turn/model-usage.ts +120 -45
  71. package/src/activities/agent-turn/quiescence.ts +32 -17
  72. package/src/activities/agent-turn/run.ts +98 -9
  73. package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
  74. package/src/activities/agent-turn/session-title.ts +55 -2
  75. package/src/activities/agent-turn/stream-attempt.ts +281 -136
  76. package/src/activities/agent-turn/tool-environment.ts +132 -39
  77. package/src/activities/agent-turn/tool-policy.ts +71 -12
  78. package/src/activities/agent-turn/turn-context.ts +21 -2
  79. package/src/activities/agent-turn.ts +5 -1
  80. package/src/activities/automations.ts +122 -67
  81. package/src/activities/capabilities.ts +129 -2
  82. package/src/activities/codemode-dispatcher.ts +235 -59
  83. package/src/activities/codex-capacity.ts +89 -10
  84. package/src/activities/codex-fleet-shadow.ts +8 -1
  85. package/src/activities/codex-image-generation.ts +19 -2
  86. package/src/activities/codex-rotation.ts +45 -142
  87. package/src/activities/context-compaction.ts +3 -0
  88. package/src/activities/goals.ts +116 -39
  89. package/src/activities/google-drive-publication.ts +4 -81
  90. package/src/activities/image-generation-operation.ts +28 -3
  91. package/src/activities/image-generation-references.ts +9 -9
  92. package/src/activities/mcp-credentials.ts +53 -0
  93. package/src/activities/rig-verification.ts +5 -3
  94. package/src/activities/sandbox-lease.ts +29 -12
  95. package/src/activities/scheduled-tasks.ts +217 -110
  96. package/src/activities/session-state.ts +17 -0
  97. package/src/activities/site-auth-maintenance.ts +12 -12
  98. package/src/activities/types.ts +22 -8
  99. package/src/activities.ts +2 -0
  100. package/src/activity-services.ts +4 -1
  101. package/src/github-rest-mcp.ts +2 -83
  102. package/src/index.ts +34 -4
  103. package/src/observability-metrics.ts +158 -14
  104. package/src/sandbox-reaper-timeout.ts +10 -4
  105. package/src/sandbox-resume.ts +58 -8
  106. package/src/sandbox-routing.ts +71 -16
  107. package/src/workflows/activities.ts +16 -0
  108. package/src/workflows/automations.ts +21 -2
  109. package/src/workflows/session.ts +85 -7
  110. package/dist/activities/api-integrations.d.ts +0 -21
  111. package/dist/activities-control-52W5IDWM.js.map +0 -1
  112. package/dist/activities-turn-A2G2KLSY.js.map +0 -1
  113. package/dist/chunk-GFNXVZVQ.js.map +0 -1
  114. package/dist/chunk-IPVMCAW5.js.map +0 -1
  115. package/src/activities/api-integrations.ts +0 -141
@@ -75,6 +75,11 @@ export type InspectSessionAttemptActivity = (input: {
75
75
  * agent runtime and document parser so each process can load only its role. */
76
76
  export type SharedActivityServices = {
77
77
  settings: Settings;
78
+ /**
79
+ * Original environment/host settings retained across live database-catalog
80
+ * refreshes. Older embedded test hosts may omit it and fall back to settings.
81
+ */
82
+ catalogSourceSettings?: Settings;
78
83
  db: Database;
79
84
  bus: EventBus;
80
85
  objectStorage: ObjectStorage | null;
@@ -338,6 +343,18 @@ export type PeekSessionWorkInput = {
338
343
  sessionId: string;
339
344
  };
340
345
 
346
+ export type SettleSessionInputWaitInput = {
347
+ accountId: string;
348
+ workspaceId: string;
349
+ sessionId: string;
350
+ waitTurnId: string;
351
+ disposition: "held" | "timeout" | "superseded";
352
+ };
353
+
354
+ export type SettleSessionInputWaitResult = {
355
+ action: "held" | "timeout" | "superseded" | "stale";
356
+ };
357
+
341
358
  export type ExpireSessionHumanInputInput = {
342
359
  accountId: string;
343
360
  workspaceId: string;
@@ -373,14 +390,10 @@ export type MaybeContinueGoalInput = {
373
390
  };
374
391
 
375
392
  export type MaybeContinueGoalResult = {
376
- // `held`: an agent-declared `goal_wait` hold is current. No continuation was
377
- // materialized and the goal obligation stays armed; the workflow closes like
378
- // `none` and the wake outbox (deadline) or any producer's signalWithStart
379
- // restarts it.
380
- // `deferred`: idle backoff between consecutive no-input continuations. Same
381
- // shape as `held`: nothing materialized, obligation armed, a delayed outbox
382
- // wake at the pacing deadline (pulled to now by any new input) restarts it.
383
- action: "none" | "queue" | "continue" | "paused" | "held" | "deferred";
393
+ // `deferred`: idle backoff between consecutive no-input continuations.
394
+ // Nothing was materialized; the obligation remains armed and a delayed
395
+ // outbox wake at the pacing deadline restarts it.
396
+ action: "none" | "queue" | "continue" | "paused" | "deferred";
384
397
  };
385
398
 
386
399
  export type DispatchScheduledTaskRunInput = {
@@ -461,6 +474,7 @@ export type DispatchAutomationRunResult =
461
474
  | { action: "started"; sessionId: string }
462
475
  | { action: "already_dispatched"; sessionId: string }
463
476
  | { action: "skipped"; reason: string }
477
+ | { action: "failed"; reason: string }
464
478
  | { action: "not_found" };
465
479
 
466
480
  export type RunKnowledgeSourceSyncBatchResult =
package/src/activities.ts CHANGED
@@ -101,12 +101,14 @@ export const reconcileSessionAttemptQuiescence =
101
101
  export const recoverDispatch = defaultControlActivities.recoverDispatch;
102
102
  export const recoverEscapedMcpTimeout = defaultControlActivities.recoverEscapedMcpTimeout;
103
103
  export const peekSessionWork = defaultControlActivities.peekSessionWork;
104
+ export const settleSessionInputWait = defaultControlActivities.settleSessionInputWait;
104
105
  export const expireSessionHumanInput = defaultControlActivities.expireSessionHumanInput;
105
106
  export const expireSessionInteractionIntervention =
106
107
  defaultControlActivities.expireSessionInteractionIntervention;
107
108
  export const markSessionIdle = defaultControlActivities.markSessionIdle;
108
109
  export const dispatchScheduledTaskRun = defaultControlActivities.dispatchScheduledTaskRun;
109
110
  export const dispatchAutomationRun = defaultControlActivities.dispatchAutomationRun;
111
+ export const settleAutomationRunFailure = defaultControlActivities.settleAutomationRunFailure;
110
112
  export const runKnowledgeSourceSyncBatch = defaultControlActivities.runKnowledgeSourceSyncBatch;
111
113
  export const enqueueGoalRetryWake = defaultControlActivities.enqueueGoalRetryWake;
112
114
  export const maybeContinueGoal = defaultControlActivities.maybeContinueGoal;
@@ -2,6 +2,7 @@ import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengen
2
2
  import { configureChildLifecycleNotices, createDb } from "@opengeni/db";
3
3
  import { createNatsEventBus } from "@opengeni/events";
4
4
  import { createObservability } from "@opengeni/observability";
5
+ import { resolveCatalogSettings } from "@opengeni/core";
5
6
  import { createObjectStorage } from "@opengeni/storage";
6
7
  import type { ActivityDependencies, SharedActivityServices } from "./activities/types";
7
8
  import { observabilityEventLogger } from "./observability-metrics";
@@ -35,11 +36,13 @@ export function createSharedActivityServices(
35
36
  });
36
37
  const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
37
38
  const db = dependencies.db ?? dbClient!.db;
39
+ const resolvedSettings = (await resolveCatalogSettings(db, settings)).settings;
38
40
  // Child lifecycle notice producers are process-global in @opengeni/db;
39
41
  // install the boot-validated rollout flag once for this worker.
40
42
  configureChildLifecycleNotices({ enabled: settings.childLifecycleNoticesEnabled });
41
43
  return {
42
- settings,
44
+ settings: resolvedSettings,
45
+ catalogSourceSettings: settings,
43
46
  db,
44
47
  bus:
45
48
  dependencies.bus ??
@@ -11,26 +11,20 @@ import {
11
11
  } from "@opengeni/contracts/personal-github";
12
12
  import { personalGitHubRepositoryResources } from "@opengeni/core";
13
13
  import {
14
- beginConnectorActionExecution,
15
- completeConnectorActionExecution,
16
14
  getConnectionMetadata,
17
15
  getPersonalGitHubRepositorySelectionState,
18
16
  hasAuditableGitHubInstallationAuthority,
19
17
  listGitHubInstallationAccessForWorkspace,
20
- prepareConnectorActionApproval,
21
18
  resolveAcceptedConnectionUse,
22
19
  type Database,
23
20
  type ResolveConnectionCredentialInput,
24
21
  type ResolveConnectionCredentialResult,
25
22
  type SessionTurnForExecution,
26
23
  } from "@opengeni/db";
27
- import {
28
- AttemptToolApprovalRequiredError,
29
- type AttemptToolAuthorization,
30
- } from "@opengeni/codemode";
31
24
  import { createGitHubAppInstallationTokenWithExpiry, githubAppBotIdentity } from "@opengeni/github";
32
25
  import {
33
26
  type AttemptConnectorActionBinding,
27
+ ConnectorActionBindingRejectedError,
34
28
  type LocalMcpServerRegistration,
35
29
  prefixedMcpToolName,
36
30
  } from "@opengeni/runtime";
@@ -43,7 +37,6 @@ import {
43
37
  GitHubRestAuthorityError,
44
38
  githubRestConnectorActionOutcome,
45
39
  GitHubRestMcpServer,
46
- GitHubRestMutationNotExecutedError,
47
40
  type GitHubRestRepository,
48
41
  } from "@opengeni/runtime/github-rest-mcp";
49
42
 
@@ -59,7 +52,6 @@ export type GitHubRestMcpRuntime = {
59
52
  tools: ToolRef[];
60
53
  localMcpServers: LocalMcpServerRegistration[];
61
54
  connectorBindings: AttemptConnectorActionBinding[];
62
- authorizeCodemodeCall: AttemptToolAuthorization | null;
63
55
  };
64
56
 
65
57
  export async function buildGitHubRestMcpForTurn(input: {
@@ -80,7 +72,6 @@ export async function buildGitHubRestMcpForTurn(input: {
80
72
  tools: input.tools,
81
73
  localMcpServers: [],
82
74
  connectorBindings: [],
83
- authorizeCodemodeCall: null,
84
75
  };
85
76
  }
86
77
 
@@ -95,7 +86,6 @@ export async function buildGitHubRestMcpForTurn(input: {
95
86
  tools: input.tools,
96
87
  localMcpServers: [],
97
88
  connectorBindings: [],
98
- authorizeCodemodeCall: null,
99
89
  };
100
90
  }
101
91
  const configuredIds = new Set(input.settings.mcpServers.map((server) => server.id));
@@ -111,7 +101,6 @@ export async function buildGitHubRestMcpForTurn(input: {
111
101
  const localMcpServers: LocalMcpServerRegistration[] = [];
112
102
  const toolRefs: ToolRef[] = [];
113
103
  const connectorBindings: AttemptConnectorActionBinding[] = [];
114
- const codemodeActions = createCodemodeActionLifecycle(input, connectorBindings);
115
104
 
116
105
  if (appRepositories.length > 0) {
117
106
  const appResolver = workspaceAppAuthorityResolver(input, appRepositories);
@@ -122,7 +111,6 @@ export async function buildGitHubRestMcpForTurn(input: {
122
111
  authorityKind: "workspace_app",
123
112
  repositories: appRepositories,
124
113
  resolveAuthority: appResolver,
125
- completeCodemodeAction: codemodeActions.complete,
126
114
  }),
127
115
  });
128
116
  configs.push(internalMcpConfig(GITHUB_REST_MCP_APP_SERVER_ID, "GitHub — OpenGeni bot"));
@@ -140,7 +128,6 @@ export async function buildGitHubRestMcpForTurn(input: {
140
128
  authorityKind: "personal_oauth",
141
129
  repositories: personalRepositories,
142
130
  resolveAuthority: personalAuthorityResolver(input),
143
- completeCodemodeAction: codemodeActions.complete,
144
131
  }),
145
132
  });
146
133
  configs.push(internalMcpConfig(GITHUB_REST_MCP_PERSONAL_SERVER_ID, "GitHub — My account"));
@@ -158,74 +145,6 @@ export async function buildGitHubRestMcpForTurn(input: {
158
145
  tools: mergeToolRefs(input.tools, toolRefs),
159
146
  localMcpServers,
160
147
  connectorBindings,
161
- authorizeCodemodeCall: codemodeActions.authorize,
162
- };
163
- }
164
-
165
- function createCodemodeActionLifecycle(
166
- input: {
167
- db: Database;
168
- accountId: string;
169
- workspaceId: string;
170
- sessionId: string;
171
- attemptId: string;
172
- turn: SessionTurnForExecution;
173
- },
174
- bindings: AttemptConnectorActionBinding[],
175
- ): {
176
- authorize: AttemptToolAuthorization;
177
- complete: NonNullable<
178
- ConstructorParameters<typeof GitHubRestMcpServer>[0]["completeCodemodeAction"]
179
- >;
180
- } {
181
- const identity = {
182
- accountId: input.accountId,
183
- workspaceId: input.workspaceId,
184
- sessionId: input.sessionId,
185
- turnId: input.turn.id,
186
- attemptId: input.attemptId,
187
- executionGeneration: input.turn.executionGeneration,
188
- initiator: input.turn.initiator,
189
- };
190
- const executing = new Map<string, string>();
191
- return {
192
- authorize: async ({ call, entry }) => {
193
- if (call.caller.kind !== "codemode") return;
194
- const binding = bindings.find((candidate) => candidate.modelName === entry.modelName);
195
- if (!binding) return;
196
- const invocation = binding.call(call.operationId, call.arguments);
197
- const policy = await prepareConnectorActionApproval(input.db, identity, invocation);
198
- if (policy.managed && policy.decision === "ask") {
199
- throw new AttemptToolApprovalRequiredError();
200
- }
201
- if (policy.managed && policy.decision === "block") {
202
- throw new GitHubRestMutationNotExecutedError(
203
- "GitHub connector action is blocked by policy",
204
- );
205
- }
206
- const admission = await beginConnectorActionExecution(input.db, identity, invocation);
207
- if (!admission.allowed) {
208
- if (admission.reason === "approval_required") {
209
- throw new AttemptToolApprovalRequiredError();
210
- }
211
- throw new GitHubRestMutationNotExecutedError(
212
- `GitHub connector action was not executed: ${admission.reason}`,
213
- );
214
- }
215
- if (admission.managed) executing.set(call.operationId, admission.requestId);
216
- },
217
- complete: async ({ operationId, outcome }) => {
218
- const requestId = executing.get(operationId);
219
- if (!requestId) return;
220
- executing.delete(operationId);
221
- await completeConnectorActionExecution(input.db, {
222
- accountId: input.accountId,
223
- workspaceId: input.workspaceId,
224
- requestId,
225
- attemptId: input.attemptId,
226
- outcome,
227
- });
228
- },
229
148
  };
230
149
  }
231
150
 
@@ -624,7 +543,7 @@ function githubConnectorBindings(
624
543
  const repository =
625
544
  typeof args.repository === "string" ? byName.get(args.repository.toLowerCase()) : null;
626
545
  if (!repository) {
627
- throw new GitHubRestAuthorityError(
546
+ throw new ConnectorActionBindingRejectedError(
628
547
  "GitHub action repository does not match an accepted resource",
629
548
  );
630
549
  }
package/src/index.ts CHANGED
@@ -11,6 +11,7 @@ import {
11
11
  assertRuntimeDatabasePosture,
12
12
  countSessionRecoveryBacklog,
13
13
  createDb,
14
+ getContextCompactionPendingSummary,
14
15
  markSessionWorkflowWakeDelivered,
15
16
  type Database,
16
17
  type RuntimeDatabasePostureOptions,
@@ -56,14 +57,17 @@ import {
56
57
  type WorkerLifecycleState,
57
58
  } from "./http";
58
59
  import {
60
+ initializeContextCompactionMetrics,
59
61
  initializeWorkerOutcomeMetrics,
60
62
  normalizeTurnTaskQueueStats,
61
63
  observabilityEventLogger,
64
+ startContextCompactionPendingMonitor,
62
65
  startSessionRecoveryMonitor,
63
66
  startTurnCapacityMonitor,
64
67
  type TurnTaskQueueStats,
65
68
  } from "./observability-metrics";
66
69
  import {
70
+ resolveCatalogSettings,
67
71
  SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
68
72
  SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
69
73
  SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
@@ -224,6 +228,9 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
224
228
  options.activityDependencies?.observability ??
225
229
  createObservability(settings, { component: `worker-${options.role}` });
226
230
  initializeWorkerOutcomeMetrics(observability);
231
+ if (options.role === "turn") {
232
+ initializeContextCompactionMetrics(observability);
233
+ }
227
234
  if (options.role === "turn" && options.workflowBundle) {
228
235
  throw new Error("workflowBundle is valid only for the control worker role");
229
236
  }
@@ -799,11 +806,24 @@ export async function createOpenGeniWorkerService(
799
806
  let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
800
807
  let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
801
808
  let sessionRecoveryMonitor: ReturnType<typeof startSessionRecoveryMonitor> | undefined;
809
+ let contextCompactionPendingMonitor:
810
+ | ReturnType<typeof startContextCompactionPendingMonitor>
811
+ | undefined;
802
812
  const schedules: Array<{ close: () => Promise<void> }> = [];
803
813
  let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
804
814
  let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
805
815
 
806
816
  try {
817
+ const resolvedCatalog = await retryStartupDependency(
818
+ "model catalog",
819
+ () => resolveCatalogSettings(options.activityDependencies.db, settings),
820
+ { ...retryOptions, onRetry },
821
+ );
822
+ observability.info("OpenGeni model catalog resolved", {
823
+ role: options.role,
824
+ catalogSource: resolvedCatalog.source,
825
+ catalogVersion: resolvedCatalog.version,
826
+ });
807
827
  const needsSignaler =
808
828
  options.role === "turn" ||
809
829
  !options.activityDependencies.wakeSessionWorkflow ||
@@ -877,6 +897,10 @@ export async function createOpenGeniWorkerService(
877
897
  observability,
878
898
  read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db),
879
899
  });
900
+ contextCompactionPendingMonitor = startContextCompactionPendingMonitor({
901
+ observability,
902
+ read: async () => await getContextCompactionPendingSummary(options.activityDependencies.db),
903
+ });
880
904
  }
881
905
 
882
906
  if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
@@ -911,14 +935,18 @@ export async function createOpenGeniWorkerService(
911
935
  }
912
936
 
913
937
  if (options.http !== false) {
938
+ const databaseReady = dbReadyCheck(
939
+ options.http?.readinessDb ?? options.activityDependencies.db,
940
+ options.databasePosture,
941
+ );
914
942
  httpServer = startWorkerHttpServer({
915
943
  settings,
916
944
  observability,
917
945
  checks: {
918
- db: dbReadyCheck(
919
- options.http?.readinessDb ?? options.activityDependencies.db,
920
- options.databasePosture,
921
- ),
946
+ db: async () => {
947
+ await databaseReady();
948
+ await resolveCatalogSettings(options.activityDependencies.db, settings);
949
+ },
922
950
  nats: natsReadyCheck(options.activityDependencies.bus),
923
951
  temporal: temporalReadyCheck(workerBundle.connection),
924
952
  },
@@ -931,6 +959,7 @@ export async function createOpenGeniWorkerService(
931
959
  await Promise.allSettled([
932
960
  turnCapacityMonitor?.close(),
933
961
  sessionRecoveryMonitor?.close(),
962
+ contextCompactionPendingMonitor?.close(),
934
963
  workerBundle?.connection.close(),
935
964
  signaler?.close(),
936
965
  ...schedules.map((schedule) => schedule.close()),
@@ -954,6 +983,7 @@ export async function createOpenGeniWorkerService(
954
983
  await Promise.allSettled([
955
984
  turnCapacityMonitor?.close(),
956
985
  sessionRecoveryMonitor?.close(),
986
+ contextCompactionPendingMonitor?.close(),
957
987
  activeWorkerBundle.connection.close(),
958
988
  activeSignaler?.close(),
959
989
  ...schedules.map((schedule) => schedule.close()),
@@ -34,6 +34,10 @@ export type SessionRecoveryBacklog = {
34
34
  quiescence_missing: number;
35
35
  projection_stale: number;
36
36
  };
37
+ export type ContextCompactionPendingSummary = {
38
+ pendingCount: number;
39
+ oldestStartedAt: Date | null;
40
+ };
37
41
 
38
42
  export type TemporalTurnTaskQueueStats = {
39
43
  approximateBacklogCount?: unknown;
@@ -49,6 +53,7 @@ const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
49
53
  const modelRequestTrackers = new WeakMap<Observability, ModelRequestLifecycleMetrics>();
50
54
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
51
55
  const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
56
+ const initializedContextCompactionMetrics = new WeakSet<Observability>();
52
57
 
53
58
  // A worker process reaching one trillion tokens in one provider/cache counter is
54
59
  // already far outside an ordinary scrape lifetime. Refuse further increments
@@ -61,6 +66,18 @@ const WORKER_DEATH_RECOVERY_OUTCOMES: readonly WorkerDeathRecoveryOutcome[] = [
61
66
  "exhausted",
62
67
  ];
63
68
  const WORKER_DEATH_TIMEOUT_TYPES = ["heartbeat", "schedule_to_start"] as const;
69
+ const CONTEXT_COMPACTION_TRIGGERS = ["auto", "operator", "proactive", "overflow"] as const;
70
+
71
+ export type ContextCompactionTrigger = (typeof CONTEXT_COMPACTION_TRIGGERS)[number];
72
+
73
+ const CONTEXT_COMPACTION_STARTS_METRIC = {
74
+ name: "opengeni_context_compaction_starts_total",
75
+ help: "Total durable context compaction starts, by trigger.",
76
+ } as const;
77
+ const CONTEXT_COMPACTIONS_METRIC = {
78
+ name: "opengeni_context_compactions_total",
79
+ help: "Total completed context compactions, by trigger.",
80
+ } as const;
64
81
 
65
82
  export function observabilityEventLogger(observability: Observability): EventLogger {
66
83
  return {
@@ -346,6 +363,29 @@ export function initializeWorkerOutcomeMetrics(observability: Observability): vo
346
363
  }
347
364
  }
348
365
 
366
+ /**
367
+ * Publish every bounded compaction lifecycle series before the first event.
368
+ * The zero counters make the closed trigger catalog explicit before a scrape.
369
+ * Alerting uses the separate durable control-worker projection below; these
370
+ * process-local counters are lifecycle-rate diagnostics only.
371
+ */
372
+ export function initializeContextCompactionMetrics(observability: Observability): void {
373
+ if (initializedContextCompactionMetrics.has(observability)) return;
374
+ for (const trigger of CONTEXT_COMPACTION_TRIGGERS) {
375
+ observability.incrementCounter({
376
+ ...CONTEXT_COMPACTION_STARTS_METRIC,
377
+ labels: { trigger },
378
+ amount: 0,
379
+ });
380
+ observability.incrementCounter({
381
+ ...CONTEXT_COMPACTIONS_METRIC,
382
+ labels: { trigger },
383
+ amount: 0,
384
+ });
385
+ }
386
+ initializedContextCompactionMetrics.add(observability);
387
+ }
388
+
349
389
  export function recordWorkerDeathRecoveryMetrics(
350
390
  observability: Observability,
351
391
  input: {
@@ -830,6 +870,96 @@ export function startSessionRecoveryMonitor(input: {
830
870
  };
831
871
  }
832
872
 
873
+ export function recordContextCompactionPendingGauges(
874
+ observability: Observability,
875
+ summary: ContextCompactionPendingSummary,
876
+ nowMs = Date.now(),
877
+ ): void {
878
+ const oldestStartedAtMs = summary.oldestStartedAt?.getTime();
879
+ const oldestPendingAgeSeconds =
880
+ oldestStartedAtMs === undefined || !Number.isFinite(oldestStartedAtMs)
881
+ ? 0
882
+ : Math.max(0, (nowMs - oldestStartedAtMs) / 1_000);
883
+ observability.setGauge({
884
+ name: "opengeni_context_compaction_pending",
885
+ help: "Current exact active attempts whose latest automatic compaction landmark is still started.",
886
+ value: nonnegativeFinite(summary.pendingCount),
887
+ });
888
+ observability.setGauge({
889
+ name: "opengeni_context_compaction_oldest_pending_age_seconds",
890
+ help: "Age in seconds of the oldest durably pending automatic compaction, or zero when none are pending.",
891
+ value: oldestPendingAgeSeconds,
892
+ });
893
+ }
894
+
895
+ export function startContextCompactionPendingMonitor(input: {
896
+ observability: Observability;
897
+ read: () => Promise<ContextCompactionPendingSummary>;
898
+ intervalMs?: number;
899
+ now?: () => number;
900
+ }): { close: () => Promise<void> } {
901
+ const intervalMs = input.intervalMs ?? 60_000;
902
+ const now = input.now ?? Date.now;
903
+ const startedAt = now();
904
+ let lastSuccessAt: number | null = null;
905
+ let lastReadSucceeded = false;
906
+ let stopped = false;
907
+ let running: Promise<void> | null = null;
908
+ const recordStatus = () => {
909
+ const observedAt = now();
910
+ const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
911
+ const set = (name: string, help: string, value: number) =>
912
+ input.observability.setGauge({ name, help, value });
913
+ set(
914
+ "opengeni_context_compaction_monitor_last_read_success",
915
+ "Whether the latest durable context-compaction aggregate read completed successfully.",
916
+ lastReadSucceeded ? 1 : 0,
917
+ );
918
+ set(
919
+ "opengeni_context_compaction_monitor_last_success_timestamp_seconds",
920
+ "Unix timestamp of the latest successful durable context-compaction aggregate read, or zero before one succeeds.",
921
+ lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
922
+ );
923
+ set(
924
+ "opengeni_context_compaction_monitor_fresh",
925
+ "Whether durable context-compaction pending gauges have a successful read within three monitor intervals.",
926
+ lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
927
+ );
928
+ };
929
+ const refresh = () => {
930
+ recordStatus();
931
+ if (stopped || running) return;
932
+ running = input
933
+ .read()
934
+ .then((summary) => {
935
+ recordContextCompactionPendingGauges(input.observability, summary, now());
936
+ lastSuccessAt = now();
937
+ lastReadSucceeded = true;
938
+ recordStatus();
939
+ })
940
+ .catch((error) => {
941
+ lastReadSucceeded = false;
942
+ recordStatus();
943
+ input.observability.warn("context compaction monitor: durable aggregate read failed", {
944
+ error: error instanceof Error ? error.message : String(error),
945
+ });
946
+ })
947
+ .finally(() => {
948
+ running = null;
949
+ });
950
+ };
951
+ refresh();
952
+ const timer = setInterval(refresh, intervalMs);
953
+ timer.unref?.();
954
+ return {
955
+ close: async () => {
956
+ stopped = true;
957
+ clearInterval(timer);
958
+ await running;
959
+ },
960
+ };
961
+ }
962
+
833
963
  export function recordSandboxLeaseGauges(
834
964
  observability: Observability,
835
965
  counts: Partial<Record<SandboxLeaseLiveness, number>>,
@@ -1023,6 +1153,7 @@ export function recordSandboxRotationBacklogGauges(
1023
1153
  turnBlocked: number;
1024
1154
  directBlocked: number;
1025
1155
  processBlocked: number;
1156
+ interactionBlocked: number;
1026
1157
  },
1027
1158
  ): void {
1028
1159
  const values = {
@@ -1031,6 +1162,7 @@ export function recordSandboxRotationBacklogGauges(
1031
1162
  turn_blocked: backlog.turnBlocked,
1032
1163
  direct_blocked: backlog.directBlocked,
1033
1164
  process_blocked: backlog.processBlocked,
1165
+ interaction_blocked: backlog.interactionBlocked,
1034
1166
  } as const;
1035
1167
  for (const [kind, value] of Object.entries(values)) {
1036
1168
  observability.setGauge({
@@ -1228,6 +1360,7 @@ export type TurnStartupPhase =
1228
1360
  | "tool_optional_connect"
1229
1361
  | "tool_attempt_catalog_build"
1230
1362
  | "tool_attempt_catalog_persist"
1363
+ | "tool_workspace_gateway_catalog_build"
1231
1364
  | "post_tool_preparation"
1232
1365
  | "agent_construction"
1233
1366
  | "post_agent_preparation"
@@ -1276,6 +1409,9 @@ export type TurnStartupPhase =
1276
1409
  | "model_prepare_runner_before_mcp_tools"
1277
1410
  | "model_prepare_mcp_tools_snapshot"
1278
1411
  | "model_prepare_mcp_tools_before_input_filter"
1412
+ | "model_prepare_mcp_tools_before_repository_skill_discovery"
1413
+ | "model_prepare_repository_skill_discovery"
1414
+ | "model_prepare_repository_skill_discovery_before_input_filter"
1279
1415
  | "model_prepare_input_filter_base"
1280
1416
  | "model_prepare_input_filter_genesis"
1281
1417
  | "model_prepare_input_filter_host"
@@ -1749,15 +1885,14 @@ export function recordSessionEventPublishLatency(
1749
1885
  });
1750
1886
  }
1751
1887
 
1752
- // Context tokens per response span a wide range; buckets track the pressure toward
1753
- // a model's window so "sessions are running hot but never compacting" is queryable.
1888
+ // Context tokens per response span a wide range; buckets retain context-pressure
1889
+ // diagnostics while compaction alerting follows the durable model-aware start.
1754
1890
  const MODEL_INPUT_TOKENS_BUCKETS = [
1755
1891
  1_000, 5_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 300_000, 500_000, 1_000_000,
1756
1892
  ];
1757
1893
 
1758
- /** Observed input (context) tokens per model response — the context-pressure
1759
- * signal. Paired with `opengeni_context_compactions_total`, it makes the
1760
- * "compaction never firing while contexts run hot" failure mode expressible. */
1894
+ /** Observed input (context) tokens per model response for context-pressure
1895
+ * dashboards and provider diagnostics. */
1761
1896
  export function recordModelInputTokens(
1762
1897
  observability: Observability,
1763
1898
  provider: string,
@@ -1776,15 +1911,24 @@ export function recordModelInputTokens(
1776
1911
  });
1777
1912
  }
1778
1913
 
1779
- /** A context compaction actually fired, by trigger (operator | overflow | proactive
1780
- * | auto). The rate of this — against the input-tokens histogram above — is how an
1781
- * operator sees compaction working (or silently not). */
1782
- export function recordContextCompaction(observability: Observability, trigger: string): void {
1783
- observability.incrementCounter({
1784
- name: "opengeni_context_compactions_total",
1785
- help: "Total context compactions performed, by trigger.",
1786
- labels: { trigger },
1787
- });
1914
+ /** A context compaction successfully completed, by bounded trigger. */
1915
+ export function recordContextCompaction(
1916
+ observability: Observability,
1917
+ trigger: ContextCompactionTrigger,
1918
+ ): void {
1919
+ observability.incrementCounter({ ...CONTEXT_COMPACTIONS_METRIC, labels: { trigger } });
1920
+ }
1921
+
1922
+ /**
1923
+ * A durable context-compaction start, recorded only after the attempt-fenced
1924
+ * `compaction.started` transaction commits. This is the model-aware threshold
1925
+ * signal; it does not guess from a static token count shared by unlike models.
1926
+ */
1927
+ export function recordContextCompactionStarted(
1928
+ observability: Observability,
1929
+ trigger: ContextCompactionTrigger,
1930
+ ): void {
1931
+ observability.incrementCounter({ ...CONTEXT_COMPACTION_STARTS_METRIC, labels: { trigger } });
1788
1932
  }
1789
1933
 
1790
1934
  const MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2_048, 8_192];
@@ -1,6 +1,7 @@
1
1
  import {
2
2
  SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
3
3
  SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS,
4
+ effectiveSandboxDrainSnapshotTimeoutMs,
4
5
  sandboxArchiveCaptureTimeoutMs,
5
6
  type Settings,
6
7
  } from "@opengeni/config";
@@ -13,16 +14,21 @@ import {
13
14
 
14
15
  const STRICT_TIMEOUT_FENCE_MS = 1;
15
16
 
16
- export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTimeoutMs">): {
17
+ export function sandboxDrainTiming(
18
+ settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
19
+ ): {
17
20
  snapshotTimeoutMs: number;
18
21
  captureTimeoutMs: number;
19
22
  timeoutClass: SandboxDrainTimeoutClass;
20
23
  activityTimeoutMs: number;
21
24
  } {
22
- const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
25
+ const snapshotTimeoutMs = effectiveSandboxDrainSnapshotTimeoutMs(settings);
26
+ const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs({
27
+ sandboxSnapshotTimeoutMs: snapshotTimeoutMs,
28
+ });
23
29
  const timeoutClass = sandboxDrainTimeoutClass(captureTimeoutMs);
24
30
  return {
25
- snapshotTimeoutMs: settings.sandboxSnapshotTimeoutMs,
31
+ snapshotTimeoutMs,
26
32
  captureTimeoutMs,
27
33
  timeoutClass,
28
34
  activityTimeoutMs: sandboxDrainActivityTimeoutMs(timeoutClass),
@@ -30,7 +36,7 @@ export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTime
30
36
  }
31
37
 
32
38
  export function assertSandboxReaperActivityTimeout(
33
- settings: Pick<Settings, "sandboxSnapshotTimeoutMs">,
39
+ settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
34
40
  ): void {
35
41
  const timing = sandboxDrainTiming(settings);
36
42
  if (timing.snapshotTimeoutMs >= timing.captureTimeoutMs) {