@opengeni/worker-bundle 0.20.3 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
package/src/http.ts CHANGED
@@ -118,12 +118,22 @@ export function dbReadyCheck(
118
118
  db: Database,
119
119
  posture?: RuntimeDatabasePostureOptions,
120
120
  ): () => Promise<void> {
121
- return async () => {
122
- if (posture) {
123
- await assertRuntimeDatabasePosture(db, posture);
124
- return;
125
- }
126
- await db.execute(dbSql`select 1`);
121
+ let inFlight: Promise<void> | undefined;
122
+ return () => {
123
+ // The HTTP timeout cannot cancel a driver query. Reuse an outstanding
124
+ // attempt so repeated kubelet probes cannot queue an unbounded tail behind
125
+ // one slow or disconnected database connection.
126
+ if (inFlight) return inFlight;
127
+ inFlight = (async () => {
128
+ if (posture) {
129
+ await assertRuntimeDatabasePosture(db, posture);
130
+ return;
131
+ }
132
+ await db.execute(dbSql`select 1`);
133
+ })().finally(() => {
134
+ inFlight = undefined;
135
+ });
136
+ return inFlight;
127
137
  };
128
138
  }
129
139
 
package/src/index.ts CHANGED
@@ -742,8 +742,13 @@ export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependen
742
742
  * in the same deployment owns them.
743
743
  */
744
744
  internalSchedules?: "role-default" | "none";
745
- /** Set false only when the host exposes equivalent lifecycle endpoints itself. */
746
- http?: false | { readinessTimeoutMs?: number };
745
+ /**
746
+ * Set false only when the host exposes equivalent lifecycle endpoints itself.
747
+ * A dedicated readiness database prevents ordinary activity-pool saturation
748
+ * from making an otherwise healthy Temporal worker fail Kubernetes readiness.
749
+ * The embedding host retains ownership of this handle.
750
+ */
751
+ http?: false | { readinessTimeoutMs?: number; readinessDb?: Database };
747
752
  };
748
753
 
749
754
  export type OpenGeniWorkerService = {
@@ -894,7 +899,10 @@ export async function createOpenGeniWorkerService(
894
899
  settings,
895
900
  observability,
896
901
  checks: {
897
- db: dbReadyCheck(options.activityDependencies.db, options.databasePosture),
902
+ db: dbReadyCheck(
903
+ options.http?.readinessDb ?? options.activityDependencies.db,
904
+ options.databasePosture,
905
+ ),
898
906
  nats: natsReadyCheck(options.activityDependencies.bus),
899
907
  temporal: temporalReadyCheck(workerBundle.connection),
900
908
  },
@@ -1027,10 +1035,22 @@ export async function startWorker() {
1027
1035
  ...(searchPath ? { searchPath } : {}),
1028
1036
  rlsStrategy: settings.rlsStrategy,
1029
1037
  });
1038
+ // Readiness must prove PostgreSQL independently of activity demand. Sharing
1039
+ // the default application pool makes the probe wait behind live turns when
1040
+ // all ten slots are occupied, even while PostgreSQL itself is healthy. Since
1041
+ // Kubernetes readiness cannot stop Temporal polling, that false negative
1042
+ // only wedges rollouts and service health; reserve one probe connection.
1043
+ const readinessDbClient = createDb(settings.databaseUrl, {
1044
+ ...(searchPath ? { searchPath } : {}),
1045
+ rlsStrategy: settings.rlsStrategy,
1046
+ max: 1,
1047
+ });
1030
1048
  const databasePosture = {
1031
1049
  rlsStrategy: settings.rlsStrategy,
1032
1050
  expectedRole: settings.runtimeDatabaseRole,
1033
1051
  targetSchema: settings.dbSchema.trim() || "public",
1052
+ organizationTenancyCanonicalActivationEnabled:
1053
+ settings.organizationTenancyCanonicalActivationEnabled,
1034
1054
  } as const;
1035
1055
  const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
1036
1056
  let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
@@ -1056,6 +1076,7 @@ export async function startWorker() {
1056
1076
  role,
1057
1077
  settings,
1058
1078
  databasePosture,
1079
+ http: { readinessDb: readinessDbClient.db },
1059
1080
  activityDependencies: {
1060
1081
  observability,
1061
1082
  db: dbClient.db,
@@ -1063,10 +1084,24 @@ export async function startWorker() {
1063
1084
  },
1064
1085
  });
1065
1086
  } finally {
1066
- await Promise.allSettled([bus?.close(), dbClient.close()]);
1087
+ await Promise.allSettled([bus?.close(), readinessDbClient.close(), dbClient.close()]);
1067
1088
  }
1068
1089
  }
1069
1090
 
1070
1091
  if (import.meta.main) {
1071
- await startWorker();
1092
+ // `bun --watch` keeps its supervisor alive after the application consumes
1093
+ // SIGINT. Once the worker, NATS, DB, and Temporal resources have all closed,
1094
+ // explicitly terminate that supervisor too; otherwise a later file edit can
1095
+ // resurrect this stopped process as a duplicate worker with stale env/code.
1096
+ let interrupted = false;
1097
+ const markInterrupted = () => {
1098
+ interrupted = true;
1099
+ };
1100
+ process.once("SIGINT", markInterrupted);
1101
+ try {
1102
+ await startWorker();
1103
+ } finally {
1104
+ process.off("SIGINT", markInterrupted);
1105
+ if (interrupted) process.exit(0);
1106
+ }
1072
1107
  }
@@ -0,0 +1,2 @@
1
+ export const KNOWLEDGE_SOURCE_SYNC_ACTIVITY_HEARTBEAT_TIMEOUT_MS = 120_000;
2
+ export const KNOWLEDGE_SOURCE_SYNC_ACTIVITY_MAXIMUM_ATTEMPTS = 1;
@@ -1,8 +1,11 @@
1
1
  import type {
2
+ ModelContextContributionSource,
3
+ ModelContextContributionSummary,
2
4
  PreferenceRegistrySnapshot,
3
5
  ResolvedCompanyProfileSnapshot,
4
6
  ResolvedWorkspaceInstructionPolicySnapshot,
5
- WorkspaceMemoryPromptMode,
7
+ SandboxBackend,
8
+ HistoricalMemoryPromptMode,
6
9
  } from "@opengeni/contracts";
7
10
  import { composeRuntimeSkills, type RuntimeSkillActivation } from "@opengeni/runtime";
8
11
 
@@ -14,13 +17,7 @@ export type CompanyBrainContributionCategory =
14
17
 
15
18
  export type CompanyBrainContribution = Readonly<{
16
19
  category: CompanyBrainContributionCategory;
17
- source:
18
- | "workspace_instruction_policy"
19
- | "legacy_workspace_instructions"
20
- | "preference_registry_descriptor"
21
- | "company_profile"
22
- | "legacy_memory_v1"
23
- | "runtime_skill_catalog";
20
+ source: ModelContextContributionSource;
24
21
  inclusionReason:
25
22
  | "active_instruction_policy"
26
23
  | "legacy_instruction_fallback"
@@ -33,11 +30,35 @@ export type CompanyBrainContribution = Readonly<{
33
30
  estimatedTokens: number;
34
31
  }>;
35
32
 
33
+ export function summarizeCompanyBrainContributions(
34
+ receipt: CompanyBrainContributionReceipt,
35
+ ): readonly ModelContextContributionSummary[] {
36
+ const bySource = new Map<ModelContextContributionSource, ModelContextContributionSummary>();
37
+ for (const entry of receipt.contributions) {
38
+ const current = bySource.get(entry.source);
39
+ bySource.set(entry.source, {
40
+ source: entry.source,
41
+ items: (current?.items ?? 0) + 1,
42
+ utf8Bytes: (current?.utf8Bytes ?? 0) + entry.utf8Bytes,
43
+ estimatedTokens: (current?.estimatedTokens ?? 0) + entry.estimatedTokens,
44
+ });
45
+ }
46
+ return Object.freeze([...bySource.values()].map((summary) => Object.freeze(summary)));
47
+ }
48
+
49
+ export function modelVisibleCompanyBrainSkillActivations(
50
+ sandboxBackend: SandboxBackend,
51
+ skillActivations: readonly RuntimeSkillActivation[],
52
+ ): readonly RuntimeSkillActivation[] {
53
+ return sandboxBackend === "none" ? [] : skillActivations;
54
+ }
55
+
36
56
  export type CompanyBrainContributionReceipt = Readonly<{
57
+ contextSelectionReceiptId: string;
37
58
  attemptId: string;
38
59
  turnId: string;
39
60
  sessionRole: "root" | "child";
40
- memoryPromptMode: WorkspaceMemoryPromptMode;
61
+ memoryPromptMode: HistoricalMemoryPromptMode;
41
62
  instructionPolicySnapshotId: string;
42
63
  preferenceSnapshotId: string | null;
43
64
  companyProfileSnapshotId: string;
@@ -51,10 +72,11 @@ export type CompanyBrainContributionReceipt = Readonly<{
51
72
  * of truth for billing and context pressure.
52
73
  */
53
74
  export function buildCompanyBrainContributionReceipt(input: {
75
+ contextSelectionReceiptId: string;
54
76
  attemptId: string;
55
77
  turnId: string;
56
78
  nestedAgentDepth: number;
57
- memoryPromptMode: WorkspaceMemoryPromptMode;
79
+ memoryPromptMode: HistoricalMemoryPromptMode;
58
80
  instructionPolicy: ResolvedWorkspaceInstructionPolicySnapshot;
59
81
  workspaceAgentInstructions: string | null;
60
82
  preferences: PreferenceRegistrySnapshot | null;
@@ -131,6 +153,7 @@ export function buildCompanyBrainContributionReceipt(input: {
131
153
  );
132
154
  }
133
155
  return Object.freeze({
156
+ contextSelectionReceiptId: input.contextSelectionReceiptId,
134
157
  attemptId: input.attemptId,
135
158
  turnId: input.turnId,
136
159
  sessionRole: input.nestedAgentDepth === 0 ? "root" : "child",
@@ -4,6 +4,11 @@ import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
4
4
  import type { EventLogger } from "@opengeni/events";
5
5
  import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
6
6
  import type { CompanyBrainContributionReceipt } from "./model-context-contributions";
7
+ import {
8
+ OPENSANDBOX_BATCHSANDBOX_PHASES,
9
+ OPENSANDBOX_WORKLOAD_POD_CONDITIONS,
10
+ type OpenSandboxKubernetesInventory,
11
+ } from "./opensandbox-kubernetes-inventory";
7
12
  import {
8
13
  SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
9
14
  modelUsageTokenCountOrNull,
@@ -35,6 +40,7 @@ export type TemporalTurnTaskQueueStats = {
35
40
  } | null;
36
41
 
37
42
  const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
43
+ const modelRequestTrackers = new WeakMap<Observability, ModelRequestLifecycleMetrics>();
38
44
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
39
45
  const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
40
46
 
@@ -116,6 +122,47 @@ export function runtimeMetricsHooksForObservability(
116
122
  labels: { backend, stage },
117
123
  });
118
124
  },
125
+ onSandboxProviderApiThrottle: ({ backend, operation }) => {
126
+ observability.incrementCounter({
127
+ name: "opengeni_sandbox_provider_api_throttles_total",
128
+ help: "Total sandbox provider API throttle responses.",
129
+ labels: { backend, operation },
130
+ });
131
+ },
132
+ onSandboxTtlRenewal: ({ backend, outcome }) => {
133
+ observability.incrementCounter({
134
+ name: "opengeni_sandbox_ttl_renewals_total",
135
+ help: "Total renewable sandbox provider TTL refresh attempts.",
136
+ labels: { backend, outcome },
137
+ });
138
+ },
139
+ onOpenSandboxSignedEndpoint: ({ outcome, port }) => {
140
+ observability.incrementCounter({
141
+ name: "opengeni_opensandbox_signed_endpoint_total",
142
+ help: "OpenSandbox signed Channel B mint and host-fetch outcomes.",
143
+ labels: { outcome, port: String(port) },
144
+ });
145
+ },
146
+ onWorkspaceArchiveObject: ({ outcome, backend }) => {
147
+ observability.incrementCounter({
148
+ name: "opengeni_workspace_archive_object_total",
149
+ help: "Workspace archive object-storage put/delete outcomes.",
150
+ labels: { outcome, backend },
151
+ });
152
+ },
153
+ onMcpToolCall: ({ outcome, durationSeconds }) => {
154
+ observability.incrementCounter({
155
+ name: "opengeni_mcp_tool_calls_total",
156
+ help: "Total physical MCP tool calls by bounded structural outcome.",
157
+ labels: { outcome },
158
+ });
159
+ observability.observeHistogram({
160
+ name: "opengeni_mcp_tool_call_duration_seconds",
161
+ help: "MCP tool-call duration in seconds by bounded structural outcome.",
162
+ labels: { outcome },
163
+ value: durationSeconds,
164
+ });
165
+ },
119
166
  onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
120
167
  observability.incrementCounter({
121
168
  name: "opengeni_machine_op_total",
@@ -370,6 +417,111 @@ export class TurnLifecycleMetrics {
370
417
  }
371
418
  }
372
419
 
420
+ export function modelRequestLifecycleMetricsFor(
421
+ observability: Observability,
422
+ ): ModelRequestLifecycleMetrics {
423
+ const existing = modelRequestTrackers.get(observability);
424
+ if (existing) return existing;
425
+ const tracker = new ModelRequestLifecycleMetrics(observability);
426
+ modelRequestTrackers.set(observability, tracker);
427
+ return tracker;
428
+ }
429
+
430
+ export class ModelRequestLifecycleMetrics {
431
+ private readonly requests = new Map<
432
+ string,
433
+ { provider: string; startedAt: number; lastEventAt: number }
434
+ >();
435
+ private readonly providers = new Set<string>();
436
+ private timer: ReturnType<typeof setInterval> | null = null;
437
+
438
+ constructor(
439
+ private readonly observability: Observability,
440
+ private readonly options: {
441
+ now?: () => number;
442
+ refreshIntervalMs?: number;
443
+ } = {},
444
+ ) {}
445
+
446
+ start(key: string, provider: string): void {
447
+ const now = this.now();
448
+ this.providers.add(provider);
449
+ this.requests.set(key, { provider, startedAt: now, lastEventAt: now });
450
+ this.ensureTimer();
451
+ this.refreshGauges();
452
+ }
453
+
454
+ event(key: string, interEventGapMs?: number): void {
455
+ const request = this.requests.get(key);
456
+ if (!request) return;
457
+ request.lastEventAt = this.now();
458
+ this.observability.incrementCounter({
459
+ name: "opengeni_model_request_stream_events_total",
460
+ help: "Complete, valid provider SSE data events by provider.",
461
+ labels: { provider: request.provider },
462
+ });
463
+ if (interEventGapMs !== undefined) {
464
+ this.observability.observeHistogram({
465
+ name: "opengeni_model_request_stream_event_gap_seconds",
466
+ help: "Seconds between complete, valid provider SSE data events.",
467
+ buckets: MODEL_REQUEST_PHASE_BUCKETS,
468
+ labels: { provider: request.provider },
469
+ value: Math.max(0, interEventGapMs / 1000),
470
+ });
471
+ }
472
+ }
473
+
474
+ finish(key: string): void {
475
+ this.requests.delete(key);
476
+ this.refreshGauges();
477
+ if (this.requests.size === 0) this.stopTimer();
478
+ }
479
+
480
+ refreshGauges(): void {
481
+ const now = this.now();
482
+ for (const provider of this.providers) {
483
+ const active = [...this.requests.values()].filter((request) => request.provider === provider);
484
+ this.observability.setGauge({
485
+ name: "opengeni_model_requests_inflight",
486
+ help: "Current in-flight provider model requests in this worker process.",
487
+ labels: { provider },
488
+ value: active.length,
489
+ });
490
+ this.observability.setGauge({
491
+ name: "opengeni_model_request_oldest_no_event_age_seconds",
492
+ help: "Seconds since the least recently progressing in-flight provider request received a valid SSE event.",
493
+ labels: { provider },
494
+ value:
495
+ active.length === 0
496
+ ? 0
497
+ : Math.max(0, (now - Math.min(...active.map((request) => request.lastEventAt))) / 1000),
498
+ });
499
+ }
500
+ }
501
+
502
+ stop(): void {
503
+ this.requests.clear();
504
+ this.refreshGauges();
505
+ this.stopTimer();
506
+ }
507
+
508
+ private ensureTimer(): void {
509
+ if (this.timer) return;
510
+ this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15_000);
511
+ this.timer.unref?.();
512
+ }
513
+
514
+ private stopTimer(): void {
515
+ if (!this.timer) return;
516
+ clearInterval(this.timer);
517
+ this.timer = null;
518
+ }
519
+
520
+ private now(): number {
521
+ return this.options.now?.() ?? Date.now();
522
+ }
523
+ }
524
+
373
525
  export function recordTurnsQueuedGauge(observability: Observability, value: number): void {
374
526
  observability.setGauge({
375
527
  name: "opengeni_turns_queued",
@@ -538,12 +690,45 @@ export function recordSandboxLeaseGauges(
538
690
  }
539
691
  }
540
692
 
693
+ export function recordOpenSandboxKubernetesInventoryGauges(
694
+ observability: Observability,
695
+ inventory: OpenSandboxKubernetesInventory,
696
+ ): void {
697
+ for (const phase of OPENSANDBOX_BATCHSANDBOX_PHASES) {
698
+ observability.setGauge({
699
+ name: "opengeni_opensandbox_batchsandboxes",
700
+ help: "Current OpenSandbox BatchSandboxes by bounded lifecycle phase.",
701
+ labels: { phase },
702
+ value: inventory.batchSandboxPhases[phase],
703
+ });
704
+ }
705
+ for (const condition of OPENSANDBOX_WORKLOAD_POD_CONDITIONS) {
706
+ observability.setGauge({
707
+ name: "opengeni_opensandbox_workload_pods",
708
+ help: "Current OpenSandbox workload Pods by bounded operational condition.",
709
+ labels: { condition },
710
+ value: inventory.workloadPodConditions[condition],
711
+ });
712
+ }
713
+ observability.setGauge({
714
+ name: "opengeni_opensandbox_cleanup_stuck",
715
+ help: "Current OpenSandbox BatchSandboxes deleting past the cleanup threshold with finalizers.",
716
+ value: inventory.cleanupStuck,
717
+ });
718
+ observability.setGauge({
719
+ name: "opengeni_opensandbox_expiration_overdue",
720
+ help: "Current OpenSandbox BatchSandboxes still present past the provider-expiry threshold.",
721
+ value: inventory.expirationOverdue,
722
+ });
723
+ }
724
+
541
725
  export const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [
542
726
  "leases",
543
727
  "checkpoint_artifacts",
544
728
  "rotation_backlog",
545
729
  "retained_processes",
546
730
  "expired_drains",
731
+ "opensandbox_kubernetes",
547
732
  ] as const;
548
733
 
549
734
  export type SandboxInventoryProjectionDomain =
@@ -921,6 +1106,17 @@ export type TurnStartupPhase =
921
1106
  | "model_prepare_sandbox_client_delete"
922
1107
  | "model_prepare_sandbox_client_state_serialize"
923
1108
  | "model_prepare_sandbox_client_reuse_check"
1109
+ | "model_prepare_sandbox_workspace_mutation_admission"
1110
+ | "model_prepare_sandbox_workspace_mutation_provider"
1111
+ | "model_prepare_sandbox_workspace_mutation_settlement"
1112
+ | "model_prepare_sandbox_first_routed_resolution_other"
1113
+ | "model_prepare_sandbox_first_routed_mutation_admission"
1114
+ | "model_prepare_sandbox_first_routed_provider_operation"
1115
+ | "model_prepare_sandbox_first_routed_mutation_settlement"
1116
+ | "model_prepare_sandbox_first_routed_other"
1117
+ | "model_prepare_sandbox_snapshot_wait"
1118
+ | "model_prepare_runner_before_first_sandbox_operation"
1119
+ | "model_prepare_sdk_after_first_sandbox_operation"
924
1120
  | "model_prepare_runner_before_mcp_tools"
925
1121
  | "model_prepare_mcp_tools_snapshot"
926
1122
  | "model_prepare_mcp_tools_before_input_filter"
@@ -938,6 +1134,7 @@ export type TurnStartupPhase =
938
1134
  | "stream_bootstrap";
939
1135
 
940
1136
  export type TurnStartupOutcome = "completed" | "failed";
1137
+ export type TurnStartupMilestone = "queue" | "provider_dispatch" | "first_byte";
941
1138
  export type TurnStartupCache = "hit" | "miss" | "disabled" | "none";
942
1139
  export type TurnStartupCountBucket = "0" | "1" | "2-5" | "6-20" | "21+" | "unknown";
943
1140
  export type TurnSandboxEstablishPolicy = "eager" | "on-demand";
@@ -947,12 +1144,45 @@ export type TurnSandboxEstablishReason =
947
1144
  | "machine_primary"
948
1145
  | "backend_none"
949
1146
  | "initial_run_credentials"
1147
+ | "initial_run_credentials_deferred"
950
1148
  | "generated_video_files"
951
1149
  | "signed_file_resources";
1150
+ export type SandboxLogicalProvisionCategory =
1151
+ | "create"
1152
+ | "resume"
1153
+ | "exec_readiness"
1154
+ | "sibling_warming"
1155
+ | "lease_superseded"
1156
+ | "drain_capture_wait"
1157
+ | "archive_recovery"
1158
+ | "provider_transport"
1159
+ | "configuration"
1160
+ | "unknown";
1161
+ export type SandboxLogicalProvisionStage =
1162
+ | "create"
1163
+ | "resume"
1164
+ | "exec_readiness"
1165
+ | "sibling_warming"
1166
+ | "lease_admission"
1167
+ | "lifecycle_wait"
1168
+ | "archive_recovery"
1169
+ | "configuration"
1170
+ | "provider_transport"
1171
+ | "unknown";
1172
+ export type SandboxLogicalProvisionOutcome = "completed" | "expected_transition" | "failed";
1173
+ export type SandboxProvisionAttemptOutcome = "completed" | "retrying" | "failed";
952
1174
 
953
1175
  const TURN_STARTUP_PHASE_BUCKETS = [
954
1176
  0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120,
955
1177
  ];
1178
+ const TURN_STARTUP_MILESTONE_BUCKETS = [
1179
+ ...TURN_STARTUP_PHASE_BUCKETS,
1180
+ 300,
1181
+ 600,
1182
+ 1_800,
1183
+ 3_600,
1184
+ 7_200,
1185
+ ];
956
1186
 
957
1187
  export function turnStartupCountBucket(count: number | null): TurnStartupCountBucket {
958
1188
  if (count === null || !Number.isFinite(count) || count < 0) return "unknown";
@@ -1020,6 +1250,103 @@ export function recordTurnSandboxEstablishPolicy(
1020
1250
  });
1021
1251
  }
1022
1252
 
1253
+ /** Physical establish/resume attempts inside one logical provision. Safe
1254
+ * lifecycle retries are counted here, never as another user-visible failure. */
1255
+ export function recordSandboxProvisionAttempt(
1256
+ observability: Observability,
1257
+ input: {
1258
+ backend: string;
1259
+ stage: SandboxLogicalProvisionStage;
1260
+ category: SandboxLogicalProvisionCategory;
1261
+ outcome: SandboxProvisionAttemptOutcome;
1262
+ durationSeconds: number;
1263
+ },
1264
+ ): void {
1265
+ const labels = {
1266
+ backend: input.backend,
1267
+ stage: input.stage,
1268
+ category: input.category,
1269
+ outcome: input.outcome,
1270
+ };
1271
+ try {
1272
+ observability.incrementCounter({
1273
+ name: "opengeni_sandbox_provision_attempts_total",
1274
+ help: "Internal physical attempts within logical sandbox provisions.",
1275
+ labels,
1276
+ });
1277
+ observability.observeHistogram({
1278
+ name: "opengeni_sandbox_provision_attempt_duration_seconds",
1279
+ help: "Internal sandbox provision attempt duration by bounded structural outcome.",
1280
+ buckets: TURN_STARTUP_PHASE_BUCKETS,
1281
+ labels,
1282
+ value: Math.max(0, input.durationSeconds),
1283
+ });
1284
+ } catch {
1285
+ try {
1286
+ observability.incrementCounter({
1287
+ name: "opengeni_observability_observer_errors_total",
1288
+ help: "Observability observer failures isolated from product execution.",
1289
+ labels: { observer: "sandbox_provision" },
1290
+ });
1291
+ } catch {
1292
+ // The registry itself is unhealthy; provisioning remains authoritative.
1293
+ }
1294
+ }
1295
+ }
1296
+
1297
+ /** One terminal observation for one correlation-qualified logical provision. */
1298
+ export function recordSandboxLogicalProvision(
1299
+ observability: Observability,
1300
+ input: {
1301
+ backend: string;
1302
+ stage: SandboxLogicalProvisionStage;
1303
+ category: SandboxLogicalProvisionCategory | "none";
1304
+ outcome: SandboxLogicalProvisionOutcome;
1305
+ expected: boolean;
1306
+ internalAttempts: number;
1307
+ durationSeconds: number;
1308
+ },
1309
+ ): void {
1310
+ const labels = {
1311
+ backend: input.backend,
1312
+ stage: input.stage,
1313
+ category: input.category,
1314
+ outcome: input.outcome,
1315
+ expected: input.expected ? "true" : "false",
1316
+ };
1317
+ try {
1318
+ observability.incrementCounter({
1319
+ name: "opengeni_sandbox_provisions_total",
1320
+ help: "Logical sandbox provisions by terminal outcome and bounded failure taxonomy.",
1321
+ labels,
1322
+ });
1323
+ observability.observeHistogram({
1324
+ name: "opengeni_sandbox_provision_duration_seconds",
1325
+ help: "Logical sandbox provision duration including internal safe retries.",
1326
+ buckets: TURN_STARTUP_PHASE_BUCKETS,
1327
+ labels,
1328
+ value: Math.max(0, input.durationSeconds),
1329
+ });
1330
+ observability.observeHistogram({
1331
+ name: "opengeni_sandbox_provision_internal_attempts",
1332
+ help: "Internal physical attempts per terminal logical sandbox provision.",
1333
+ buckets: [1, 2, 3, 4, 5, 8],
1334
+ labels,
1335
+ value: Math.max(1, input.internalAttempts),
1336
+ });
1337
+ } catch {
1338
+ try {
1339
+ observability.incrementCounter({
1340
+ name: "opengeni_observability_observer_errors_total",
1341
+ help: "Observability observer failures isolated from product execution.",
1342
+ labels: { observer: "sandbox_provision" },
1343
+ });
1344
+ } catch {
1345
+ // The registry itself is unhealthy; provisioning remains authoritative.
1346
+ }
1347
+ }
1348
+ }
1349
+
1023
1350
  export function recordTurnWorkerPreparationTotal(
1024
1351
  observability: Observability,
1025
1352
  input: {
@@ -1042,6 +1369,36 @@ export function recordTurnWorkerPreparationTotal(
1042
1369
  });
1043
1370
  }
1044
1371
 
1372
+ /**
1373
+ * Measure cumulative user-visible startup latency from the durable turn queue
1374
+ * timestamp to a bounded milestone. Unlike the per-phase histogram above,
1375
+ * these samples are end-to-end and can therefore back real queue/dispatch/TTFB
1376
+ * SLOs without adding turn or session identifiers to Prometheus.
1377
+ */
1378
+ export function recordTurnStartupMilestone(
1379
+ observability: Observability,
1380
+ input: {
1381
+ milestone: TurnStartupMilestone;
1382
+ provider: string;
1383
+ backend: string;
1384
+ outcome: TurnStartupOutcome;
1385
+ durationSeconds: number;
1386
+ },
1387
+ ): void {
1388
+ observability.observeHistogram({
1389
+ name: "opengeni_turn_startup_milestone_duration_seconds",
1390
+ help: "Cumulative seconds from durable turn queueing to a bounded startup milestone.",
1391
+ buckets: TURN_STARTUP_MILESTONE_BUCKETS,
1392
+ labels: {
1393
+ milestone: input.milestone,
1394
+ provider: input.provider,
1395
+ backend: input.backend,
1396
+ outcome: input.outcome,
1397
+ },
1398
+ value: Math.max(0, input.durationSeconds),
1399
+ });
1400
+ }
1401
+
1045
1402
  // ── Streaming SLIs ────────────────────────────────────────────────────────────
1046
1403
  // Instruments the token-streaming pipeline so "streaming is sluggish" is a number,
1047
1404
  // not a vibe. Split across the three attributable stages so an operator can tell