@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +66 -0
  42. package/dist/activities/image-generation-references.d.ts +2 -0
  43. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  44. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  45. package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
  46. package/dist/activities/knowledge-source-sync.d.ts +33 -1
  47. package/dist/activities/mcp-credentials.d.ts +9 -3
  48. package/dist/activities/open-suffix-resume.d.ts +77 -0
  49. package/dist/activities/packs.d.ts +3 -11
  50. package/dist/activities/parent-wake.d.ts +19 -0
  51. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  52. package/dist/activities/retained-screenshots.d.ts +1 -1
  53. package/dist/activities/rig-verification.d.ts +1 -1
  54. package/dist/activities/run-input.d.ts +16 -7
  55. package/dist/activities/sandbox-images.d.ts +1 -14
  56. package/dist/activities/sandbox-lease.d.ts +21 -3
  57. package/dist/activities/scheduled-tasks.d.ts +47 -0
  58. package/dist/activities/types.d.ts +23 -2
  59. package/dist/activities/video-media-validation.d.ts +1 -1
  60. package/dist/activities/video-output-retention.d.ts +1 -2
  61. package/dist/activities/xai-auth.d.ts +3 -0
  62. package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
  63. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  64. package/dist/activities-control.d.ts +6 -2
  65. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  66. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  67. package/dist/activities.d.ts +7 -3
  68. package/dist/artifact-materializer-entry.js +41 -22
  69. package/dist/artifact-materializer-entry.js.map +1 -1
  70. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  71. package/dist/chunk-CE7TPKBS.js.map +1 -0
  72. package/dist/chunk-CNYGVRKF.js +4698 -0
  73. package/dist/chunk-CNYGVRKF.js.map +1 -0
  74. package/dist/connection-provider-use.d.ts +18 -0
  75. package/dist/editable-artifact-materializer.d.ts +4 -3
  76. package/dist/github-rest-mcp.d.ts +26 -0
  77. package/dist/index.d.ts +10 -2
  78. package/dist/index.js +58 -14
  79. package/dist/index.js.map +1 -1
  80. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  81. package/dist/model-context-contributions.d.ts +8 -4
  82. package/dist/observability-metrics.d.ts +63 -4
  83. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  84. package/dist/personal-github-git-credentials.d.ts +9 -0
  85. package/dist/pr-review-credentials.d.ts +6 -0
  86. package/dist/sandbox-archive-storage.d.ts +45 -0
  87. package/dist/sandbox-resume.d.ts +65 -6
  88. package/dist/sandbox-routing.d.ts +38 -2
  89. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  90. package/dist/workflow-bundle.js +87 -28
  91. package/dist/workflows/activities.d.ts +1 -0
  92. package/dist/workflows/automations.d.ts +6 -0
  93. package/dist/workflows/document-index.d.ts +1 -0
  94. package/dist/workflows.d.ts +1 -0
  95. package/package.json +17 -17
  96. package/src/activities/agent-turn/admission.ts +292 -0
  97. package/src/activities/agent-turn/agent-build.ts +802 -0
  98. package/src/activities/agent-turn/claim.ts +590 -0
  99. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  100. package/src/activities/agent-turn/codex.ts +94 -0
  101. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  102. package/src/activities/agent-turn/credential-leases.ts +240 -0
  103. package/src/activities/agent-turn/errors.ts +1068 -0
  104. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  105. package/src/activities/agent-turn/file-resources.ts +227 -0
  106. package/src/activities/agent-turn/finalization.ts +754 -0
  107. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  108. package/src/activities/agent-turn/governance-model.ts +465 -0
  109. package/src/activities/agent-turn/history-sink.ts +139 -0
  110. package/src/activities/agent-turn/history.ts +200 -0
  111. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  112. package/src/activities/agent-turn/model-usage.ts +823 -0
  113. package/src/activities/agent-turn/quiescence.ts +353 -0
  114. package/src/activities/agent-turn/run-credentials.ts +761 -0
  115. package/src/activities/agent-turn/run.ts +1530 -0
  116. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  117. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  118. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  119. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  120. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  121. package/src/activities/agent-turn/tool-environment.ts +729 -0
  122. package/src/activities/agent-turn/tool-policy.ts +216 -0
  123. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  124. package/src/activities/agent-turn/turn-context.ts +281 -0
  125. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  126. package/src/activities/agent-turn.ts +12 -12575
  127. package/src/activities/api-integrations.ts +3 -0
  128. package/src/activities/automations.ts +214 -0
  129. package/src/activities/codemode-dispatcher.ts +12 -1
  130. package/src/activities/common.ts +1 -5
  131. package/src/activities/connector-attachments.ts +168 -0
  132. package/src/activities/context-compaction.ts +18 -0
  133. package/src/activities/documents.ts +6 -4
  134. package/src/activities/environment.ts +151 -55
  135. package/src/activities/generated-images.ts +26 -11
  136. package/src/activities/git-credential-renewal.ts +13 -3
  137. package/src/activities/goals.ts +85 -19
  138. package/src/activities/google-drive-provider.ts +178 -0
  139. package/src/activities/google-drive-publication.ts +938 -0
  140. package/src/activities/image-generation-references.ts +11 -2
  141. package/src/activities/incident-telemetry-authority.ts +327 -0
  142. package/src/activities/incident-telemetry-preflight.ts +317 -0
  143. package/src/activities/knowledge-source-sync-driver.ts +23 -0
  144. package/src/activities/knowledge-source-sync.ts +556 -40
  145. package/src/activities/mcp-credentials.ts +191 -17
  146. package/src/activities/open-suffix-resume.ts +422 -0
  147. package/src/activities/packs.ts +12 -60
  148. package/src/activities/parent-wake.ts +64 -14
  149. package/src/activities/retained-screenshot-reaper.ts +6 -3
  150. package/src/activities/retained-screenshots.ts +15 -47
  151. package/src/activities/rig-verification.ts +1 -1
  152. package/src/activities/run-input.ts +128 -136
  153. package/src/activities/sandbox-images.ts +7 -166
  154. package/src/activities/sandbox-lease.ts +406 -15
  155. package/src/activities/scheduled-tasks.ts +2103 -411
  156. package/src/activities/session-state.ts +10 -2
  157. package/src/activities/types.ts +32 -2
  158. package/src/activities/video-media-validation.ts +3 -3
  159. package/src/activities/video-output-retention.ts +2 -57
  160. package/src/activities/xai-auth.ts +8 -0
  161. package/src/activities-control.ts +2 -0
  162. package/src/activities.ts +3 -0
  163. package/src/activity-services.ts +13 -4
  164. package/src/connection-provider-use.ts +36 -0
  165. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  166. package/src/editable-artifact-materializer.ts +29 -18
  167. package/src/github-rest-mcp.ts +713 -0
  168. package/src/http.ts +16 -6
  169. package/src/index.ts +40 -5
  170. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  171. package/src/model-context-contributions.ts +33 -10
  172. package/src/observability-metrics.ts +357 -0
  173. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  174. package/src/personal-github-git-credentials.ts +331 -0
  175. package/src/pr-review-credentials.ts +163 -0
  176. package/src/sandbox-archive-storage.ts +159 -0
  177. package/src/sandbox-resume.ts +449 -63
  178. package/src/sandbox-routing.ts +314 -16
  179. package/src/scheduled-alert-occurrence.ts +176 -0
  180. package/src/workflows/activities.ts +18 -2
  181. package/src/workflows/automations.ts +11 -0
  182. package/src/workflows/session.ts +13 -3
  183. package/src/workflows.ts +1 -0
  184. package/dist/activities-control-SMKF2OBE.js.map +0 -1
  185. package/dist/activities-turn-IDS3WEHA.js +0 -16532
  186. package/dist/activities-turn-IDS3WEHA.js.map +0 -1
  187. package/dist/chunk-N5TJIFCE.js.map +0 -1
  188. package/dist/chunk-P6Z2SLAA.js +0 -3092
  189. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -2,11 +2,228 @@
2
2
  import { createHash } from "crypto";
3
3
  import { errorCodeToJSON } from "@opengeni/agent-proto";
4
4
  import { SandboxBackend } from "@opengeni/contracts";
5
+
6
+ // src/opensandbox-kubernetes-inventory.ts
7
+ import { readFile } from "fs/promises";
8
+ import { request as httpsRequest } from "https";
9
+ var KUBERNETES_API_URL = "https://kubernetes.default.svc";
10
+ var KUBERNETES_TOKEN_FILE = "/var/run/secrets/opengeni.io/opensandbox-kubernetes-inventory/token";
11
+ var KUBERNETES_CA_FILE = "/var/run/secrets/opengeni.io/opensandbox-kubernetes-inventory/ca.crt";
12
+ var KUBERNETES_REQUEST_TIMEOUT_MS = 1e4;
13
+ var KUBERNETES_RESPONSE_MAX_BYTES = 64 * 1024 * 1024;
14
+ var KUBERNETES_LIST_PAGE_SIZE = 200;
15
+ var KUBERNETES_LIST_MAX_PAGES = 1e4;
16
+ var OPENSANDBOX_CLEANUP_STUCK_SECONDS = 300;
17
+ var OPENSANDBOX_EXPIRATION_OVERDUE_SECONDS = 180;
18
+ var OPENSANDBOX_BATCHSANDBOX_PHASES = [
19
+ "pending",
20
+ "running",
21
+ "pausing",
22
+ "paused",
23
+ "resuming",
24
+ "failed",
25
+ "unknown"
26
+ ];
27
+ var OPENSANDBOX_WORKLOAD_POD_CONDITIONS = [
28
+ "pending",
29
+ "image_pull",
30
+ "unschedulable"
31
+ ];
32
+ var IMAGE_PULL_REASONS = /* @__PURE__ */ new Set(["ErrImagePull", "ImagePullBackOff", "InvalidImageName"]);
33
+ async function inspectOpenSandboxKubernetesInventory(input) {
34
+ const namespace = kubernetesNamespace(input.namespace);
35
+ const nowMs = input.nowMs ?? Date.now();
36
+ const requestJson = input.requestJson ?? await authenticatedKubernetesJsonRequest({
37
+ tokenFile: input.tokenFile ?? KUBERNETES_TOKEN_FILE,
38
+ caFile: input.caFile ?? KUBERNETES_CA_FILE
39
+ });
40
+ const apiUrl = new URL(input.apiUrl ?? KUBERNETES_API_URL);
41
+ if (apiUrl.protocol !== "https:") {
42
+ throw new Error("OpenSandbox Kubernetes inventory API must use HTTPS");
43
+ }
44
+ const inventory = {
45
+ batchSandboxPhases: zeroRecord(OPENSANDBOX_BATCHSANDBOX_PHASES),
46
+ workloadPodConditions: zeroRecord(OPENSANDBOX_WORKLOAD_POD_CONDITIONS),
47
+ cleanupStuck: 0,
48
+ expirationOverdue: 0
49
+ };
50
+ await visitKubernetesList({
51
+ apiUrl,
52
+ requestJson,
53
+ path: `/apis/sandbox.opensandbox.io/v1alpha1/namespaces/${encodeURIComponent(namespace)}/batchsandboxes`,
54
+ visit: (item) => observeBatchSandbox(inventory, item, nowMs)
55
+ });
56
+ await visitKubernetesList({
57
+ apiUrl,
58
+ requestJson,
59
+ path: `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods`,
60
+ query: { labelSelector: "opensandbox.io/id" },
61
+ visit: (item) => observeWorkloadPod(inventory, item)
62
+ });
63
+ return inventory;
64
+ }
65
+ function observeBatchSandbox(inventory, item, nowMs) {
66
+ const resource = record(item);
67
+ const metadata = record(resource.metadata);
68
+ const spec = record(resource.spec);
69
+ const status = record(resource.status);
70
+ inventory.batchSandboxPhases[batchSandboxPhase(status.phase)] += 1;
71
+ const deletionTimestampMs = dateMilliseconds(metadata.deletionTimestamp);
72
+ const finalizers = Array.isArray(metadata.finalizers) ? metadata.finalizers : [];
73
+ if (deletionTimestampMs !== null && finalizers.length > 0 && nowMs - deletionTimestampMs >= OPENSANDBOX_CLEANUP_STUCK_SECONDS * 1e3) {
74
+ inventory.cleanupStuck += 1;
75
+ }
76
+ const expirationMs = dateMilliseconds(spec.expireTime);
77
+ if (expirationMs !== null && nowMs - expirationMs >= OPENSANDBOX_EXPIRATION_OVERDUE_SECONDS * 1e3) {
78
+ inventory.expirationOverdue += 1;
79
+ }
80
+ }
81
+ function observeWorkloadPod(inventory, item) {
82
+ const pod = record(item);
83
+ const metadata = record(pod.metadata);
84
+ if (dateMilliseconds(metadata.deletionTimestamp) !== null) return;
85
+ const status = record(pod.status);
86
+ if (status.phase === "Pending") inventory.workloadPodConditions.pending += 1;
87
+ if (podHasImagePullFailure(status)) inventory.workloadPodConditions.image_pull += 1;
88
+ if (podIsUnschedulable(status)) inventory.workloadPodConditions.unschedulable += 1;
89
+ }
90
+ function podHasImagePullFailure(status) {
91
+ for (const key of ["initContainerStatuses", "containerStatuses", "ephemeralContainerStatuses"]) {
92
+ const statuses = Array.isArray(status[key]) ? status[key] : [];
93
+ for (const value of statuses) {
94
+ const waiting = record(record(record(value).state).waiting);
95
+ if (typeof waiting.reason === "string" && IMAGE_PULL_REASONS.has(waiting.reason)) return true;
96
+ }
97
+ }
98
+ return false;
99
+ }
100
+ function podIsUnschedulable(status) {
101
+ const conditions = Array.isArray(status.conditions) ? status.conditions : [];
102
+ return conditions.some((value) => {
103
+ const condition = record(value);
104
+ return condition.type === "PodScheduled" && condition.status === "False" && condition.reason === "Unschedulable";
105
+ });
106
+ }
107
+ function batchSandboxPhase(value) {
108
+ switch (value) {
109
+ case "Pending":
110
+ return "pending";
111
+ case "Succeed":
112
+ case "Running":
113
+ return "running";
114
+ case "Pausing":
115
+ return "pausing";
116
+ case "Paused":
117
+ return "paused";
118
+ case "Resuming":
119
+ return "resuming";
120
+ case "Failed":
121
+ return "failed";
122
+ default:
123
+ return "unknown";
124
+ }
125
+ }
126
+ async function visitKubernetesList(input) {
127
+ let continuation = "";
128
+ for (let page = 0; page < KUBERNETES_LIST_MAX_PAGES; page += 1) {
129
+ const url = new URL(input.path, input.apiUrl);
130
+ url.searchParams.set("limit", String(KUBERNETES_LIST_PAGE_SIZE));
131
+ for (const [key, value] of Object.entries(input.query ?? {})) {
132
+ url.searchParams.set(key, value);
133
+ }
134
+ if (continuation) url.searchParams.set("continue", continuation);
135
+ const payload = record(await input.requestJson(url));
136
+ if (!Array.isArray(payload.items)) {
137
+ throw new Error(`Kubernetes list ${input.path} returned no items array`);
138
+ }
139
+ for (const item of payload.items) input.visit(item);
140
+ const next = record(payload.metadata).continue;
141
+ if (typeof next !== "string" || next.length === 0) return;
142
+ if (next === continuation) {
143
+ throw new Error(`Kubernetes list ${input.path} repeated its continuation token`);
144
+ }
145
+ continuation = next;
146
+ }
147
+ throw new Error(`Kubernetes list ${input.path} exceeded its page safety bound`);
148
+ }
149
+ async function authenticatedKubernetesJsonRequest(input) {
150
+ const [tokenText, ca] = await Promise.all([
151
+ readFile(input.tokenFile, "utf8"),
152
+ readFile(input.caFile)
153
+ ]);
154
+ const token = tokenText.trim();
155
+ if (!token) throw new Error("OpenSandbox Kubernetes inventory token is empty");
156
+ return async (url) => await new Promise((resolve, reject) => {
157
+ const request = httpsRequest(
158
+ url,
159
+ {
160
+ method: "GET",
161
+ ca,
162
+ headers: {
163
+ accept: "application/json",
164
+ authorization: `Bearer ${token}`
165
+ }
166
+ },
167
+ (response) => {
168
+ const chunks = [];
169
+ let bytes = 0;
170
+ response.on("data", (chunk) => {
171
+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
172
+ bytes += buffer.length;
173
+ if (bytes > KUBERNETES_RESPONSE_MAX_BYTES) {
174
+ request.destroy(new Error("Kubernetes inventory response exceeded its byte limit"));
175
+ return;
176
+ }
177
+ chunks.push(buffer);
178
+ });
179
+ response.on("end", () => {
180
+ const statusCode = response.statusCode ?? 0;
181
+ if (statusCode < 200 || statusCode >= 300) {
182
+ reject(new Error(`Kubernetes inventory request failed with HTTP ${statusCode}`));
183
+ return;
184
+ }
185
+ try {
186
+ resolve(JSON.parse(Buffer.concat(chunks).toString("utf8")));
187
+ } catch {
188
+ reject(new Error("Kubernetes inventory response was not valid JSON"));
189
+ }
190
+ });
191
+ response.on("error", reject);
192
+ }
193
+ );
194
+ request.setTimeout(KUBERNETES_REQUEST_TIMEOUT_MS, () => {
195
+ request.destroy(new Error("Kubernetes inventory request timed out"));
196
+ });
197
+ request.on("error", reject);
198
+ request.end();
199
+ });
200
+ }
201
+ function kubernetesNamespace(value) {
202
+ const namespace = value.trim();
203
+ if (namespace.length === 0 || namespace.length > 63 || !/^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/u.test(namespace)) {
204
+ throw new Error("OpenSandbox Kubernetes inventory namespace is invalid");
205
+ }
206
+ return namespace;
207
+ }
208
+ function dateMilliseconds(value) {
209
+ if (typeof value !== "string" || value.length === 0) return null;
210
+ const milliseconds = Date.parse(value);
211
+ return Number.isFinite(milliseconds) ? milliseconds : null;
212
+ }
213
+ function record(value) {
214
+ return value && typeof value === "object" && !Array.isArray(value) ? value : {};
215
+ }
216
+ function zeroRecord(keys) {
217
+ return Object.fromEntries(keys.map((key) => [key, 0]));
218
+ }
219
+
220
+ // src/observability-metrics.ts
5
221
  import {
6
222
  SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
7
223
  modelUsageTokenCountOrNull
8
224
  } from "@opengeni/runtime";
9
225
  var turnTrackers = /* @__PURE__ */ new WeakMap();
226
+ var modelRequestTrackers = /* @__PURE__ */ new WeakMap();
10
227
  var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
11
228
  var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
12
229
  var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
@@ -71,6 +288,47 @@ function runtimeMetricsHooksForObservability(observability) {
71
288
  labels: { backend, stage }
72
289
  });
73
290
  },
291
+ onSandboxProviderApiThrottle: ({ backend, operation }) => {
292
+ observability.incrementCounter({
293
+ name: "opengeni_sandbox_provider_api_throttles_total",
294
+ help: "Total sandbox provider API throttle responses.",
295
+ labels: { backend, operation }
296
+ });
297
+ },
298
+ onSandboxTtlRenewal: ({ backend, outcome }) => {
299
+ observability.incrementCounter({
300
+ name: "opengeni_sandbox_ttl_renewals_total",
301
+ help: "Total renewable sandbox provider TTL refresh attempts.",
302
+ labels: { backend, outcome }
303
+ });
304
+ },
305
+ onOpenSandboxSignedEndpoint: ({ outcome, port }) => {
306
+ observability.incrementCounter({
307
+ name: "opengeni_opensandbox_signed_endpoint_total",
308
+ help: "OpenSandbox signed Channel B mint and host-fetch outcomes.",
309
+ labels: { outcome, port: String(port) }
310
+ });
311
+ },
312
+ onWorkspaceArchiveObject: ({ outcome, backend }) => {
313
+ observability.incrementCounter({
314
+ name: "opengeni_workspace_archive_object_total",
315
+ help: "Workspace archive object-storage put/delete outcomes.",
316
+ labels: { outcome, backend }
317
+ });
318
+ },
319
+ onMcpToolCall: ({ outcome, durationSeconds }) => {
320
+ observability.incrementCounter({
321
+ name: "opengeni_mcp_tool_calls_total",
322
+ help: "Total physical MCP tool calls by bounded structural outcome.",
323
+ labels: { outcome }
324
+ });
325
+ observability.observeHistogram({
326
+ name: "opengeni_mcp_tool_call_duration_seconds",
327
+ help: "MCP tool-call duration in seconds by bounded structural outcome.",
328
+ labels: { outcome },
329
+ value: durationSeconds
330
+ });
331
+ },
74
332
  onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
75
333
  observability.incrementCounter({
76
334
  name: "opengeni_machine_op_total",
@@ -263,6 +521,89 @@ var TurnLifecycleMetrics = class {
263
521
  return this.options.now?.() ?? Date.now();
264
522
  }
265
523
  };
524
+ function modelRequestLifecycleMetricsFor(observability) {
525
+ const existing = modelRequestTrackers.get(observability);
526
+ if (existing) return existing;
527
+ const tracker = new ModelRequestLifecycleMetrics(observability);
528
+ modelRequestTrackers.set(observability, tracker);
529
+ return tracker;
530
+ }
531
+ var ModelRequestLifecycleMetrics = class {
532
+ constructor(observability, options = {}) {
533
+ this.observability = observability;
534
+ this.options = options;
535
+ }
536
+ requests = /* @__PURE__ */ new Map();
537
+ providers = /* @__PURE__ */ new Set();
538
+ timer = null;
539
+ start(key, provider) {
540
+ const now = this.now();
541
+ this.providers.add(provider);
542
+ this.requests.set(key, { provider, startedAt: now, lastEventAt: now });
543
+ this.ensureTimer();
544
+ this.refreshGauges();
545
+ }
546
+ event(key, interEventGapMs) {
547
+ const request = this.requests.get(key);
548
+ if (!request) return;
549
+ request.lastEventAt = this.now();
550
+ this.observability.incrementCounter({
551
+ name: "opengeni_model_request_stream_events_total",
552
+ help: "Complete, valid provider SSE data events by provider.",
553
+ labels: { provider: request.provider }
554
+ });
555
+ if (interEventGapMs !== void 0) {
556
+ this.observability.observeHistogram({
557
+ name: "opengeni_model_request_stream_event_gap_seconds",
558
+ help: "Seconds between complete, valid provider SSE data events.",
559
+ buckets: MODEL_REQUEST_PHASE_BUCKETS,
560
+ labels: { provider: request.provider },
561
+ value: Math.max(0, interEventGapMs / 1e3)
562
+ });
563
+ }
564
+ }
565
+ finish(key) {
566
+ this.requests.delete(key);
567
+ this.refreshGauges();
568
+ if (this.requests.size === 0) this.stopTimer();
569
+ }
570
+ refreshGauges() {
571
+ const now = this.now();
572
+ for (const provider of this.providers) {
573
+ const active = [...this.requests.values()].filter((request) => request.provider === provider);
574
+ this.observability.setGauge({
575
+ name: "opengeni_model_requests_inflight",
576
+ help: "Current in-flight provider model requests in this worker process.",
577
+ labels: { provider },
578
+ value: active.length
579
+ });
580
+ this.observability.setGauge({
581
+ name: "opengeni_model_request_oldest_no_event_age_seconds",
582
+ help: "Seconds since the least recently progressing in-flight provider request received a valid SSE event.",
583
+ labels: { provider },
584
+ value: active.length === 0 ? 0 : Math.max(0, (now - Math.min(...active.map((request) => request.lastEventAt))) / 1e3)
585
+ });
586
+ }
587
+ }
588
+ stop() {
589
+ this.requests.clear();
590
+ this.refreshGauges();
591
+ this.stopTimer();
592
+ }
593
+ ensureTimer() {
594
+ if (this.timer) return;
595
+ this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15e3);
596
+ this.timer.unref?.();
597
+ }
598
+ stopTimer() {
599
+ if (!this.timer) return;
600
+ clearInterval(this.timer);
601
+ this.timer = null;
602
+ }
603
+ now() {
604
+ return this.options.now?.() ?? Date.now();
605
+ }
606
+ };
266
607
  function recordTurnsQueuedGauge(observability, value) {
267
608
  observability.setGauge({
268
609
  name: "opengeni_turns_queued",
@@ -399,6 +740,34 @@ function recordSandboxLeaseGauges(observability, counts) {
399
740
  });
400
741
  }
401
742
  }
743
+ function recordOpenSandboxKubernetesInventoryGauges(observability, inventory) {
744
+ for (const phase of OPENSANDBOX_BATCHSANDBOX_PHASES) {
745
+ observability.setGauge({
746
+ name: "opengeni_opensandbox_batchsandboxes",
747
+ help: "Current OpenSandbox BatchSandboxes by bounded lifecycle phase.",
748
+ labels: { phase },
749
+ value: inventory.batchSandboxPhases[phase]
750
+ });
751
+ }
752
+ for (const condition of OPENSANDBOX_WORKLOAD_POD_CONDITIONS) {
753
+ observability.setGauge({
754
+ name: "opengeni_opensandbox_workload_pods",
755
+ help: "Current OpenSandbox workload Pods by bounded operational condition.",
756
+ labels: { condition },
757
+ value: inventory.workloadPodConditions[condition]
758
+ });
759
+ }
760
+ observability.setGauge({
761
+ name: "opengeni_opensandbox_cleanup_stuck",
762
+ help: "Current OpenSandbox BatchSandboxes deleting past the cleanup threshold with finalizers.",
763
+ value: inventory.cleanupStuck
764
+ });
765
+ observability.setGauge({
766
+ name: "opengeni_opensandbox_expiration_overdue",
767
+ help: "Current OpenSandbox BatchSandboxes still present past the provider-expiry threshold.",
768
+ value: inventory.expirationOverdue
769
+ });
770
+ }
402
771
  function recordSandboxInventoryProjectionSuccess(observability, domain, timestampSeconds = Date.now() / 1e3) {
403
772
  observability.setGauge({
404
773
  name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
@@ -635,6 +1004,14 @@ var TURN_STARTUP_PHASE_BUCKETS = [
635
1004
  60,
636
1005
  120
637
1006
  ];
1007
+ var TURN_STARTUP_MILESTONE_BUCKETS = [
1008
+ ...TURN_STARTUP_PHASE_BUCKETS,
1009
+ 300,
1010
+ 600,
1011
+ 1800,
1012
+ 3600,
1013
+ 7200
1014
+ ];
638
1015
  function turnStartupCountBucket(count) {
639
1016
  if (count === null || !Number.isFinite(count) || count < 0) return "unknown";
640
1017
  if (count === 0) return "0";
@@ -670,6 +1047,76 @@ function recordTurnSandboxEstablishPolicy(observability, input) {
670
1047
  }
671
1048
  });
672
1049
  }
1050
+ function recordSandboxProvisionAttempt(observability, input) {
1051
+ const labels = {
1052
+ backend: input.backend,
1053
+ stage: input.stage,
1054
+ category: input.category,
1055
+ outcome: input.outcome
1056
+ };
1057
+ try {
1058
+ observability.incrementCounter({
1059
+ name: "opengeni_sandbox_provision_attempts_total",
1060
+ help: "Internal physical attempts within logical sandbox provisions.",
1061
+ labels
1062
+ });
1063
+ observability.observeHistogram({
1064
+ name: "opengeni_sandbox_provision_attempt_duration_seconds",
1065
+ help: "Internal sandbox provision attempt duration by bounded structural outcome.",
1066
+ buckets: TURN_STARTUP_PHASE_BUCKETS,
1067
+ labels,
1068
+ value: Math.max(0, input.durationSeconds)
1069
+ });
1070
+ } catch {
1071
+ try {
1072
+ observability.incrementCounter({
1073
+ name: "opengeni_observability_observer_errors_total",
1074
+ help: "Observability observer failures isolated from product execution.",
1075
+ labels: { observer: "sandbox_provision" }
1076
+ });
1077
+ } catch {
1078
+ }
1079
+ }
1080
+ }
1081
+ function recordSandboxLogicalProvision(observability, input) {
1082
+ const labels = {
1083
+ backend: input.backend,
1084
+ stage: input.stage,
1085
+ category: input.category,
1086
+ outcome: input.outcome,
1087
+ expected: input.expected ? "true" : "false"
1088
+ };
1089
+ try {
1090
+ observability.incrementCounter({
1091
+ name: "opengeni_sandbox_provisions_total",
1092
+ help: "Logical sandbox provisions by terminal outcome and bounded failure taxonomy.",
1093
+ labels
1094
+ });
1095
+ observability.observeHistogram({
1096
+ name: "opengeni_sandbox_provision_duration_seconds",
1097
+ help: "Logical sandbox provision duration including internal safe retries.",
1098
+ buckets: TURN_STARTUP_PHASE_BUCKETS,
1099
+ labels,
1100
+ value: Math.max(0, input.durationSeconds)
1101
+ });
1102
+ observability.observeHistogram({
1103
+ name: "opengeni_sandbox_provision_internal_attempts",
1104
+ help: "Internal physical attempts per terminal logical sandbox provision.",
1105
+ buckets: [1, 2, 3, 4, 5, 8],
1106
+ labels,
1107
+ value: Math.max(1, input.internalAttempts)
1108
+ });
1109
+ } catch {
1110
+ try {
1111
+ observability.incrementCounter({
1112
+ name: "opengeni_observability_observer_errors_total",
1113
+ help: "Observability observer failures isolated from product execution.",
1114
+ labels: { observer: "sandbox_provision" }
1115
+ });
1116
+ } catch {
1117
+ }
1118
+ }
1119
+ }
673
1120
  function recordTurnWorkerPreparationTotal(observability, input) {
674
1121
  observability.observeHistogram({
675
1122
  name: "opengeni_turn_worker_preparation_duration_seconds",
@@ -683,6 +1130,20 @@ function recordTurnWorkerPreparationTotal(observability, input) {
683
1130
  value: Math.max(0, input.durationSeconds)
684
1131
  });
685
1132
  }
1133
+ function recordTurnStartupMilestone(observability, input) {
1134
+ observability.observeHistogram({
1135
+ name: "opengeni_turn_startup_milestone_duration_seconds",
1136
+ help: "Cumulative seconds from durable turn queueing to a bounded startup milestone.",
1137
+ buckets: TURN_STARTUP_MILESTONE_BUCKETS,
1138
+ labels: {
1139
+ milestone: input.milestone,
1140
+ provider: input.provider,
1141
+ backend: input.backend,
1142
+ outcome: input.outcome
1143
+ },
1144
+ value: Math.max(0, input.durationSeconds)
1145
+ });
1146
+ }
686
1147
  function contentDeltaClass(type) {
687
1148
  if (type === "agent.message.delta") {
688
1149
  return "message";
@@ -937,14 +1398,17 @@ function modelCallAccountContext(input) {
937
1398
  }
938
1399
 
939
1400
  export {
1401
+ inspectOpenSandboxKubernetesInventory,
940
1402
  observabilityEventLogger,
941
1403
  runtimeMetricsHooksForObservability,
942
1404
  makeMachineOpObserver,
943
1405
  turnLifecycleMetricsFor,
1406
+ modelRequestLifecycleMetricsFor,
944
1407
  recordTurnsQueuedGauge,
945
1408
  normalizeTurnTaskQueueStats,
946
1409
  startTurnCapacityMonitor,
947
1410
  recordSandboxLeaseGauges,
1411
+ recordOpenSandboxKubernetesInventoryGauges,
948
1412
  recordSandboxInventoryProjectionSuccess,
949
1413
  recordSandboxInventoryProjectionFailure,
950
1414
  recordCreditBalanceGauges,
@@ -960,7 +1424,10 @@ export {
960
1424
  recordModelRequestPhase,
961
1425
  recordTurnStartupPhase,
962
1426
  recordTurnSandboxEstablishPolicy,
1427
+ recordSandboxProvisionAttempt,
1428
+ recordSandboxLogicalProvision,
963
1429
  recordTurnWorkerPreparationTotal,
1430
+ recordTurnStartupMilestone,
964
1431
  StreamTimingMetrics,
965
1432
  recordBatchFlush,
966
1433
  recordSessionEventAppendLatency,
@@ -971,4 +1438,4 @@ export {
971
1438
  recordModelCacheTokens,
972
1439
  modelCallAccountContext
973
1440
  };
974
- //# sourceMappingURL=chunk-N5TJIFCE.js.map
1441
+ //# sourceMappingURL=chunk-CE7TPKBS.js.map