@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +185 -26
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +4 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
  123. package/dist/activities-control-5IIVRXLA.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
  126. package/dist/activities-turn-FLYJOM54.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +107 -36
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/chunk-MVESOYFC.js +3746 -0
  140. package/dist/chunk-MVESOYFC.js.map +1 -0
  141. package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
  142. package/dist/chunk-N57B3PJ5.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1152 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +42 -12
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +114 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +634 -111
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +182 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +124 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +780 -147
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +85 -39
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
@@ -1,17 +1,43 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { errorCodeToJSON } from "@opengeni/agent-proto";
3
- import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
4
- import type { SessionEventAppendPhaseObservation } from "@opengeni/db";
5
- import type { EventLogger } from "@opengeni/events";
3
+ import {
4
+ BundledSkillId,
5
+ CREDIT_GRANT_CLASSES,
6
+ SandboxBackend,
7
+ type SessionEventType,
8
+ type SkillReadKind,
9
+ type SkillUseSource,
10
+ } from "@opengeni/contracts";
11
+ import {
12
+ ACTIVE_USER_WINDOWS,
13
+ type ActiveUserWindow,
14
+ type CreditGrantTotals,
15
+ type IdleInteractionInspection,
16
+ type SessionEventAppendPhaseObservation,
17
+ } from "@opengeni/db";
18
+ import {
19
+ natsSubscriptionTerminationCounter,
20
+ type EventBusOptions,
21
+ type EventLogger,
22
+ } from "@opengeni/events";
6
23
  import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
7
24
  import type { CompanyBrainContributionReceipt } from "./model-context-contributions";
25
+ import {
26
+ createTelemetryClock,
27
+ telemetryInterval,
28
+ telemetryScheduler,
29
+ type TelemetryScheduler,
30
+ } from "./telemetry-scheduler";
8
31
  import {
9
32
  OPENSANDBOX_BATCHSANDBOX_PHASES,
10
33
  OPENSANDBOX_WORKLOAD_POD_CONDITIONS,
11
34
  type OpenSandboxKubernetesInventory,
12
35
  } from "./opensandbox-kubernetes-inventory";
13
36
  import {
37
+ MCP_TOOL_METRIC_EXTERNAL_LABEL,
14
38
  SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
39
+ WORKSPACE_CAPTURE_SKIP_REASONS,
40
+ isMcpToolMetricLabel,
15
41
  modelUsageTokenCountOrNull,
16
42
  type RuntimeMetricsHooks,
17
43
  type SelfhostedOpObservation,
@@ -31,9 +57,23 @@ export type TurnTaskQueueStats = {
31
57
  tasksAddRate: number;
32
58
  tasksDispatchRate: number;
33
59
  };
60
+ export type TurnTaskQueueIdentity = { temporalNamespace: string; taskQueue: string };
61
+ export type TurnTaskQueueReadOptions = { signal: AbortSignal; deadline: number };
62
+
63
+ export function turnTaskQueueMetricLabels(identity: TurnTaskQueueIdentity) {
64
+ return { temporal_namespace: identity.temporalNamespace, task_queue: identity.taskQueue };
65
+ }
66
+ export type SessionRecoveryBacklogStateSummary = {
67
+ /** Every durable recovering session in this state with no active attempt. */
68
+ count: number;
69
+ /** Of those, sessions still inside the backoff their recovery request recorded. */
70
+ scheduled: number;
71
+ /** Seconds the single oldest candidate has been past its recovery due time; 0 when none is. */
72
+ oldestOverdueSeconds: number;
73
+ };
34
74
  export type SessionRecoveryBacklog = {
35
- quiescence_missing: number;
36
- projection_stale: number;
75
+ quiescence_missing: SessionRecoveryBacklogStateSummary;
76
+ projection_stale: SessionRecoveryBacklogStateSummary;
37
77
  };
38
78
  export type ContextCompactionPendingSummary = {
39
79
  pendingCount: number;
@@ -80,6 +120,16 @@ const CONTEXT_COMPACTIONS_METRIC = {
80
120
  help: "Total completed context compactions, by trigger.",
81
121
  } as const;
82
122
 
123
+ /** Logger plus the closed-label subscription-termination counter for NATS connections. */
124
+ export function observabilityEventBusOptions(
125
+ observability: Observability,
126
+ ): Pick<EventBusOptions, "logger" | "onSubscriptionTerminated"> {
127
+ return {
128
+ logger: observabilityEventLogger(observability),
129
+ onSubscriptionTerminated: natsSubscriptionTerminationCounter(observability),
130
+ };
131
+ }
132
+
83
133
  export function observabilityEventLogger(observability: Observability): EventLogger {
84
134
  return {
85
135
  debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
@@ -156,6 +206,13 @@ export function runtimeMetricsHooksForObservability(
156
206
  labels: { backend, stage },
157
207
  });
158
208
  },
209
+ onSandboxReadinessReplacement: ({ backend, outcome }) => {
210
+ observability.incrementCounter({
211
+ name: "opengeni_sandbox_readiness_replacements_total",
212
+ help: "Fresh sandbox command-readiness replacement decisions by backend and outcome.",
213
+ labels: { backend, outcome },
214
+ });
215
+ },
159
216
  onSandboxProviderApiThrottle: ({ backend, operation }) => {
160
217
  observability.incrementCounter({
161
218
  name: "opengeni_sandbox_provider_api_throttles_total",
@@ -191,6 +248,17 @@ export function runtimeMetricsHooksForObservability(
191
248
  value: durationSeconds,
192
249
  });
193
250
  },
251
+ onWorkspaceCaptureSkipped: ({ backend, reason }) => {
252
+ const safeBackend = SandboxBackend.safeParse(backend).success ? backend : "unknown";
253
+ const safeReason = (WORKSPACE_CAPTURE_SKIP_REASONS as readonly string[]).includes(reason)
254
+ ? reason
255
+ : "unknown";
256
+ observability.incrementCounter({
257
+ name: "opengeni_workspace_capture_skipped_total",
258
+ help: "Due warm workspace checkpoints that could not start, by backend and fixed reason. Not-yet-due (throttled) and already-complete checkpoints are excluded.",
259
+ labels: { backend: safeBackend, reason: safeReason },
260
+ });
261
+ },
194
262
  onWorkspaceArchiveObject: ({ outcome, backend }) => {
195
263
  observability.incrementCounter({
196
264
  name: "opengeni_workspace_archive_object_total",
@@ -198,17 +266,23 @@ export function runtimeMetricsHooksForObservability(
198
266
  labels: { outcome, backend },
199
267
  });
200
268
  },
201
- onMcpToolCall: ({ outcome, durationSeconds }) => {
202
- completedOperationSpan(observability, "worker.mcp.tool_call", durationSeconds, { outcome });
269
+ onMcpToolCall: ({ outcome, tool, durationSeconds }) => {
270
+ // Re-check the closed label set at the export boundary so a caller can
271
+ // never turn a raw user-defined tool name into a metric series.
272
+ const safeTool = isMcpToolMetricLabel(tool) ? tool : MCP_TOOL_METRIC_EXTERNAL_LABEL;
273
+ completedOperationSpan(observability, "worker.mcp.tool_call", durationSeconds, {
274
+ outcome,
275
+ tool: safeTool,
276
+ });
203
277
  observability.incrementCounter({
204
278
  name: "opengeni_mcp_tool_calls_total",
205
- help: "Total physical MCP tool calls by bounded structural outcome.",
206
- labels: { outcome },
279
+ help: "Total physical MCP tool calls by bounded structural outcome and first-party tool (other tools are 'external').",
280
+ labels: { outcome, tool: safeTool },
207
281
  });
208
282
  observability.observeHistogram({
209
283
  name: "opengeni_mcp_tool_call_duration_seconds",
210
- help: "MCP tool-call duration in seconds by bounded structural outcome.",
211
- labels: { outcome },
284
+ help: "MCP tool-call duration in seconds by bounded structural outcome and first-party tool (other tools are 'external').",
285
+ labels: { outcome, tool: safeTool },
212
286
  value: durationSeconds,
213
287
  });
214
288
  },
@@ -667,7 +741,7 @@ export function recordTurnsQueuedGauge(observability: Observability, value: numb
667
741
  }
668
742
 
669
743
  /**
670
- * Record the authoritative turn activity queue rather than aggregate Postgres
744
+ * Record the approximate shared turn-worker activity queue rather than Postgres
671
745
  * prompts. A paused human prompt is durable queue truth but is not runnable and
672
746
  * never reaches this Temporal task queue. DescribeTaskQueue's approximate
673
747
  * backlog count and age are explicitly documented by Temporal as autoscaling
@@ -676,26 +750,40 @@ export function recordTurnsQueuedGauge(observability: Observability, value: numb
676
750
  export function recordTurnTaskQueueStats(
677
751
  observability: Observability,
678
752
  stats: TurnTaskQueueStats,
753
+ identity: TurnTaskQueueIdentity,
679
754
  ): void {
755
+ const labels = turnTaskQueueMetricLabels(identity);
756
+ // Invalid observations are failures, not a fabricated fresh zero. Validate
757
+ // every field before publishing any portion of a new observation.
758
+ const values = {
759
+ backlog: requiredTemporalNumber(stats.eligibleBacklog, "eligibleBacklog", { integer: true }),
760
+ age: requiredTemporalNumber(stats.oldestBacklogAgeSeconds, "oldestBacklogAgeSeconds"),
761
+ add: requiredTemporalNumber(stats.tasksAddRate, "tasksAddRate"),
762
+ dispatch: requiredTemporalNumber(stats.tasksDispatchRate, "tasksDispatchRate"),
763
+ };
680
764
  observability.setGauge({
681
765
  name: "opengeni_turn_eligible_backlog",
682
- help: "Temporal runAgentTurn activity tasks eligible for immediate worker admission.",
683
- value: nonnegativeFinite(stats.eligibleBacklog),
766
+ help: "Approximate Temporal activity backlog on the shared turn-worker queue, including video and retries.",
767
+ labels,
768
+ value: values.backlog,
684
769
  });
685
770
  observability.setGauge({
686
771
  name: "opengeni_turn_eligible_backlog_oldest_age_seconds",
687
- help: "Approximate age of the oldest eligible runAgentTurn activity task.",
688
- value: nonnegativeFinite(stats.oldestBacklogAgeSeconds),
772
+ help: "Approximate age of the oldest eligible activity task on the shared turn-worker queue.",
773
+ labels,
774
+ value: values.age,
689
775
  });
690
776
  observability.setGauge({
691
777
  name: "opengeni_turn_eligible_tasks_add_rate",
692
- help: "Temporal runAgentTurn tasks added per second over its rolling window.",
693
- value: nonnegativeFinite(stats.tasksAddRate),
778
+ help: "Temporal activities added per second to the shared turn-worker queue over its rolling window.",
779
+ labels,
780
+ value: values.add,
694
781
  });
695
782
  observability.setGauge({
696
783
  name: "opengeni_turn_eligible_tasks_dispatch_rate",
697
- help: "Temporal runAgentTurn tasks dispatched per second over its rolling window.",
698
- value: nonnegativeFinite(stats.tasksDispatchRate),
784
+ help: "Temporal activities dispatched per second from the shared turn-worker queue over its rolling window.",
785
+ labels,
786
+ value: values.dispatch,
699
787
  });
700
788
  }
701
789
 
@@ -705,13 +793,22 @@ export function normalizeTurnTaskQueueStats(
705
793
  if (!stats) {
706
794
  throw new Error("Temporal DescribeTaskQueue response omitted required stats");
707
795
  }
796
+ if (typeof stats !== "object" || Array.isArray(stats)) {
797
+ throw new Error("Temporal DescribeTaskQueue returned malformed stats");
798
+ }
708
799
  const eligibleBacklog = requiredTemporalNumber(
709
800
  stats.approximateBacklogCount,
710
801
  "approximateBacklogCount",
711
802
  { integer: true },
712
803
  );
713
804
  let oldestBacklogAgeSeconds = 0;
714
- if (stats.approximateBacklogAge) {
805
+ if (stats.approximateBacklogAge !== null && stats.approximateBacklogAge !== undefined) {
806
+ if (
807
+ typeof stats.approximateBacklogAge !== "object" ||
808
+ Array.isArray(stats.approximateBacklogAge)
809
+ ) {
810
+ throw new Error("Temporal stats returned invalid approximateBacklogAge");
811
+ }
715
812
  const seconds = optionalTemporalNumber(
716
813
  stats.approximateBacklogAge.seconds,
717
814
  "approximateBacklogAge.seconds",
@@ -737,24 +834,62 @@ export function normalizeTurnTaskQueueStats(
737
834
  };
738
835
  }
739
836
 
837
+ /** Requires a fresh metric registry/schema; pre-registered legacy unlabelled
838
+ * queue gauges are not migrated in place. Each collector owns its native read. */
740
839
  export function startTurnCapacityMonitor(input: {
741
840
  observability: Observability;
742
- read: () => Promise<TurnTaskQueueStats>;
841
+ identity: TurnTaskQueueIdentity;
842
+ read: (options: TurnTaskQueueReadOptions) => Promise<TurnTaskQueueStats>;
743
843
  intervalMs?: number;
844
+ readTimeoutMs?: number;
845
+ closeTimeoutMs?: number;
744
846
  now?: () => number;
847
+ monotonicNow?: () => number;
848
+ scheduler?: TelemetryScheduler;
745
849
  }): { close: () => Promise<void> } {
746
- const intervalMs = input.intervalMs ?? 15_000;
747
- const now = input.now ?? Date.now;
748
- const startedAt = now();
850
+ const intervalMs = telemetryInterval(input.intervalMs ?? 15_000, "queue monitor intervalMs");
851
+ const readTimeoutMs = telemetryInterval(
852
+ input.readTimeoutMs ?? 5_000,
853
+ "queue monitor readTimeoutMs",
854
+ );
855
+ const closeTimeoutMs = telemetryInterval(
856
+ input.closeTimeoutMs ?? 1_000,
857
+ "queue monitor closeTimeoutMs",
858
+ );
859
+ const scheduler = input.scheduler ?? telemetryScheduler;
860
+ const labels = turnTaskQueueMetricLabels(input.identity);
861
+ const wallClock = createTelemetryClock(input.now ?? Date.now);
862
+ const monotonicClock = createTelemetryClock(input.monotonicNow ?? (() => performance.now()));
863
+ let startedAt = wallClock.read();
864
+ const readClock = () => {
865
+ const wall = wallClock.read();
866
+ const monotonic = monotonicClock.read();
867
+ return wall === null || monotonic === null ? null : { wall, monotonic };
868
+ };
749
869
  let lastSuccessAt: number | null = null;
750
870
  let lastReadSucceeded = false;
751
871
  let stopped = false;
752
- let running: Promise<void> | null = null;
753
- const recordStatus = () => {
754
- const observedAt = now();
755
- const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
872
+ let running: {
873
+ controller: AbortController;
874
+ done: Promise<void>;
875
+ cancelTimeout: () => void;
876
+ } | null = null;
877
+ let closePromise: Promise<void> | undefined;
878
+ const safely = (action: () => void) => {
879
+ try {
880
+ action();
881
+ } catch {
882
+ /* Telemetry must not break the worker. */
883
+ }
884
+ };
885
+ const recordStatus = (observation = readClock()) => {
886
+ if (observation === null) lastReadSucceeded = false;
887
+ const observedAt = wallClock.latest();
888
+ startedAt ??= observedAt;
889
+ const successAgeMs =
890
+ observedAt === null ? 0 : observedAt - (lastSuccessAt ?? startedAt ?? observedAt);
756
891
  const set = (name: string, help: string, value: number) =>
757
- input.observability.setGauge({ name, help, value });
892
+ safely(() => input.observability.setGauge({ name, help, labels, value }));
758
893
  set(
759
894
  "opengeni_turn_capacity_monitor_last_read_success",
760
895
  "Whether the latest Temporal turn-queue capacity read completed successfully.",
@@ -772,56 +907,127 @@ export function startTurnCapacityMonitor(input: {
772
907
  );
773
908
  set(
774
909
  "opengeni_turn_capacity_monitor_fresh",
775
- "Whether eligible-backlog gauges have a successful Temporal read within three monitor intervals.",
910
+ "Whether the latest read succeeded and the shared activity-queue gauges are within three monitor intervals.",
776
911
  lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
777
912
  );
778
913
  };
779
914
  const refresh = () => {
780
915
  // Advance freshness age even while a Temporal read is hung. Old backlog
781
916
  // values remain observable for diagnosis but cease to be authoritative.
782
- recordStatus();
783
- if (stopped || running) return;
784
- running = input
785
- .read()
917
+ if (stopped) return;
918
+ const observation = readClock();
919
+ recordStatus(observation);
920
+ if (running || observation === null) return;
921
+ const deadline = observation.wall + readTimeoutMs;
922
+ if (!Number.isFinite(deadline) || deadline > Number.MAX_SAFE_INTEGER) {
923
+ recordStatus(null);
924
+ return;
925
+ }
926
+ const controller = new AbortController();
927
+ const attempt = { controller, done: Promise.resolve(), cancelTimeout: () => {} };
928
+ running = attempt;
929
+ attempt.cancelTimeout = scheduler.timeout(() => {
930
+ if (stopped || running !== attempt) return;
931
+ lastReadSucceeded = false;
932
+ recordStatus();
933
+ controller.abort();
934
+ // Keep the attempt owned until its native call settles. A misbehaving
935
+ // reader ignoring cancellation must not cause overlapping RPCs.
936
+ }, readTimeoutMs);
937
+ let read: Promise<TurnTaskQueueStats>;
938
+ try {
939
+ read = input.read({ signal: controller.signal, deadline });
940
+ } catch (error) {
941
+ read = Promise.reject(error);
942
+ }
943
+ attempt.done = read
786
944
  .then((stats) => {
787
- recordTurnTaskQueueStats(input.observability, stats);
788
- lastSuccessAt = now();
945
+ if (stopped || controller.signal.aborted || running !== attempt) return;
946
+ const completedAt = readClock();
947
+ // A native RPC may settle before its deadline but its Promise callback
948
+ // can run after it, ahead of a delayed JS timeout. Never restamp that old
949
+ // response as fresh. Monotonic age also fences wall-clock adjustments.
950
+ if (
951
+ completedAt === null ||
952
+ completedAt.wall >= deadline ||
953
+ completedAt.monotonic - observation.monotonic >= readTimeoutMs
954
+ ) {
955
+ lastReadSucceeded = false;
956
+ recordStatus(completedAt);
957
+ controller.abort();
958
+ return;
959
+ }
960
+ recordTurnTaskQueueStats(input.observability, stats, input.identity);
961
+ lastSuccessAt = completedAt.wall;
789
962
  lastReadSucceeded = true;
790
- recordStatus();
963
+ recordStatus(completedAt);
791
964
  })
792
- .catch((error) => {
965
+ .catch(() => {
966
+ if (stopped || controller.signal.aborted || running !== attempt) return;
793
967
  lastReadSucceeded = false;
794
968
  recordStatus();
795
- input.observability.warn("turn capacity monitor: Temporal task-queue stats failed", {
796
- error: error instanceof Error ? error.message : String(error),
797
- });
969
+ safely(() =>
970
+ input.observability.warn("turn capacity monitor: Temporal task-queue stats failed", {
971
+ // Do not export arbitrary server errors or credentials.
972
+ errorClass: "WorkerTelemetryReadError",
973
+ origin: "turn-capacity-monitor",
974
+ }),
975
+ );
798
976
  })
799
977
  .finally(() => {
800
- running = null;
978
+ attempt.cancelTimeout();
979
+ if (running === attempt) running = null;
801
980
  });
802
981
  };
803
982
  refresh();
804
- const timer = setInterval(refresh, intervalMs);
805
- timer.unref?.();
983
+ const stopTimer = scheduler.interval(refresh, intervalMs);
806
984
  return {
807
- close: async () => {
985
+ close: () => {
986
+ if (closePromise) return closePromise;
808
987
  stopped = true;
809
- clearInterval(timer);
810
- await running;
988
+ stopTimer();
989
+ lastReadSucceeded = false;
990
+ recordStatus();
991
+ const attempt = running;
992
+ attempt?.cancelTimeout();
993
+ attempt?.controller.abort();
994
+ closePromise = attempt
995
+ ? new Promise<void>((resolve) => {
996
+ const cancel = scheduler.timeout(resolve, closeTimeoutMs);
997
+ void attempt.done.then(() => {
998
+ cancel();
999
+ resolve();
1000
+ });
1001
+ })
1002
+ : Promise.resolve();
1003
+ return closePromise;
811
1004
  },
812
1005
  };
813
1006
  }
814
1007
 
815
1008
  export function recordSessionRecoveryBacklogGauges(
816
1009
  observability: Observability,
817
- counts: SessionRecoveryBacklog,
1010
+ backlog: SessionRecoveryBacklog,
818
1011
  ): void {
819
1012
  for (const state of ["quiescence_missing", "projection_stale"] as const) {
1013
+ const summary = backlog[state];
820
1014
  observability.setGauge({
821
1015
  name: "opengeni_session_recovery_backlog",
822
- help: "Current durable recovering-session obligations with no active attempt, by bounded reconciliation state.",
1016
+ help: "Current durable recovering-session obligations with no active attempt, by bounded reconciliation state. Includes sessions legitimately sleeping in their recorded recovery backoff.",
1017
+ labels: { state },
1018
+ value: nonnegativeFinite(summary.count),
1019
+ });
1020
+ observability.setGauge({
1021
+ name: "opengeni_session_recovery_scheduled",
1022
+ help: "Durable recovering sessions still inside the recovery backoff (provider Retry-After or connectivity pacing) their recovery request recorded, by bounded reconciliation state.",
823
1023
  labels: { state },
824
- value: nonnegativeFinite(counts[state]),
1024
+ value: nonnegativeFinite(summary.scheduled),
1025
+ });
1026
+ observability.setGauge({
1027
+ name: "opengeni_session_recovery_oldest_overdue_seconds",
1028
+ help: "Seconds the single oldest durable recovering session with no active attempt has been past its recovery due time, by bounded reconciliation state; 0 when none is overdue.",
1029
+ labels: { state },
1030
+ value: nonnegativeFinite(summary.oldestOverdueSeconds),
825
1031
  });
826
1032
  }
827
1033
  }
@@ -865,8 +1071,8 @@ export function startSessionRecoveryMonitor(input: {
865
1071
  if (stopped || running) return;
866
1072
  running = input
867
1073
  .read()
868
- .then((counts) => {
869
- recordSessionRecoveryBacklogGauges(input.observability, counts);
1074
+ .then((backlog) => {
1075
+ recordSessionRecoveryBacklogGauges(input.observability, backlog);
870
1076
  lastSuccessAt = now();
871
1077
  lastReadSucceeded = true;
872
1078
  recordStatus();
@@ -1033,10 +1239,14 @@ export function recordOpenSandboxKubernetesInventoryGauges(
1033
1239
  export const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [
1034
1240
  "leases",
1035
1241
  "checkpoint_artifacts",
1242
+ "recovery_observations",
1036
1243
  "rotation_backlog",
1037
1244
  "retained_processes",
1038
1245
  "expired_drains",
1039
1246
  "opensandbox_kubernetes",
1247
+ "modal_provider",
1248
+ "interaction_idle",
1249
+ "checkpoint_staleness",
1040
1250
  ] as const;
1041
1251
 
1042
1252
  export type SandboxInventoryProjectionDomain =
@@ -1094,6 +1304,151 @@ export function recordCreditBalanceGauges(
1094
1304
  creditBalanceGaugeAccounts.set(observability, current);
1095
1305
  }
1096
1306
 
1307
+ /**
1308
+ * Distinct managed people with authenticated browser activity in each fixed
1309
+ * window (migration 0565 presence). Every control worker publishes the same
1310
+ * global value, so dashboards must aggregate with `max()` across pods, never
1311
+ * `sum()`. API keys, services and embedded hosts are never counted.
1312
+ */
1313
+ export function recordActiveUserGauges(
1314
+ observability: Observability,
1315
+ counts: Record<ActiveUserWindow, number>,
1316
+ ): void {
1317
+ for (const window of ACTIVE_USER_WINDOWS) {
1318
+ observability.setGauge({
1319
+ name: "opengeni_active_users",
1320
+ help: "Distinct managed users with authenticated activity within the window (global; use max across pods).",
1321
+ labels: { window },
1322
+ value: counts[window],
1323
+ });
1324
+ }
1325
+ }
1326
+
1327
+ /**
1328
+ * Positive credit grants observed by the ledger trigger since migration 0565,
1329
+ * by closed class. These are cumulative database totals published as gauges by
1330
+ * every control worker: aggregate with `max()` across pods, then subtract the
1331
+ * same expression `offset` by the window. A per-pod `increase()` sees only one
1332
+ * pod's lifetime and undercounts across control-worker restarts.
1333
+ * Counting in the database covers the grants no application process writes:
1334
+ * the verified-signup trial grant (a setup trigger) and operator grants.
1335
+ */
1336
+ export function recordCreditGrantGauges(
1337
+ observability: Observability,
1338
+ totals: CreditGrantTotals,
1339
+ ): void {
1340
+ for (const grantClass of CREDIT_GRANT_CLASSES) {
1341
+ const total = totals[grantClass];
1342
+ observability.setGauge({
1343
+ name: "opengeni_credit_grants_total",
1344
+ help: "Positive credit grants observed in the ledger since migration 0565, by grant class (global cumulative; use max across pods).",
1345
+ labels: { grant_class: grantClass },
1346
+ value: total.count,
1347
+ });
1348
+ observability.setGauge({
1349
+ name: "opengeni_credit_granted_micros_total",
1350
+ help: "Credit micros granted in the ledger since migration 0565, by grant class (global cumulative; use max across pods).",
1351
+ labels: { grant_class: grantClass },
1352
+ value: total.micros,
1353
+ });
1354
+ }
1355
+ }
1356
+
1357
+ /**
1358
+ * The deployment-level runtime switch for the one-time verified signup trial
1359
+ * credit (migration 0521): 1 while it allows grants, 0 when an operator has
1360
+ * disabled it or no revision exists. A grant also needs the API's
1361
+ * OPENGENI_VERIFIED_SIGNUP_TRIAL_CREDITS_ENABLED master opt-in, which
1362
+ * {@link recordVerifiedSignupTrialDeploymentFlagGauge} reports separately.
1363
+ */
1364
+ export function recordVerifiedSignupTrialSwitchGauge(
1365
+ observability: Observability,
1366
+ grantsEnabled: boolean,
1367
+ ): void {
1368
+ observability.setGauge({
1369
+ name: "opengeni_verified_signup_trial_credits_runtime_enabled",
1370
+ help: "Whether the runtime switch allows new verified signup trial credit grants (1) or blocks them (0).",
1371
+ value: grantsEnabled ? 1 : 0,
1372
+ });
1373
+ }
1374
+
1375
+ /**
1376
+ * The deployment-level runtime switch for new managed account sign-ups
1377
+ * (migration 0585): 1 while it allows new accounts (also when no revision
1378
+ * exists), 0 when an operator has paused them. The API additionally honours
1379
+ * its OPENGENI_MANAGED_AUTH_NEW_SIGNUPS_ENABLED ceiling, which this gauge does
1380
+ * not see; `GET /v1/config/client` reports the combined decision.
1381
+ */
1382
+ export function recordManagedAuthNewSignupsSwitchGauge(
1383
+ observability: Observability,
1384
+ signupsEnabled: boolean,
1385
+ ): void {
1386
+ observability.setGauge({
1387
+ name: "opengeni_managed_auth_new_signups_runtime_enabled",
1388
+ help: "Whether the runtime switch allows new managed account sign-ups (1) or has paused them (0).",
1389
+ value: signupsEnabled ? 1 : 0,
1390
+ });
1391
+ }
1392
+
1393
+ /**
1394
+ * The OPENGENI_VERIFIED_SIGNUP_TRIAL_CREDITS_ENABLED master opt-in as this
1395
+ * worker's configuration sees it. The API reads the same shared setting; new
1396
+ * grants happen only while this gauge and the runtime switch gauge are both 1.
1397
+ */
1398
+ export function recordVerifiedSignupTrialDeploymentFlagGauge(
1399
+ observability: Observability,
1400
+ enabled: boolean,
1401
+ ): void {
1402
+ observability.setGauge({
1403
+ name: "opengeni_verified_signup_trial_credits_deployment_enabled",
1404
+ help: "Whether the OPENGENI_VERIFIED_SIGNUP_TRIAL_CREDITS_ENABLED master opt-in is on (1) or off (0) in this deployment's configuration.",
1405
+ value: enabled ? 1 : 0,
1406
+ });
1407
+ }
1408
+
1409
+ /**
1410
+ * Provider-side Modal inventory reconciled against live leases by the orphan
1411
+ * sweep: `running` = every running box in the app, `unleased` = running boxes
1412
+ * no live lease protects (orphans, counted before termination), and
1413
+ * `lease_missing_instance` = warm leases whose exact provider instance is no
1414
+ * longer running (zombies).
1415
+ */
1416
+ export function recordModalSandboxInventoryGauges(
1417
+ observability: Observability,
1418
+ inventory: { running: number; unleased: number; liveLeaseInstancesMissing: number },
1419
+ ): void {
1420
+ const states = {
1421
+ running: inventory.running,
1422
+ unleased: inventory.unleased,
1423
+ lease_missing_instance: inventory.liveLeaseInstancesMissing,
1424
+ };
1425
+ for (const [state, value] of Object.entries(states)) {
1426
+ observability.setGauge({
1427
+ name: "opengeni_modal_sandbox_inventory",
1428
+ help: "Running Modal sandboxes reconciled against live sandbox leases by the orphan sweep.",
1429
+ labels: { state },
1430
+ value: Math.max(0, value),
1431
+ });
1432
+ }
1433
+ }
1434
+
1435
+ /** Warm leases held only by Browser/Computer interaction holders, by time since
1436
+ * the newest interaction activity. Such boxes never drain on their own until the
1437
+ * interaction session ends or the provider deadline. */
1438
+ export function recordInteractionOnlyLeaseGauges(
1439
+ observability: Observability,
1440
+ counts: Record<string, number>,
1441
+ ): void {
1442
+ for (const [idleBucket, value] of Object.entries(counts)) {
1443
+ observability.setGauge({
1444
+ name: "opengeni_sandbox_leases_interaction_only",
1445
+ help: "Warm sandbox leases held only by Browser/Computer sessions, by time since last interaction activity.",
1446
+ labels: { idle_bucket: idleBucket },
1447
+ value: Math.max(0, value),
1448
+ });
1449
+ }
1450
+ }
1451
+
1097
1452
  export function recordSandboxOrphansTerminated(observability: Observability, count: number): void {
1098
1453
  if (count <= 0) {
1099
1454
  return;
@@ -1169,6 +1524,349 @@ export function recordSandboxDeadlineRotationsRequested(
1169
1524
  });
1170
1525
  }
1171
1526
 
1527
+ /** Fixed outcomes of legacy retained-command containment: per-candidate
1528
+ * enrollment inspection, then the enrolled drain's exact cold commit. */
1529
+ export const SANDBOX_COMMAND_CONTAINMENT_OUTCOMES = [
1530
+ "idle_enrolled",
1531
+ "deadline_enrolled",
1532
+ "forced_deadline_enrolled",
1533
+ "quiescence_enrolled",
1534
+ "resumed_enrolled",
1535
+ "not_eligible",
1536
+ "inspection_failed",
1537
+ "contained",
1538
+ "provider_missing",
1539
+ ] as const;
1540
+
1541
+ export type SandboxCommandContainmentOutcome =
1542
+ (typeof SANDBOX_COMMAND_CONTAINMENT_OUTCOMES)[number];
1543
+
1544
+ /**
1545
+ * Shared subscription core shadow (docs/subscription-accounts.md, SUB-COMPAT-03).
1546
+ * Every label is a closed set: provider, security parity, placement agreement,
1547
+ * contract requirement, skip reason and the legacy decision inputs present.
1548
+ */
1549
+ export const SUBSCRIPTION_CORE_SHADOW_PROVIDERS = ["codex", "xai", "claude"] as const;
1550
+ /** Is the account the legacy selector chose inside the core's eligible set? */
1551
+ export const SUBSCRIPTION_CORE_SHADOW_PARITY = [
1552
+ "eligible",
1553
+ "not_authorized",
1554
+ "not_servable",
1555
+ "unknown_connection",
1556
+ "no_selection",
1557
+ ] as const;
1558
+ /** How the core's decision compares with the legacy decision (would-switch). */
1559
+ export const SUBSCRIPTION_CORE_SHADOW_PLACEMENTS = [
1560
+ "same_account",
1561
+ "different_account",
1562
+ "core_waits",
1563
+ "core_runs_legacy_waits",
1564
+ "both_wait",
1565
+ ] as const;
1566
+ /** Requirements the reference checker can report, plus checker failure. */
1567
+ export const SUBSCRIPTION_CORE_SHADOW_REQUIREMENTS = [
1568
+ "SUB-ACCESS-06",
1569
+ "SUB-CREDITS-01",
1570
+ "SUB-ELIG-01",
1571
+ "SUB-ELIG-02",
1572
+ "SUB-ELIG-03",
1573
+ "SUB-ELIG-04",
1574
+ "SUB-ELIG-05",
1575
+ "SUB-FAIL-02",
1576
+ "SUB-FAIL-03",
1577
+ "SUB-FAIL-04",
1578
+ "SUB-OWN-05",
1579
+ "SUB-SEL-02",
1580
+ "SUB-SEL-04",
1581
+ "SUB-SET-06",
1582
+ "SUB-STICK-02",
1583
+ "SUB-WAIT-01",
1584
+ "SUB-WAIT-02",
1585
+ "other",
1586
+ "checker_error",
1587
+ ] as const;
1588
+ export const SUBSCRIPTION_CORE_SHADOW_SKIPS = [
1589
+ "busy",
1590
+ "no_session_actor",
1591
+ "session_not_visible",
1592
+ "timeout",
1593
+ "cancelled",
1594
+ "error",
1595
+ ] as const;
1596
+ /**
1597
+ * Legacy decision inputs the Codex fleet shadow does not record (inventory
1598
+ * Part 3, answer (c)), counted when present in a compared decision.
1599
+ */
1600
+ export const SUBSCRIPTION_CORE_SHADOW_INPUTS = [
1601
+ "manual_pin",
1602
+ "policy_pin",
1603
+ "last_account",
1604
+ "rotation_off",
1605
+ "rotation_on",
1606
+ "organization_pool",
1607
+ "personal_pool",
1608
+ "codex_mode_override",
1609
+ "model_policy",
1610
+ "model_filtered",
1611
+ "plan_excluded",
1612
+ "model_cooldown",
1613
+ "exhausted",
1614
+ "unknown_quota",
1615
+ "compaction_lock",
1616
+ "lease_reused",
1617
+ "truncated",
1618
+ ] as const;
1619
+
1620
+ export type SubscriptionCoreShadowProvider = (typeof SUBSCRIPTION_CORE_SHADOW_PROVIDERS)[number];
1621
+ export type SubscriptionCoreShadowParity = (typeof SUBSCRIPTION_CORE_SHADOW_PARITY)[number];
1622
+ export type SubscriptionCoreShadowPlacement = (typeof SUBSCRIPTION_CORE_SHADOW_PLACEMENTS)[number];
1623
+ export type SubscriptionCoreShadowSkip = (typeof SUBSCRIPTION_CORE_SHADOW_SKIPS)[number];
1624
+ export type SubscriptionCoreShadowInput = (typeof SUBSCRIPTION_CORE_SHADOW_INPUTS)[number];
1625
+
1626
+ export type SubscriptionCoreShadowObservation =
1627
+ | { outcome: "skipped"; reason: SubscriptionCoreShadowSkip }
1628
+ | {
1629
+ outcome: "compared";
1630
+ parity: SubscriptionCoreShadowParity;
1631
+ /** The first reason the legacy account is outside the eligible set. */
1632
+ parityReason: string | null;
1633
+ placement: SubscriptionCoreShadowPlacement;
1634
+ violations: readonly string[];
1635
+ inputs: readonly SubscriptionCoreShadowInput[];
1636
+ };
1637
+
1638
+ const SUBSCRIPTION_CORE_SHADOW_REQUIREMENT_SET = new Set<string>(
1639
+ SUBSCRIPTION_CORE_SHADOW_REQUIREMENTS,
1640
+ );
1641
+ /** First reasons a legacy account is outside the eligible set (policy ineligibility). */
1642
+ export const SUBSCRIPTION_CORE_SHADOW_PARITY_REASONS = [
1643
+ "provider_disabled",
1644
+ "organization_accounts_off",
1645
+ "out_of_scope",
1646
+ "personal_connections_disabled",
1647
+ "personal_owner_inactive",
1648
+ "personal_not_owners_work",
1649
+ "personal_authority_missing",
1650
+ "unhealthy",
1651
+ "allocator_disabled",
1652
+ "wrong_provider",
1653
+ "model_unknown",
1654
+ "model_not_allowed_by_workspace",
1655
+ "model_not_entitled",
1656
+ "model_not_allowed_by_connection",
1657
+ "model_cooling_down",
1658
+ "exhausted",
1659
+ ] as const;
1660
+ const SUBSCRIPTION_CORE_SHADOW_PARITY_REASON_SET = new Set<string>(
1661
+ SUBSCRIPTION_CORE_SHADOW_PARITY_REASONS,
1662
+ );
1663
+
1664
+ const SUBSCRIPTION_CORE_SHADOW_DURATION_BUCKETS = [
1665
+ 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1,
1666
+ ];
1667
+
1668
+ export function recordSubscriptionCoreShadow(
1669
+ observability: Pick<Observability, "incrementCounter" | "observeHistogram">,
1670
+ provider: SubscriptionCoreShadowProvider,
1671
+ observation: SubscriptionCoreShadowObservation,
1672
+ durationSeconds: number,
1673
+ ): void {
1674
+ observability.observeHistogram({
1675
+ name: "opengeni_subscription_core_shadow_duration_seconds",
1676
+ help: "Shared subscription core shadow comparison duration by provider, including skips.",
1677
+ labels: { provider },
1678
+ value: durationSeconds,
1679
+ buckets: SUBSCRIPTION_CORE_SHADOW_DURATION_BUCKETS,
1680
+ });
1681
+ if (observation.outcome === "skipped") {
1682
+ observability.incrementCounter({
1683
+ name: "opengeni_subscription_core_shadow_skips_total",
1684
+ help: "Shadow comparisons not made, by fixed reason; placement is unaffected.",
1685
+ labels: { provider, reason: observation.reason },
1686
+ });
1687
+ return;
1688
+ }
1689
+ observability.incrementCounter({
1690
+ name: "opengeni_subscription_core_shadow_comparisons_total",
1691
+ help: "Shadow comparisons by security parity of the legacy account and core placement agreement.",
1692
+ labels: { provider, parity: observation.parity, placement: observation.placement },
1693
+ });
1694
+ if (observation.parityReason !== null) {
1695
+ observability.incrementCounter({
1696
+ name: "opengeni_subscription_core_shadow_parity_failures_total",
1697
+ help: "Legacy-selected accounts outside the core's eligible set, by first fixed reason.",
1698
+ labels: {
1699
+ provider,
1700
+ reason: SUBSCRIPTION_CORE_SHADOW_PARITY_REASON_SET.has(observation.parityReason)
1701
+ ? observation.parityReason
1702
+ : "other",
1703
+ },
1704
+ });
1705
+ }
1706
+ for (const requirement of new Set(observation.violations)) {
1707
+ observability.incrementCounter({
1708
+ name: "opengeni_subscription_core_shadow_violations_total",
1709
+ help: "Contract invariant violations of the core's own shadow decision, by requirement.",
1710
+ labels: {
1711
+ provider,
1712
+ requirement: SUBSCRIPTION_CORE_SHADOW_REQUIREMENT_SET.has(requirement)
1713
+ ? requirement
1714
+ : "other",
1715
+ },
1716
+ });
1717
+ }
1718
+ for (const input of observation.inputs) {
1719
+ observability.incrementCounter({
1720
+ name: "opengeni_subscription_core_shadow_inputs_total",
1721
+ help: "Legacy decision inputs present in compared shadow decisions, by fixed input.",
1722
+ labels: { provider, input },
1723
+ });
1724
+ }
1725
+ }
1726
+
1727
+ /** A shadow load that outlived its slot ceiling; its slot was given back. */
1728
+ export function recordSubscriptionCoreShadowStuckLoad(
1729
+ observability: Pick<Observability, "incrementCounter">,
1730
+ provider: SubscriptionCoreShadowProvider,
1731
+ ): void {
1732
+ observability.incrementCounter({
1733
+ name: "opengeni_subscription_core_shadow_stuck_loads_total",
1734
+ help: "Shadow world loads still unsettled at the slot ceiling; the slot was released.",
1735
+ labels: { provider },
1736
+ });
1737
+ }
1738
+
1739
+ /** Idle browser/desktop release inspections by fixed outcome: `checkpointing`
1740
+ * (idle browser saves prepared), `released` (holders released without a save). */
1741
+ export function recordSandboxIdleInteractionRelease(
1742
+ observability: Observability,
1743
+ outcome: IdleInteractionInspection,
1744
+ ): void {
1745
+ observability.incrementCounter({
1746
+ name: "opengeni_sandbox_idle_interaction_release_total",
1747
+ help: "Idle browser and desktop release inspections on warm boxes, by fixed outcome.",
1748
+ labels: { outcome },
1749
+ });
1750
+ }
1751
+
1752
+ export function recordSandboxCommandContainment(
1753
+ observability: Observability,
1754
+ outcome: SandboxCommandContainmentOutcome,
1755
+ ): void {
1756
+ observability.incrementCounter({
1757
+ name: "opengeni_sandbox_command_containment_total",
1758
+ help: "Legacy retained-command containment inspections and enrolled drain commits by fixed outcome.",
1759
+ labels: { outcome },
1760
+ });
1761
+ }
1762
+
1763
+ /** Only call after the exact draining->cold commit reports wentCold. The
1764
+ * backend is validated against the closed contract so provider IDs and other
1765
+ * per-sandbox values can never become metric labels. */
1766
+ export function recordSandboxProviderMissingBeforeCapture(
1767
+ observability: Observability,
1768
+ backend: string,
1769
+ ): void {
1770
+ const safeBackend = SandboxBackend.safeParse(backend).success ? backend : "unknown";
1771
+ observability.incrementCounter({
1772
+ name: "opengeni_sandbox_provider_missing_before_capture_total",
1773
+ help: "Exact sandbox cold commits after definitive provider disappearance before workspace capture.",
1774
+ labels: { backend: safeBackend },
1775
+ });
1776
+ }
1777
+
1778
+ /** Host workspace directory outcome after a Docker drain's exact-container
1779
+ * teardown (`retained` means the directory stays on disk). */
1780
+ export function recordSandboxDockerWorkspaceRelease(
1781
+ observability: Observability,
1782
+ status: "released" | "already_released" | "retained",
1783
+ ): void {
1784
+ observability.incrementCounter({
1785
+ name: "opengeni_sandbox_docker_workspace_release_total",
1786
+ help: "Docker drain host workspace directories released or retained after exact-container teardown.",
1787
+ labels: { status },
1788
+ });
1789
+ }
1790
+
1791
+ export function recordSandboxRecoveryObservationGauges(
1792
+ observability: Observability,
1793
+ observations: {
1794
+ providerLosses: number;
1795
+ fallbackSelections: number;
1796
+ freshWorkspaceSelections?: number;
1797
+ },
1798
+ ): void {
1799
+ for (const [kind, value] of [
1800
+ ["provider_missing_before_capture", observations.providerLosses],
1801
+ ["checkpoint_fallback_selected", observations.fallbackSelections],
1802
+ ["fresh_workspace_selected", observations.freshWorkspaceSelections ?? 0],
1803
+ ] as const) {
1804
+ observability.setGauge({
1805
+ name: "opengeni_sandbox_recovery_observations_recent",
1806
+ help: "Committed sandbox recovery observations in the last 30 minutes by fixed kind.",
1807
+ labels: { kind },
1808
+ value,
1809
+ });
1810
+ }
1811
+ }
1812
+
1813
+ /** Fixed outcomes for a committed automatic continuity decision after
1814
+ * definitive managed-provider loss: a singleton checkpoint (`selected`), a
1815
+ * shared-group checkpoint (`selected_shared`), or a new empty workspace
1816
+ * (`fresh_workspace`). Call only after the durable authorization committed. */
1817
+ export const SANDBOX_AUTOMATIC_RECOVERY_OUTCOMES = [
1818
+ "selected",
1819
+ "selected_shared",
1820
+ "fresh_workspace",
1821
+ ] as const;
1822
+ export type SandboxAutomaticRecoveryOutcome = (typeof SANDBOX_AUTOMATIC_RECOVERY_OUTCOMES)[number];
1823
+
1824
+ export function sandboxAutomaticRecoveryOutcome(input: {
1825
+ lane: "checkpoint" | "fresh_workspace";
1826
+ groupSessionCount: number;
1827
+ }): SandboxAutomaticRecoveryOutcome {
1828
+ if (input.lane === "fresh_workspace") return "fresh_workspace";
1829
+ return input.groupSessionCount > 1 ? "selected_shared" : "selected";
1830
+ }
1831
+
1832
+ export function recordSandboxAutomaticRecoverySelected(
1833
+ observability: Observability,
1834
+ backend: string,
1835
+ outcome: SandboxAutomaticRecoveryOutcome,
1836
+ ): void {
1837
+ const safeBackend = SandboxBackend.safeParse(backend).success ? backend : "unknown";
1838
+ observability.incrementCounter({
1839
+ name: "opengeni_sandbox_checkpoint_fallback_total",
1840
+ help: "System-selected continuity after managed provider loss: verified checkpoint or empty workspace.",
1841
+ labels: { backend: safeBackend, outcome },
1842
+ });
1843
+ }
1844
+
1845
+ /** How long live boxes have held changes no checkpoint covers yet. */
1846
+ export function recordSandboxCheckpointStalenessGauges(
1847
+ observability: Observability,
1848
+ staleness: { dirty: number; stale4h: number; stale12h: number; maxAgeSeconds: number },
1849
+ ): void {
1850
+ const values = {
1851
+ dirty: staleness.dirty,
1852
+ stale_4h: staleness.stale4h,
1853
+ stale_12h: staleness.stale12h,
1854
+ } as const;
1855
+ for (const [kind, value] of Object.entries(values)) {
1856
+ observability.setGauge({
1857
+ name: "opengeni_sandbox_checkpoint_staleness",
1858
+ help: "Live Modal sandboxes holding a write their last checkpoint did not capture, by age of the first such write.",
1859
+ labels: { kind },
1860
+ value,
1861
+ });
1862
+ }
1863
+ observability.setGauge({
1864
+ name: "opengeni_sandbox_checkpoint_age_max_seconds",
1865
+ help: "Age in seconds of the oldest write a live Modal sandbox's last checkpoint did not capture.",
1866
+ value: Number.isFinite(staleness.maxAgeSeconds) ? Math.max(0, staleness.maxAgeSeconds) : 0,
1867
+ });
1868
+ }
1869
+
1172
1870
  export function recordSandboxRotationBacklogGauges(
1173
1871
  observability: Observability,
1174
1872
  backlog: {
@@ -1199,6 +1897,11 @@ export function recordSandboxRotationBacklogGauges(
1199
1897
  }
1200
1898
 
1201
1899
  const RETAINED_PROCESS_OWNER_STATES = [
1900
+ // Session-owned background commands (migration 0637). Running is live
1901
+ // session work after its launch turn ended; stopping already has a stop
1902
+ // request, so it still counts as terminal-owner backlog.
1903
+ "background_running",
1904
+ "background_stopping",
1202
1905
  "direct",
1203
1906
  "queued",
1204
1907
  "running",
@@ -1246,7 +1949,7 @@ export function recordRetainedProcessInventoryGauges(
1246
1949
  });
1247
1950
  observability.setGauge({
1248
1951
  name: "opengeni_retained_processes_terminal_owner_backlog",
1249
- help: "Current active retained processes whose exact owner attempt is terminal.",
1952
+ help: "Current active retained processes whose owner is gone: a closed launch attempt or released direct request, or a background command already asked to stop. A running session background command is never counted.",
1250
1953
  labels: { owner_state: ownerState },
1251
1954
  value: count.terminal,
1252
1955
  });
@@ -1286,11 +1989,13 @@ export function recordExpiredDrainingSandboxLeaseGauges(
1286
1989
 
1287
1990
  export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
1288
1991
  "claim_failed",
1992
+ "background_recovered",
1289
1993
  "proof_exited",
1290
1994
  "proof_lost",
1291
1995
  "proof_checkpoint_failed",
1292
1996
  "settled_exited",
1293
1997
  "settled_lost",
1998
+ "provider_lost_whole_box",
1294
1999
  "settlement_failed",
1295
2000
  "identity_mismatch",
1296
2001
  "resume_state_missing",
@@ -1339,6 +2044,147 @@ export function recordCreditMicros(
1339
2044
  });
1340
2045
  }
1341
2046
 
2047
+ // ── Per-model usage and money ───────────────────────────────────────────────
2048
+ // `model` is the deployment catalog product id (resolved by the caller), or
2049
+ // `custom` for a workspace-owned model. The recorder bounds it again: a value
2050
+ // outside a conservative id shape becomes `custom`, and after
2051
+ // MAX_MODEL_METRIC_LABELS distinct values per process every new value becomes
2052
+ // `other`, so a catalog change or a bug can never grow the series count without
2053
+ // bound. Account, workspace, user and session ids are never labels here.
2054
+
2055
+ /** Who pays the upstream provider for the call. */
2056
+ export type ModelUsagePayer = "deployment" | "external";
2057
+
2058
+ const MODEL_METRIC_LABEL_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,127}$/;
2059
+ const MAX_MODEL_METRIC_LABELS = 64;
2060
+ const modelMetricLabels = new WeakMap<Observability, Set<string>>();
2061
+
2062
+ export function boundedModelMetricLabel(observability: Observability, model: string): string {
2063
+ if (model === "custom" || model === "other") return model;
2064
+ if (!MODEL_METRIC_LABEL_PATTERN.test(model)) return "custom";
2065
+ let seen = modelMetricLabels.get(observability);
2066
+ if (!seen) {
2067
+ seen = new Set<string>();
2068
+ modelMetricLabels.set(observability, seen);
2069
+ }
2070
+ if (seen.has(model)) return model;
2071
+ if (seen.size >= MAX_MODEL_METRIC_LABELS) return "other";
2072
+ seen.add(model);
2073
+ return model;
2074
+ }
2075
+
2076
+ function nonnegativeSafeInteger(value: unknown): number {
2077
+ return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : 0;
2078
+ }
2079
+
2080
+ /**
2081
+ * One authoritative model response, by provider and catalog model. Recorded
2082
+ * behind the same durable usage-event fence as the provider-only cache metrics,
2083
+ * so a duplicate or late response is not counted twice.
2084
+ * - `opengeni_model_responses_total{provider,model,priced}` — responses with
2085
+ * reported usage; `priced="false"` means no provider cost estimate exists.
2086
+ * - `opengeni_model_tokens_total{provider,model,type}` — `input` (all prompt
2087
+ * tokens, including cached and cache-write), `cached_input` and
2088
+ * `cache_write` (subsets of `input`), `output`, and `reasoning` (a subset
2089
+ * of `output`).
2090
+ * - `opengeni_model_provider_cost_micros_total{provider,model,payer,pricing_source}`
2091
+ * — estimated upstream cost in USD micros: configured list price, or the
2092
+ * gateway-reported cost. `payer="external"` is a customer-paid subscription
2093
+ * or key, so that cost is a list-price equivalent Opengeni does not pay.
2094
+ */
2095
+ export function recordModelResponseUsage(
2096
+ observability: Observability,
2097
+ input: {
2098
+ provider: string;
2099
+ model: string;
2100
+ payer: ModelUsagePayer;
2101
+ tokens: {
2102
+ inputTokens: number | null;
2103
+ cachedTokens: number | null;
2104
+ cacheWriteTokens: number | null;
2105
+ outputTokens: number | null;
2106
+ reasoningTokens: number | null;
2107
+ };
2108
+ estimatedProviderCostMicros: number | null;
2109
+ pricingSource: "configured_list_price" | "gateway_reported" | null;
2110
+ webSearchRequests?: number;
2111
+ },
2112
+ ): void {
2113
+ const model = boundedModelMetricLabel(observability, input.model);
2114
+ const base = { provider: input.provider, model };
2115
+ const priced = input.pricingSource !== null && input.estimatedProviderCostMicros !== null;
2116
+ observability.incrementCounter({
2117
+ name: "opengeni_model_responses_total",
2118
+ help: "Authoritative model responses with reported usage, by provider, catalog model and whether a cost estimate exists.",
2119
+ labels: { ...base, priced: priced ? "true" : "false" },
2120
+ });
2121
+ const tokenTypes = [
2122
+ ["input", input.tokens.inputTokens],
2123
+ ["cached_input", input.tokens.cachedTokens],
2124
+ ["cache_write", input.tokens.cacheWriteTokens],
2125
+ ["output", input.tokens.outputTokens],
2126
+ ["reasoning", input.tokens.reasoningTokens],
2127
+ ] as const;
2128
+ for (const [type, value] of tokenTypes) {
2129
+ const amount = modelUsageTokenCountOrNull(value);
2130
+ if (amount === null || amount === 0) continue;
2131
+ incrementBoundedModelCacheCounter(observability, {
2132
+ name: "opengeni_model_tokens_total",
2133
+ help: "Tokens of authoritative model responses by provider, catalog model and token type (cached_input and cache_write are subsets of input; reasoning is a subset of output).",
2134
+ provider: input.provider,
2135
+ labels: { ...base, type },
2136
+ amount,
2137
+ });
2138
+ }
2139
+ const searches = nonnegativeSafeInteger(input.webSearchRequests ?? 0);
2140
+ if (searches > 0)
2141
+ observability.incrementCounter({
2142
+ name: "opengeni_model_web_search_requests_total",
2143
+ help: "Provider-run web searches billed with authoritative model responses, by provider, catalog model and payer.",
2144
+ labels: { ...base, payer: input.payer },
2145
+ amount: searches,
2146
+ });
2147
+ const cost = nonnegativeSafeInteger(input.estimatedProviderCostMicros);
2148
+ if (priced && input.pricingSource !== null && cost > 0) {
2149
+ observability.incrementCounter({
2150
+ name: "opengeni_model_provider_cost_micros_total",
2151
+ help: "Estimated upstream provider cost in USD micros by provider, catalog model, payer and pricing source.",
2152
+ labels: { ...base, payer: input.payer, pricing_source: input.pricingSource },
2153
+ amount: cost,
2154
+ });
2155
+ }
2156
+ }
2157
+
2158
+ /**
2159
+ * Credits actually debited for one model response, recorded once when the
2160
+ * ledger row is inserted (an idempotent replay debits 0 and records nothing).
2161
+ * `funding="promotional"` is the part paid by scoped promotional grants
2162
+ * (verified-signup trial, scoped coupon offers); `funding="general"` is the
2163
+ * rest, paid from general credit: purchased credits plus any unscoped grant.
2164
+ */
2165
+ export function recordModelCreditsCharged(
2166
+ observability: Observability | undefined,
2167
+ input: { provider: string; model: string; debitedMicros: number; grantDebitedMicros: number },
2168
+ ): void {
2169
+ if (!observability) return;
2170
+ const debited = nonnegativeSafeInteger(input.debitedMicros);
2171
+ if (debited === 0) return;
2172
+ const promotional = Math.min(debited, nonnegativeSafeInteger(input.grantDebitedMicros));
2173
+ const model = boundedModelMetricLabel(observability, input.model);
2174
+ for (const [funding, amount] of [
2175
+ ["promotional", promotional],
2176
+ ["general", debited - promotional],
2177
+ ] as const) {
2178
+ if (amount === 0) continue;
2179
+ observability.incrementCounter({
2180
+ name: "opengeni_model_credits_charged_micros_total",
2181
+ help: "Credit micros debited for model usage by provider, catalog model and funding (promotional grant or general credit).",
2182
+ labels: { provider: input.provider, model, funding },
2183
+ amount,
2184
+ });
2185
+ }
2186
+ }
2187
+
1342
2188
  const MODEL_REQUEST_PHASE_BUCKETS = [
1343
2189
  0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5, 10, 30, 60, 300,
1344
2190
  ];
@@ -1373,10 +2219,18 @@ export function recordModelRequestPhase(
1373
2219
  }
1374
2220
 
1375
2221
  export type TurnStartupPhase =
2222
+ | "services_initialization"
2223
+ | "claim_catalog_read"
2224
+ | "claim_atomic"
2225
+ | "claim_session_read"
2226
+ | "claim_capability_settings"
2227
+ | "learning_policy_freeze"
2228
+ | "attachment_authority_projection"
1376
2229
  | "claim_and_policy"
1377
2230
  | "turn_start_settlement"
1378
2231
  | "credential_selection"
1379
2232
  | "runtime_preparation"
2233
+ | "session_mcp_settings"
1380
2234
  | "sandbox_establish"
1381
2235
  | "file_resolution"
1382
2236
  | "tool_context_preparation"
@@ -1556,6 +2410,32 @@ export function recordTurnStartupPhase(
1556
2410
  });
1557
2411
  }
1558
2412
 
2413
+ /** Observe an existing dependency only. Child intervals can overlap and are
2414
+ * not additive; observer failure must never change authority or error identity. */
2415
+ export async function measureTurnStartupPhase<T>(
2416
+ observability: Observability,
2417
+ input: Omit<Parameters<typeof recordTurnStartupPhase>[1], "durationSeconds" | "outcome">,
2418
+ work: () => Promise<T>,
2419
+ ): Promise<T> {
2420
+ const started = performance.now();
2421
+ let outcome: TurnStartupOutcome = "failed";
2422
+ try {
2423
+ const result = await work();
2424
+ outcome = "completed";
2425
+ return result;
2426
+ } finally {
2427
+ try {
2428
+ recordTurnStartupPhase(observability, {
2429
+ ...input,
2430
+ outcome,
2431
+ durationSeconds: (performance.now() - started) / 1_000,
2432
+ });
2433
+ } catch {
2434
+ // Telemetry is not an execution gate.
2435
+ }
2436
+ }
2437
+ }
2438
+
1559
2439
  /** Background MCP preparation must not inflate startup phase distributions. */
1560
2440
  export function recordToolPreparationPhase(
1561
2441
  observability: Observability,
@@ -1602,6 +2482,7 @@ function completedOperationSpan(
1602
2482
  outcome: string;
1603
2483
  provider?: string;
1604
2484
  backend?: string;
2485
+ tool?: string;
1605
2486
  correlationId?: string | undefined;
1606
2487
  },
1607
2488
  ): void {
@@ -1838,34 +2719,61 @@ function contentDeltaClass(type: SessionEventType): StreamDeltaClass | null {
1838
2719
  return null;
1839
2720
  }
1840
2721
 
1841
- // TTFT and inter-delta live on a human-perceptible scale (tens of ms to a few
1842
- // seconds), so they get their own SHORT buckets — the default duration buckets
1843
- // (which run to 3600s) would collapse every real streaming value into one bucket.
1844
- const STREAM_TTFT_BUCKETS = [0.02, 0.05, 0.1, 0.2, 0.35, 0.5, 0.75, 1, 1.5, 2, 3, 5, 10];
2722
+ // TTFT lives on a human-perceptible scale, but reasoning models legitimately
2723
+ // think for tens of seconds before their first streamed token. The finite
2724
+ // buckets therefore run well past 10s: a top bucket at 10s made every slower
2725
+ // first token land in +Inf, so p50/p99 saturated at exactly "10" and hid how
2726
+ // slow the tail really was.
2727
+ const STREAM_TTFT_BUCKETS = [
2728
+ 0.02, 0.05, 0.1, 0.2, 0.35, 0.5, 0.75, 1, 1.5, 2, 3, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 300,
2729
+ ];
1845
2730
  const STREAM_INTER_DELTA_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5];
2731
+ // Opengeni's own per-request work before the provider sees bytes: admission,
2732
+ // durable history/audit checkpoints, request build. Normally milliseconds to a
2733
+ // second; tens of seconds means our database or consumer is the bottleneck.
2734
+ const MODEL_REQUEST_PRE_DISPATCH_BUCKETS = [
2735
+ 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120,
2736
+ ];
2737
+
2738
+ /** First streamed content of a provider response. `any` is the first reasoning
2739
+ * or answer delta; `text` is the first answer-text delta. They overlap — never
2740
+ * sum across `content`. */
2741
+ export type ProviderFirstContent = "any" | "text";
1846
2742
 
1847
2743
  /**
1848
- * Per-turn stream-timing tracker fed every normalized runtime event in push order.
1849
- * It emits two model-responsiveness SLIs from the worker's seat on the stream:
2744
+ * Per-stream timing tracker fed every normalized runtime event in push order,
2745
+ * plus two producer-side hooks. It separates OUR latency from the PROVIDER's:
1850
2746
  *
1851
- * - `opengeni_stream_ttft_seconds{provider}` — time from a model (re)start to its
1852
- * first streamed content delta. The anchor starts at construction (≈ runStream
1853
- * start, so the first observation is "how long until text appears") and re-arms
1854
- * on every non-content event (a tool call, a completed message, a usage frame),
1855
- * so a post-tool response measures the model's restart latency, NOT our own
1856
- * tool-execution time.
2747
+ * - `opengeni_stream_ttft_seconds{provider}` — user-perceived (re)start latency:
2748
+ * time from a stream start / structural boundary (tool call, tool result,
2749
+ * completed message, usage frame) to the first streamed content delta. It
2750
+ * mixes our between-call work with provider time; keep it for the absolute
2751
+ * dashboard view, not for provider alerting.
2752
+ * - `opengeni_model_request_pre_dispatch_seconds{provider}` — OUR per-request
2753
+ * latency: SDK model entry (the first admission check) to the literal
2754
+ * provider dispatch. Covers admission, the durable history checkpoint,
2755
+ * credit revalidation, the durable request audit and request build.
2756
+ * - `opengeni_model_provider_ttft_seconds{provider,content}` — the PROVIDER's
2757
+ * latency: literal dispatch to the first streamed `any` (reasoning or text)
2758
+ * delta and to the first answer `text` delta. Includes network, provider
2759
+ * queueing, prompt prefill and reasoning before the first streamed token.
1857
2760
  * - `opengeni_stream_inter_delta_gap_seconds{provider,class}` — gap between
1858
- * consecutive content deltas of the SAME class. The run resets on any
1859
- * non-content event so a gap never spans a tool call or a model boundary — it
1860
- * measures only the choppiness of a live token stream.
2761
+ * consecutive content deltas of the SAME class, reset at every structural
2762
+ * event so a gap never spans a tool call or a model boundary.
1861
2763
  *
1862
- * Purely observational and clock-injectable; it never touches the events it sees.
2764
+ * Dispatch timing deliberately survives structural events: the consumer may
2765
+ * still be persisting the previous response's tool results when the producer
2766
+ * dispatches the next request, but the next request's first delta always
2767
+ * follows its own dispatch. Purely observational and clock-injectable.
1863
2768
  */
1864
2769
  export class StreamTimingMetrics {
1865
2770
  private readonly now: () => number;
1866
2771
  private ttftAnchor: number;
1867
2772
  private ttftArmed = true;
1868
2773
  private readonly lastDeltaAt = new Map<StreamDeltaClass, number>();
2774
+ private modelEntryAt: number | null = null;
2775
+ private dispatchedAt: number | null = null;
2776
+ private readonly providerFirstRecorded = new Set<ProviderFirstContent>();
1869
2777
 
1870
2778
  constructor(
1871
2779
  private readonly observability: Observability,
@@ -1875,6 +2783,31 @@ export class StreamTimingMetrics {
1875
2783
  this.ttftAnchor = this.now();
1876
2784
  }
1877
2785
 
2786
+ /** Producer side: the SDK entered a model request (first admission check).
2787
+ * Admission can be re-entered for the same request; keep the earliest. */
2788
+ onModelRequestEntry(): void {
2789
+ if (this.modelEntryAt === null) this.modelEntryAt = this.now();
2790
+ }
2791
+
2792
+ /** Producer side: request bytes are about to leave this process. */
2793
+ onProviderDispatch(): void {
2794
+ const at = this.now();
2795
+ if (this.modelEntryAt !== null) {
2796
+ this.observability.observeHistogram({
2797
+ name: "opengeni_model_request_pre_dispatch_seconds",
2798
+ help: "Seconds of Opengeni work from SDK model-request entry to literal provider dispatch.",
2799
+ buckets: MODEL_REQUEST_PRE_DISPATCH_BUCKETS,
2800
+ labels: { provider: this.options.provider },
2801
+ value: Math.max(0, (at - this.modelEntryAt) / 1000),
2802
+ });
2803
+ this.modelEntryAt = null;
2804
+ }
2805
+ // A transport retry re-dispatches the same request; the first delta then
2806
+ // belongs to the latest dispatch.
2807
+ this.dispatchedAt = at;
2808
+ this.providerFirstRecorded.clear();
2809
+ }
2810
+
1878
2811
  onEvent(type: SessionEventType): void {
1879
2812
  const deltaClass = contentDeltaClass(type);
1880
2813
  if (deltaClass === null) {
@@ -1890,13 +2823,17 @@ export class StreamTimingMetrics {
1890
2823
  if (this.ttftArmed) {
1891
2824
  this.observability.observeHistogram({
1892
2825
  name: "opengeni_stream_ttft_seconds",
1893
- help: "Seconds from a model (re)start to its first streamed content delta.",
2826
+ help: "Seconds from a stream start or structural boundary to the next streamed content delta (user-perceived; includes Opengeni between-call work).",
1894
2827
  buckets: STREAM_TTFT_BUCKETS,
1895
2828
  labels: { provider: this.options.provider },
1896
2829
  value: Math.max(0, (at - this.ttftAnchor) / 1000),
1897
2830
  });
1898
2831
  this.ttftArmed = false;
1899
2832
  }
2833
+ if (this.dispatchedAt !== null) {
2834
+ this.observeProviderFirst("any", at);
2835
+ if (deltaClass === "message") this.observeProviderFirst("text", at);
2836
+ }
1900
2837
  const last = this.lastDeltaAt.get(deltaClass);
1901
2838
  if (last !== undefined) {
1902
2839
  this.observability.observeHistogram({
@@ -1909,6 +2846,18 @@ export class StreamTimingMetrics {
1909
2846
  }
1910
2847
  this.lastDeltaAt.set(deltaClass, at);
1911
2848
  }
2849
+
2850
+ private observeProviderFirst(content: ProviderFirstContent, at: number): void {
2851
+ if (this.dispatchedAt === null || this.providerFirstRecorded.has(content)) return;
2852
+ this.providerFirstRecorded.add(content);
2853
+ this.observability.observeHistogram({
2854
+ name: "opengeni_model_provider_ttft_seconds",
2855
+ help: "Seconds from literal provider dispatch to the first streamed content delta (any = reasoning or text, text = answer text).",
2856
+ buckets: STREAM_TTFT_BUCKETS,
2857
+ labels: { provider: this.options.provider, content },
2858
+ value: Math.max(0, (at - this.dispatchedAt) / 1000),
2859
+ });
2860
+ }
1912
2861
  }
1913
2862
 
1914
2863
  // Batch shapes: sizes are small integers; durations are the append+publish round
@@ -2206,13 +3155,26 @@ function requiredTemporalNumber(
2206
3155
  if (value === null || value === undefined) {
2207
3156
  throw new Error(`Temporal stats omitted required ${field}`);
2208
3157
  }
2209
- const parsed = Number(value);
3158
+ // protobuf Long is supported through its string representation. Refuse JS
3159
+ // coercions (booleans, arrays, blanks) and hostile toString implementations.
3160
+ let raw: unknown = value;
3161
+ try {
3162
+ if (typeof raw === "object" && raw !== null && !Array.isArray(raw)) raw = raw.toString();
3163
+ } catch {
3164
+ throw new Error(`Temporal stats returned invalid ${field}`);
3165
+ }
3166
+ const parsed =
3167
+ typeof raw === "number"
3168
+ ? raw
3169
+ : typeof raw === "string" && /^\d+(?:\.\d+)?(?:[eE][+-]?\d+)?$/.test(raw)
3170
+ ? Number(raw)
3171
+ : Number.NaN;
2210
3172
  if (
2211
3173
  !Number.isFinite(parsed) ||
2212
3174
  parsed < 0 ||
2213
3175
  (options.integer && !Number.isSafeInteger(parsed))
2214
3176
  ) {
2215
- throw new Error(`Temporal stats returned invalid ${field}: ${String(value)}`);
3177
+ throw new Error(`Temporal stats returned invalid ${field}`);
2216
3178
  }
2217
3179
  return parsed;
2218
3180
  }
@@ -2264,3 +3226,141 @@ export function modelCallAccountContext(input: {
2264
3226
  accountChangedFromPrevCall,
2265
3227
  };
2266
3228
  }
3229
+
3230
+ export type CodeSearchCallOutcome =
3231
+ | "completed"
3232
+ | "jev_unavailable"
3233
+ | "jev_rejected"
3234
+ | "workspace_unavailable"
3235
+ | "invalid_arguments"
3236
+ | "breaker_open"
3237
+ | "cancelled"
3238
+ | "failed";
3239
+
3240
+ /** One `code_search` tool call: outcome, wall time and the Jev work it used. */
3241
+ export function recordCodeSearchCall(
3242
+ observability: Observability,
3243
+ input: {
3244
+ outcome: CodeSearchCallOutcome;
3245
+ durationSeconds: number;
3246
+ jevRequests: number;
3247
+ jevCostUsd: number;
3248
+ },
3249
+ ): void {
3250
+ observability.incrementCounter({
3251
+ name: "opengeni_code_search_calls_total",
3252
+ help: "Jev-backed code_search tool calls by outcome.",
3253
+ labels: { outcome: input.outcome },
3254
+ });
3255
+ observability.observeHistogram({
3256
+ name: "opengeni_code_search_duration_seconds",
3257
+ help: "Wall time of one code_search tool call.",
3258
+ buckets: [0.5, 1, 2, 4, 8, 15, 30, 60],
3259
+ labels: { outcome: input.outcome },
3260
+ value: Math.max(0, input.durationSeconds),
3261
+ });
3262
+ if (input.jevRequests > 0) {
3263
+ observability.incrementCounter({
3264
+ name: "opengeni_code_search_jev_requests_total",
3265
+ help: "Jev requests made by code_search.",
3266
+ amount: input.jevRequests,
3267
+ });
3268
+ }
3269
+ if (input.jevCostUsd > 0) {
3270
+ observability.incrementCounter({
3271
+ name: "opengeni_code_search_jev_cost_micro_usd_total",
3272
+ help: "Estimated Jev list-price cost of code_search, in micro-USD.",
3273
+ amount: Math.round(input.jevCostUsd * 1_000_000),
3274
+ });
3275
+ }
3276
+ }
3277
+
3278
+ /**
3279
+ * One skill_read call. Labels are closed sets: `skill` is a built-in id or
3280
+ * `custom` for every other Skill, so tenant Skill ids, names, and requested
3281
+ * identifiers never become labels. `source` is `unknown` when the read was
3282
+ * refused before a Skill resolved.
3283
+ */
3284
+ export function recordSkillRead(
3285
+ observability: Observability,
3286
+ read: {
3287
+ caller: "model" | "codemode";
3288
+ kind: SkillReadKind;
3289
+ source: SkillUseSource | null;
3290
+ /** The resolved Skill id, or the requested identifier when none resolved. */
3291
+ skill: string;
3292
+ },
3293
+ ): void {
3294
+ observability.incrementCounter({
3295
+ name: "opengeni_skill_reads_total",
3296
+ help: "skill_read calls by Skill source, built-in Skill id (custom for any other Skill), result kind, and caller.",
3297
+ labels: {
3298
+ source: read.source ?? "unknown",
3299
+ skill: skillReadMetricLabel(read.source, read.skill),
3300
+ kind: read.kind,
3301
+ caller: read.caller,
3302
+ },
3303
+ });
3304
+ }
3305
+
3306
+ /**
3307
+ * One skill_checkout call. Labels are closed sets and never carry Skill ids,
3308
+ * names, or paths. Phases that did not run are not observed.
3309
+ */
3310
+ export function recordSkillCheckout(
3311
+ observability: Observability,
3312
+ checkout: {
3313
+ outcome: "written" | "unchanged" | "refused" | "failed";
3314
+ selection: "all" | "paths";
3315
+ written: number;
3316
+ unchanged: number;
3317
+ resolveSeconds: number | null;
3318
+ sandboxSeconds: number | null;
3319
+ writeSeconds: number | null;
3320
+ totalSeconds: number;
3321
+ },
3322
+ ): void {
3323
+ observability.incrementCounter({
3324
+ name: "opengeni_skill_checkouts_total",
3325
+ help: "skill_checkout calls by outcome and whether they selected specific paths.",
3326
+ labels: { outcome: checkout.outcome, selection: checkout.selection },
3327
+ });
3328
+ const phases = [
3329
+ ["resolve", checkout.resolveSeconds],
3330
+ ["sandbox", checkout.sandboxSeconds],
3331
+ ["write", checkout.writeSeconds],
3332
+ ["total", checkout.totalSeconds],
3333
+ ] as const;
3334
+ for (const [phase, seconds] of phases) {
3335
+ if (seconds === null) continue;
3336
+ observability.observeHistogram({
3337
+ name: "opengeni_skill_checkout_duration_seconds",
3338
+ help: "Wall time of skill_checkout phases: resolve (authority and Skill read), sandbox (filesystem handle, including lazy box start), write (batched file write), and total.",
3339
+ buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 4, 8, 15, 30, 60],
3340
+ labels: { phase, outcome: checkout.outcome },
3341
+ value: Math.max(0, seconds),
3342
+ });
3343
+ }
3344
+ for (const [result, amount] of [
3345
+ ["written", checkout.written],
3346
+ ["unchanged", checkout.unchanged],
3347
+ ] as const) {
3348
+ if (amount <= 0) continue;
3349
+ observability.incrementCounter({
3350
+ name: "opengeni_skill_checkout_files_total",
3351
+ help: "Files skill_checkout wrote or found already present with the same content.",
3352
+ labels: { result },
3353
+ amount,
3354
+ });
3355
+ }
3356
+ }
3357
+
3358
+ function skillReadMetricLabel(source: SkillUseSource | null, skill: string): string {
3359
+ if (source !== null && source !== "builtin") return "custom";
3360
+ // A refused read may name a built-in by its plain name.
3361
+ for (const candidate of source === null ? [skill, `builtin:${skill}`] : [skill]) {
3362
+ const parsed = BundledSkillId.safeParse(candidate);
3363
+ if (parsed.success) return parsed.data;
3364
+ }
3365
+ return "custom";
3366
+ }