@opengeni/worker-bundle 0.16.21 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/dist/activities/agent-turn.d.ts +145 -32
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +3 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -7
  10. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +2 -2
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  25. package/dist/activities/retained-screenshots.d.ts +129 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +2 -2
  28. package/dist/activities/run-input.d.ts +5 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +11 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -147
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  64. package/dist/chunk-RAMNYSAW.js.map +1 -0
  65. package/dist/concurrency.d.ts +108 -5
  66. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  67. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  68. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  69. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  70. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  71. package/dist/editable-artifact-materializer.d.ts +262 -0
  72. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  73. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  74. package/dist/host-export.d.ts +1 -1
  75. package/dist/host-export.js +1 -1
  76. package/dist/index.d.ts +176 -10
  77. package/dist/index.js +675 -16460
  78. package/dist/index.js.map +1 -1
  79. package/dist/memory-pressure-guard.d.ts +57 -0
  80. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  81. package/dist/observability-metrics.d.ts +33 -0
  82. package/dist/sandbox-reaper-contract.d.ts +40 -0
  83. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  84. package/dist/sandbox-resume.d.ts +40 -7
  85. package/dist/sandbox-routing.d.ts +13 -6
  86. package/dist/worker-service-lifecycle.d.ts +6 -2
  87. package/dist/workflow-bundle.js +1044 -363
  88. package/dist/workflows/activities.d.ts +7 -1
  89. package/dist/workflows/document-index.d.ts +1 -1
  90. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  91. package/dist/workflows/rig-verification.d.ts +1 -1
  92. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  93. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  94. package/dist/workflows/session.d.ts +2 -2
  95. package/dist/workflows/video-generation.d.ts +10 -0
  96. package/dist/workflows/workflow-wake.d.ts +1 -1
  97. package/dist/workflows.d.ts +9 -7
  98. package/package.json +34 -15
  99. package/src/activities/agent-turn.ts +1905 -559
  100. package/src/activities/api-integrations.ts +138 -0
  101. package/src/activities/codemode-dispatcher.ts +335 -0
  102. package/src/activities/codemode-token-renewal.ts +64 -0
  103. package/src/activities/codex-capacity.ts +16 -7
  104. package/src/activities/codex-image-generation.ts +56 -0
  105. package/src/activities/codex-rotation.ts +19 -5
  106. package/src/activities/context-compaction.ts +47 -6
  107. package/src/activities/documents.ts +14 -4
  108. package/src/activities/environment.ts +52 -60
  109. package/src/activities/expiring-material-renewal.ts +16 -7
  110. package/src/activities/file-upload-reaper.ts +34 -3
  111. package/src/activities/gateway-image-generation.ts +146 -0
  112. package/src/activities/gateway-video-generation.ts +298 -0
  113. package/src/activities/generated-images.ts +860 -0
  114. package/src/activities/git-credential-renewal.ts +3 -3
  115. package/src/activities/goals.ts +6 -2
  116. package/src/activities/google-drive-changes.ts +454 -0
  117. package/src/activities/image-generation-operation.ts +333 -0
  118. package/src/activities/image-generation-references.ts +118 -0
  119. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  120. package/src/activities/knowledge-source-sync.ts +2840 -0
  121. package/src/activities/mcp-credentials.ts +1 -0
  122. package/src/activities/packs.ts +55 -55
  123. package/src/activities/recording.ts +4 -5
  124. package/src/activities/retained-screenshot-reaper.ts +202 -0
  125. package/src/activities/retained-screenshots.ts +1317 -0
  126. package/src/activities/rig-verification.ts +567 -77
  127. package/src/activities/run-credential-renewal.ts +9 -2
  128. package/src/activities/run-input.ts +40 -10
  129. package/src/activities/sandbox-images.ts +167 -0
  130. package/src/activities/sandbox-lease.ts +1065 -483
  131. package/src/activities/scheduled-tasks.ts +96 -7
  132. package/src/activities/session-state.ts +4 -4
  133. package/src/activities/streaming.ts +42 -3
  134. package/src/activities/types.ts +87 -5
  135. package/src/activities/video-generation-admission.ts +270 -0
  136. package/src/activities/video-generation-credential.ts +34 -0
  137. package/src/activities/video-generation-reconciliation.ts +578 -0
  138. package/src/activities/video-media-validation.ts +326 -0
  139. package/src/activities/video-output-retention.ts +314 -0
  140. package/src/activities/video-reference-staging.ts +263 -0
  141. package/src/activities/workflow-wake.ts +2 -2
  142. package/src/activities/workspace-capture.ts +77 -13
  143. package/src/activities-control.ts +54 -0
  144. package/src/activities-turn.ts +40 -0
  145. package/src/activities.ts +50 -118
  146. package/src/activity-services.ts +62 -0
  147. package/src/artifact-materializer-entry.ts +35 -0
  148. package/src/artifact-outbox-entry.ts +29 -0
  149. package/src/concurrency.ts +484 -6
  150. package/src/editable-artifact-hint-broker.ts +235 -0
  151. package/src/editable-artifact-materializer-launcher.ts +515 -0
  152. package/src/editable-artifact-materializer-service.ts +787 -0
  153. package/src/editable-artifact-materializer-storage.ts +413 -0
  154. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  155. package/src/editable-artifact-materializer.ts +1289 -0
  156. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  157. package/src/editable-artifact-outbox-service.ts +438 -0
  158. package/src/host-export-pump.ts +28 -11
  159. package/src/http.ts +3 -6
  160. package/src/index.ts +277 -72
  161. package/src/memory-pressure-guard.ts +489 -0
  162. package/src/model-checkpoint-memory-collector.ts +116 -0
  163. package/src/observability-metrics.ts +202 -7
  164. package/src/sandbox-reaper-contract.ts +73 -0
  165. package/src/sandbox-reaper-timeout.ts +79 -0
  166. package/src/sandbox-resume.ts +331 -157
  167. package/src/sandbox-routing.ts +49 -39
  168. package/src/worker-service-lifecycle.ts +82 -12
  169. package/src/workflows/activities.ts +30 -0
  170. package/src/workflows/file-upload-reaper.ts +6 -1
  171. package/src/workflows/knowledge-source-sync.ts +83 -0
  172. package/src/workflows/sandbox-reaper.ts +183 -21
  173. package/src/workflows/scheduled-tasks.ts +42 -6
  174. package/src/workflows/video-generation.ts +36 -0
  175. package/src/workflows.ts +17 -1
  176. package/dist/activities/redaction.d.ts +0 -1
  177. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  178. package/dist/chunk-BJUW5LCM.js.map +0 -1
  179. package/src/activities/redaction.ts +0 -9
  180. package/src/activities/toolspace-token-renewal.ts +0 -57
@@ -0,0 +1,828 @@
1
+ // src/observability-metrics.ts
2
+ import { createHash } from "crypto";
3
+ import { errorCodeToJSON } from "@opengeni/agent-proto";
4
+ import { SandboxBackend } from "@opengeni/contracts";
5
+ import {
6
+ SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
7
+ modelUsageTokenCountOrNull
8
+ } from "@opengeni/runtime";
9
+ var turnTrackers = /* @__PURE__ */ new WeakMap();
10
+ var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
11
+ var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
12
+ var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
13
+ function observabilityEventLogger(observability) {
14
+ return {
15
+ debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
16
+ warn: (message, attributes) => observability.warn(message, eventAttributes(attributes))
17
+ };
18
+ }
19
+ function eventAttributes(attributes) {
20
+ if (!attributes) {
21
+ return void 0;
22
+ }
23
+ const projected = {};
24
+ for (const [key, value] of Object.entries(attributes)) {
25
+ projected[key] = eventAttributeValue(value);
26
+ }
27
+ return projected;
28
+ }
29
+ function eventAttributeValue(value) {
30
+ if (value === null || value === void 0 || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
31
+ return value;
32
+ }
33
+ try {
34
+ return JSON.stringify(value);
35
+ } catch {
36
+ return String(value);
37
+ }
38
+ }
39
+ function runtimeMetricsHooksForObservability(observability) {
40
+ return {
41
+ onModelCall: ({ provider, outcome, durationSeconds }) => {
42
+ observability.incrementCounter({
43
+ name: "opengeni_model_calls_total",
44
+ help: "Total model calls by provider and outcome.",
45
+ labels: { provider, outcome }
46
+ });
47
+ observability.observeHistogram({
48
+ name: "opengeni_model_call_duration_seconds",
49
+ help: "Model call duration in seconds by provider.",
50
+ labels: { provider },
51
+ value: durationSeconds
52
+ });
53
+ },
54
+ onSandboxCreate: ({ backend, imageSource, outcome, durationSeconds }) => {
55
+ observability.incrementCounter({
56
+ name: "opengeni_sandbox_creates_total",
57
+ help: "Total sandbox create attempts by backend and outcome.",
58
+ labels: { backend, image_source: imageSource, outcome }
59
+ });
60
+ observability.observeHistogram({
61
+ name: "opengeni_sandbox_create_duration_seconds",
62
+ help: "Sandbox create duration in seconds by backend.",
63
+ labels: { backend, image_source: imageSource },
64
+ value: durationSeconds
65
+ });
66
+ },
67
+ onSandboxWarmingTimeout: ({ backend, stage }) => {
68
+ observability.incrementCounter({
69
+ name: "opengeni_sandbox_warming_timeouts_total",
70
+ help: "Total sandbox warming timeouts.",
71
+ labels: { backend, stage }
72
+ });
73
+ },
74
+ onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
75
+ observability.incrementCounter({
76
+ name: "opengeni_machine_op_total",
77
+ help: "Total Connected Machine control ops by op, outcome, and typed fault code.",
78
+ labels: { backend, op, outcome, code: code ?? "" }
79
+ });
80
+ observability.observeHistogram({
81
+ name: "opengeni_machine_op_duration_seconds",
82
+ help: "Connected Machine control-op duration in seconds by op.",
83
+ labels: { backend, op },
84
+ value: durationSeconds
85
+ });
86
+ if (healed) {
87
+ observability.incrementCounter({
88
+ name: "opengeni_machine_op_healed_total",
89
+ help: "Connected Machine ops that succeeded only after \u22651 in-call retry.",
90
+ labels: { backend, op }
91
+ });
92
+ }
93
+ if (replyBytes !== void 0) {
94
+ observability.observeHistogram({
95
+ name: "opengeni_machine_op_reply_bytes",
96
+ help: "Connected Machine control-op reply size in bytes (payload-wall indicator).",
97
+ labels: { backend, op },
98
+ value: replyBytes
99
+ });
100
+ }
101
+ }
102
+ };
103
+ }
104
+ function selfhostedOpObserverForMetrics(hooks) {
105
+ return (o) => {
106
+ hooks.onSandboxOp?.({
107
+ backend: "selfhosted",
108
+ op: o.op,
109
+ outcome: o.outcome,
110
+ healed: o.healed,
111
+ retries: o.retries,
112
+ durationSeconds: o.durationMs / 1e3,
113
+ ...o.code !== void 0 ? { code: errorCodeToJSON(o.code) } : {},
114
+ ...o.replyBytes !== void 0 ? { replyBytes: o.replyBytes } : {}
115
+ });
116
+ };
117
+ }
118
+ function machineOpSessionEventFor(o) {
119
+ if (o.outcome === "failed" && o.faultClass && SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES.has(o.faultClass)) {
120
+ return {
121
+ type: "machine.op.failed",
122
+ payload: {
123
+ op: o.op,
124
+ faultClass: o.faultClass,
125
+ attempts: o.retries,
126
+ ...o.machineId ? { machineId: o.machineId } : {}
127
+ }
128
+ };
129
+ }
130
+ if (o.outcome === "ok" && o.healed) {
131
+ return {
132
+ type: "machine.op.recovered",
133
+ payload: {
134
+ op: o.op,
135
+ faultClass: o.faultClass ?? "unknown",
136
+ attempts: o.retries,
137
+ ...o.machineId ? { machineId: o.machineId } : {}
138
+ }
139
+ };
140
+ }
141
+ return null;
142
+ }
143
+ function makeMachineOpObserver(hooks) {
144
+ const meter = selfhostedOpObserverForMetrics(hooks);
145
+ const buffered = [];
146
+ return {
147
+ observer: (o) => {
148
+ meter(o);
149
+ const event = machineOpSessionEventFor(o);
150
+ if (event) {
151
+ buffered.push(event);
152
+ }
153
+ },
154
+ drainEvents: () => buffered.splice(0, buffered.length)
155
+ };
156
+ }
157
+ function turnLifecycleMetricsFor(observability) {
158
+ const existing = turnTrackers.get(observability);
159
+ if (existing) {
160
+ return existing;
161
+ }
162
+ const tracker = new TurnLifecycleMetrics(observability);
163
+ turnTrackers.set(observability, tracker);
164
+ return tracker;
165
+ }
166
+ var TurnLifecycleMetrics = class {
167
+ constructor(observability, options = {}) {
168
+ this.observability = observability;
169
+ this.options = options;
170
+ }
171
+ startedTurns = /* @__PURE__ */ new Map();
172
+ timer = null;
173
+ start(turnId) {
174
+ this.startedTurns.set(turnId, this.now());
175
+ this.ensureTimer();
176
+ this.refreshGauges();
177
+ }
178
+ finish(turnId, outcome, durationSeconds) {
179
+ const startedAt = this.startedTurns.get(turnId);
180
+ if (startedAt !== void 0) {
181
+ this.startedTurns.delete(turnId);
182
+ }
183
+ if (outcome) {
184
+ const observedDuration = durationSeconds ?? (startedAt === void 0 ? 0 : Math.max(0, (this.now() - startedAt) / 1e3));
185
+ this.observability.incrementCounter({
186
+ name: "opengeni_turns_total",
187
+ help: "Total agent turns by terminal outcome.",
188
+ labels: { outcome }
189
+ });
190
+ this.observability.observeHistogram({
191
+ name: "opengeni_turn_duration_seconds",
192
+ help: "Agent turn duration in seconds by terminal outcome.",
193
+ labels: { outcome },
194
+ value: observedDuration
195
+ });
196
+ }
197
+ this.refreshGauges();
198
+ if (this.startedTurns.size === 0) {
199
+ this.stopTimer();
200
+ }
201
+ }
202
+ refreshGauges() {
203
+ this.observability.setGauge({
204
+ name: "opengeni_turns_inflight",
205
+ help: "Current number of in-flight agent turns in this worker process.",
206
+ value: this.startedTurns.size
207
+ });
208
+ this.observability.setGauge({
209
+ name: "opengeni_turn_oldest_inflight_age_seconds",
210
+ help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
211
+ value: this.oldestInflightAgeSeconds()
212
+ });
213
+ }
214
+ stop() {
215
+ this.startedTurns.clear();
216
+ this.refreshGauges();
217
+ this.stopTimer();
218
+ }
219
+ oldestInflightAgeSeconds() {
220
+ if (this.startedTurns.size === 0) {
221
+ return 0;
222
+ }
223
+ let oldest = Number.POSITIVE_INFINITY;
224
+ for (const startedAt of this.startedTurns.values()) {
225
+ oldest = Math.min(oldest, startedAt);
226
+ }
227
+ return Math.max(0, (this.now() - oldest) / 1e3);
228
+ }
229
+ ensureTimer() {
230
+ if (this.timer) {
231
+ return;
232
+ }
233
+ this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15e3);
234
+ this.timer.unref?.();
235
+ }
236
+ stopTimer() {
237
+ if (!this.timer) {
238
+ return;
239
+ }
240
+ clearInterval(this.timer);
241
+ this.timer = null;
242
+ }
243
+ now() {
244
+ return this.options.now?.() ?? Date.now();
245
+ }
246
+ };
247
+ function recordTurnsQueuedGauge(observability, value) {
248
+ observability.setGauge({
249
+ name: "opengeni_turns_queued",
250
+ help: "Current number of queued session turns.",
251
+ value
252
+ });
253
+ }
254
+ function recordTurnTaskQueueStats(observability, stats) {
255
+ observability.setGauge({
256
+ name: "opengeni_turn_eligible_backlog",
257
+ help: "Temporal runAgentTurn activity tasks eligible for immediate worker admission.",
258
+ value: nonnegativeFinite(stats.eligibleBacklog)
259
+ });
260
+ observability.setGauge({
261
+ name: "opengeni_turn_eligible_backlog_oldest_age_seconds",
262
+ help: "Approximate age of the oldest eligible runAgentTurn activity task.",
263
+ value: nonnegativeFinite(stats.oldestBacklogAgeSeconds)
264
+ });
265
+ observability.setGauge({
266
+ name: "opengeni_turn_eligible_tasks_add_rate",
267
+ help: "Temporal runAgentTurn tasks added per second over its rolling window.",
268
+ value: nonnegativeFinite(stats.tasksAddRate)
269
+ });
270
+ observability.setGauge({
271
+ name: "opengeni_turn_eligible_tasks_dispatch_rate",
272
+ help: "Temporal runAgentTurn tasks dispatched per second over its rolling window.",
273
+ value: nonnegativeFinite(stats.tasksDispatchRate)
274
+ });
275
+ }
276
+ function normalizeTurnTaskQueueStats(stats) {
277
+ if (!stats) {
278
+ throw new Error("Temporal DescribeTaskQueue response omitted required stats");
279
+ }
280
+ const eligibleBacklog = requiredTemporalNumber(
281
+ stats.approximateBacklogCount,
282
+ "approximateBacklogCount",
283
+ { integer: true }
284
+ );
285
+ let oldestBacklogAgeSeconds = 0;
286
+ if (stats.approximateBacklogAge) {
287
+ const seconds = optionalTemporalNumber(
288
+ stats.approximateBacklogAge.seconds,
289
+ "approximateBacklogAge.seconds",
290
+ { integer: true }
291
+ );
292
+ const nanos = optionalTemporalNumber(
293
+ stats.approximateBacklogAge.nanos,
294
+ "approximateBacklogAge.nanos",
295
+ { integer: true }
296
+ );
297
+ if (nanos >= 1e9) {
298
+ throw new Error("Temporal approximateBacklogAge.nanos must be less than one second");
299
+ }
300
+ oldestBacklogAgeSeconds = seconds + nanos / 1e9;
301
+ } else if (eligibleBacklog > 0) {
302
+ throw new Error("Temporal stats omitted approximateBacklogAge for a nonzero backlog");
303
+ }
304
+ return {
305
+ eligibleBacklog,
306
+ oldestBacklogAgeSeconds,
307
+ tasksAddRate: optionalTemporalNumber(stats.tasksAddRate, "tasksAddRate"),
308
+ tasksDispatchRate: optionalTemporalNumber(stats.tasksDispatchRate, "tasksDispatchRate")
309
+ };
310
+ }
311
+ function startTurnCapacityMonitor(input) {
312
+ const intervalMs = input.intervalMs ?? 15e3;
313
+ const now = input.now ?? Date.now;
314
+ const startedAt = now();
315
+ let lastSuccessAt = null;
316
+ let lastReadSucceeded = false;
317
+ let stopped = false;
318
+ let running = null;
319
+ const recordStatus = () => {
320
+ const observedAt = now();
321
+ const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
322
+ const set = (name, help, value) => input.observability.setGauge({ name, help, value });
323
+ set(
324
+ "opengeni_turn_capacity_monitor_last_read_success",
325
+ "Whether the latest Temporal turn-queue capacity read completed successfully.",
326
+ lastReadSucceeded ? 1 : 0
327
+ );
328
+ set(
329
+ "opengeni_turn_capacity_monitor_last_success_timestamp_seconds",
330
+ "Unix timestamp of the latest successful Temporal turn-queue capacity read, or zero before one succeeds.",
331
+ lastSuccessAt === null ? 0 : lastSuccessAt / 1e3
332
+ );
333
+ set(
334
+ "opengeni_turn_capacity_monitor_last_success_age_seconds",
335
+ "Age of the latest successful Temporal turn-queue capacity read, or monitor age before one succeeds.",
336
+ Math.max(0, successAgeMs / 1e3)
337
+ );
338
+ set(
339
+ "opengeni_turn_capacity_monitor_fresh",
340
+ "Whether eligible-backlog gauges have a successful Temporal read within three monitor intervals.",
341
+ lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0
342
+ );
343
+ };
344
+ const refresh = () => {
345
+ recordStatus();
346
+ if (stopped || running) return;
347
+ running = input.read().then((stats) => {
348
+ recordTurnTaskQueueStats(input.observability, stats);
349
+ lastSuccessAt = now();
350
+ lastReadSucceeded = true;
351
+ recordStatus();
352
+ }).catch((error) => {
353
+ lastReadSucceeded = false;
354
+ recordStatus();
355
+ input.observability.warn("turn capacity monitor: Temporal task-queue stats failed", {
356
+ error: error instanceof Error ? error.message : String(error)
357
+ });
358
+ }).finally(() => {
359
+ running = null;
360
+ });
361
+ };
362
+ refresh();
363
+ const timer = setInterval(refresh, intervalMs);
364
+ timer.unref?.();
365
+ return {
366
+ close: async () => {
367
+ stopped = true;
368
+ clearInterval(timer);
369
+ await running;
370
+ }
371
+ };
372
+ }
373
+ function recordSandboxLeaseGauges(observability, counts) {
374
+ for (const liveness of ["cold", "warming", "warm", "draining"]) {
375
+ observability.setGauge({
376
+ name: "opengeni_sandbox_leases",
377
+ help: "Current sandbox leases by liveness state.",
378
+ labels: { liveness },
379
+ value: counts[liveness] ?? 0
380
+ });
381
+ }
382
+ }
383
+ function recordSandboxInventoryProjectionSuccess(observability, domain, timestampSeconds = Date.now() / 1e3) {
384
+ observability.setGauge({
385
+ name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
386
+ help: "Unix timestamp of the last complete successful sandbox inventory projection by domain.",
387
+ labels: { domain },
388
+ value: timestampSeconds
389
+ });
390
+ }
391
+ function recordSandboxInventoryProjectionFailure(observability, domain) {
392
+ observability.incrementCounter({
393
+ name: "opengeni_sandbox_inventory_refresh_failures_total",
394
+ help: "Failed sandbox inventory projection refreshes by domain.",
395
+ labels: { domain }
396
+ });
397
+ }
398
+ function recordCreditBalanceGauges(observability, balances) {
399
+ const previous = creditBalanceGaugeAccounts.get(observability) ?? /* @__PURE__ */ new Set();
400
+ const current = /* @__PURE__ */ new Set();
401
+ for (const balance of balances) {
402
+ current.add(balance.accountId);
403
+ observability.setGauge({
404
+ name: "opengeni_credit_balance_micros",
405
+ help: "Current credit balance in micros by account.",
406
+ labels: { account_id: balance.accountId },
407
+ value: balance.balanceMicros
408
+ });
409
+ }
410
+ for (const accountId of previous) {
411
+ if (!current.has(accountId)) {
412
+ observability.setGauge({
413
+ name: "opengeni_credit_balance_micros",
414
+ help: "Current credit balance in micros by account.",
415
+ labels: { account_id: accountId },
416
+ value: 0
417
+ });
418
+ }
419
+ }
420
+ creditBalanceGaugeAccounts.set(observability, current);
421
+ }
422
+ function recordSandboxOrphansTerminated(observability, count) {
423
+ if (count <= 0) {
424
+ return;
425
+ }
426
+ observability.incrementCounter({
427
+ name: "opengeni_sandbox_orphans_terminated_total",
428
+ help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
429
+ amount: count
430
+ });
431
+ }
432
+ function recordSandboxCheckpointArtifactGauges(observability, counts) {
433
+ for (const state of [
434
+ "candidate",
435
+ "current",
436
+ "previous",
437
+ "delete_pending",
438
+ "deleting",
439
+ "delete_failed",
440
+ "deleted"
441
+ ]) {
442
+ observability.setGauge({
443
+ name: "opengeni_sandbox_checkpoint_artifacts",
444
+ help: "Current durable provider checkpoint artifacts by lifecycle state.",
445
+ labels: { state },
446
+ value: counts[state]
447
+ });
448
+ }
449
+ }
450
+ function recordSandboxCheckpointArtifactOutcome(observability, outcome, count = 1) {
451
+ if (count <= 0) return;
452
+ observability.incrementCounter({
453
+ name: "opengeni_sandbox_checkpoint_artifact_operations_total",
454
+ help: "Durable provider checkpoint artifact operations by fixed outcome.",
455
+ labels: { outcome },
456
+ amount: count
457
+ });
458
+ }
459
+ function recordSandboxDeadlineRotationsRequested(observability, count) {
460
+ if (count <= 0) return;
461
+ observability.incrementCounter({
462
+ name: "opengeni_sandbox_deadline_rotations_requested_total",
463
+ help: "Total finite-lifetime sandbox rotations requested before provider deadline.",
464
+ amount: count
465
+ });
466
+ }
467
+ function recordSandboxRotationBacklogGauges(observability, backlog) {
468
+ const values = {
469
+ requested: backlog.requested,
470
+ overdue: backlog.overdue,
471
+ turn_blocked: backlog.turnBlocked,
472
+ direct_blocked: backlog.directBlocked,
473
+ process_blocked: backlog.processBlocked
474
+ };
475
+ for (const [kind, value] of Object.entries(values)) {
476
+ observability.setGauge({
477
+ name: "opengeni_sandbox_rotation_backlog",
478
+ help: "Current finite-lifetime sandbox rotation backlog by fixed condition.",
479
+ labels: { kind },
480
+ value
481
+ });
482
+ }
483
+ }
484
+ var RETAINED_PROCESS_OWNER_STATES = [
485
+ "direct",
486
+ "queued",
487
+ "running",
488
+ "requires_action",
489
+ "recovering",
490
+ "waiting_capacity",
491
+ "completed",
492
+ "failed",
493
+ "cancelled",
494
+ "superseded",
495
+ "withdrawn_for_edit",
496
+ "missing",
497
+ "unknown"
498
+ ];
499
+ var RETAINED_PROCESS_OWNER_STATE_SET = new Set(RETAINED_PROCESS_OWNER_STATES);
500
+ function recordRetainedProcessInventoryGauges(observability, counts) {
501
+ const normalized = /* @__PURE__ */ new Map();
502
+ for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {
503
+ normalized.set(ownerState, { active: 0, terminal: 0 });
504
+ }
505
+ for (const count of counts) {
506
+ const ownerState = RETAINED_PROCESS_OWNER_STATE_SET.has(count.ownerState) ? count.ownerState : "unknown";
507
+ const current = normalized.get(ownerState);
508
+ current.active += Math.max(0, count.activeCount);
509
+ current.terminal += Math.max(0, count.terminalOwnerCount);
510
+ }
511
+ for (const [ownerState, count] of normalized) {
512
+ observability.setGauge({
513
+ name: "opengeni_retained_processes_active",
514
+ help: "Current active retained provider processes by durable owner state.",
515
+ labels: { owner_state: ownerState },
516
+ value: count.active
517
+ });
518
+ observability.setGauge({
519
+ name: "opengeni_retained_processes_terminal_owner_backlog",
520
+ help: "Current active retained processes whose exact owner attempt is terminal.",
521
+ labels: { owner_state: ownerState },
522
+ value: count.terminal
523
+ });
524
+ }
525
+ }
526
+ var EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, "unknown"];
527
+ var EXPIRED_DRAINING_BACKEND_SET = new Set(EXPIRED_DRAINING_BACKENDS);
528
+ var EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
529
+ function recordExpiredDrainingSandboxLeaseGauges(observability, counts) {
530
+ const normalized = /* @__PURE__ */ new Map();
531
+ for (const backend of EXPIRED_DRAINING_BACKENDS) {
532
+ for (const ageBucket of EXPIRED_DRAINING_AGE_BUCKETS) {
533
+ normalized.set(`${backend}:${ageBucket}`, 0);
534
+ }
535
+ }
536
+ for (const count of counts) {
537
+ const backend = EXPIRED_DRAINING_BACKEND_SET.has(count.backend) ? count.backend : "unknown";
538
+ if (!EXPIRED_DRAINING_AGE_BUCKETS.includes(count.ageBucket)) continue;
539
+ const key = `${backend}:${count.ageBucket}`;
540
+ normalized.set(key, (normalized.get(key) ?? 0) + Math.max(0, count.count));
541
+ }
542
+ for (const [key, value] of normalized) {
543
+ const [backend, ageBucket] = key.split(":");
544
+ observability.setGauge({
545
+ name: "opengeni_sandbox_leases_expired_draining",
546
+ help: "Current expired draining sandbox leases by backend and fixed age bucket.",
547
+ labels: { backend, age_bucket: ageBucket },
548
+ value
549
+ });
550
+ }
551
+ }
552
+ function recordRetainedProcessReconciliation(observability, outcome) {
553
+ observability.incrementCounter({
554
+ name: "opengeni_retained_process_reconciliation_total",
555
+ help: "Bounded retained-process reconciliation observations by fixed app outcome.",
556
+ labels: { outcome }
557
+ });
558
+ }
559
+ function recordCreditMicros(observability, kind, amountMicros) {
560
+ if (!observability || amountMicros <= 0) {
561
+ return;
562
+ }
563
+ observability.incrementCounter({
564
+ name: "opengeni_credit_micros_total",
565
+ help: "Total credit micros recorded by kind.",
566
+ labels: { kind },
567
+ amount: amountMicros
568
+ });
569
+ }
570
+ function contentDeltaClass(type) {
571
+ if (type === "agent.message.delta") {
572
+ return "message";
573
+ }
574
+ if (type === "agent.reasoning.delta") {
575
+ return "reasoning";
576
+ }
577
+ return null;
578
+ }
579
+ var STREAM_TTFT_BUCKETS = [0.02, 0.05, 0.1, 0.2, 0.35, 0.5, 0.75, 1, 1.5, 2, 3, 5, 10];
580
+ var STREAM_INTER_DELTA_BUCKETS = [5e-3, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5];
581
+ var StreamTimingMetrics = class {
582
+ constructor(observability, options) {
583
+ this.observability = observability;
584
+ this.options = options;
585
+ this.now = options.now ?? (() => performance.now());
586
+ this.ttftAnchor = this.now();
587
+ }
588
+ now;
589
+ ttftAnchor;
590
+ ttftArmed = true;
591
+ lastDeltaAt = /* @__PURE__ */ new Map();
592
+ onEvent(type) {
593
+ const deltaClass = contentDeltaClass(type);
594
+ if (deltaClass === null) {
595
+ this.ttftAnchor = this.now();
596
+ this.ttftArmed = true;
597
+ this.lastDeltaAt.clear();
598
+ return;
599
+ }
600
+ const at = this.now();
601
+ if (this.ttftArmed) {
602
+ this.observability.observeHistogram({
603
+ name: "opengeni_stream_ttft_seconds",
604
+ help: "Seconds from a model (re)start to its first streamed content delta.",
605
+ buckets: STREAM_TTFT_BUCKETS,
606
+ labels: { provider: this.options.provider },
607
+ value: Math.max(0, (at - this.ttftAnchor) / 1e3)
608
+ });
609
+ this.ttftArmed = false;
610
+ }
611
+ const last = this.lastDeltaAt.get(deltaClass);
612
+ if (last !== void 0) {
613
+ this.observability.observeHistogram({
614
+ name: "opengeni_stream_inter_delta_gap_seconds",
615
+ help: "Seconds between consecutive streamed content deltas of the same class.",
616
+ buckets: STREAM_INTER_DELTA_BUCKETS,
617
+ labels: { provider: this.options.provider, class: deltaClass },
618
+ value: Math.max(0, (at - last) / 1e3)
619
+ });
620
+ }
621
+ this.lastDeltaAt.set(deltaClass, at);
622
+ }
623
+ };
624
+ var STREAM_BATCH_SIZE_BUCKETS = [1, 2, 5, 10, 20, 50, 100, 200, 500];
625
+ var STREAM_IO_BUCKETS = [1e-3, 25e-4, 5e-3, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5];
626
+ function recordBatchFlush(observability, input) {
627
+ observability.observeHistogram({
628
+ name: "opengeni_stream_batch_flush_events",
629
+ help: "Session events coalesced into one streaming batcher flush.",
630
+ buckets: STREAM_BATCH_SIZE_BUCKETS,
631
+ value: input.events
632
+ });
633
+ observability.observeHistogram({
634
+ name: "opengeni_stream_batch_flush_duration_seconds",
635
+ help: "Duration in seconds of one streaming batcher flush (append + publish).",
636
+ buckets: STREAM_IO_BUCKETS,
637
+ value: input.durationSeconds
638
+ });
639
+ }
640
+ function recordSessionEventAppendLatency(observability, input) {
641
+ observability.observeHistogram({
642
+ name: "opengeni_session_event_append_seconds",
643
+ help: "Duration in seconds of an appendSessionEvents DB write (the durable write path).",
644
+ buckets: STREAM_IO_BUCKETS,
645
+ value: input.durationSeconds
646
+ });
647
+ }
648
+ function recordSessionEventPublishLatency(observability, input) {
649
+ observability.observeHistogram({
650
+ name: "opengeni_session_event_publish_seconds",
651
+ help: "Duration in seconds of the best-effort NATS live fan-out publish (the delivery path).",
652
+ buckets: STREAM_IO_BUCKETS,
653
+ value: input.durationSeconds
654
+ });
655
+ }
656
+ var MODEL_INPUT_TOKENS_BUCKETS = [
657
+ 1e3,
658
+ 5e3,
659
+ 1e4,
660
+ 25e3,
661
+ 5e4,
662
+ 1e5,
663
+ 15e4,
664
+ 2e5,
665
+ 3e5,
666
+ 5e5,
667
+ 1e6
668
+ ];
669
+ function recordModelInputTokens(observability, provider, inputTokens) {
670
+ const normalizedInputTokens = modelUsageTokenCountOrNull(inputTokens);
671
+ if (normalizedInputTokens === null || normalizedInputTokens === 0) {
672
+ return;
673
+ }
674
+ observability.observeHistogram({
675
+ name: "opengeni_model_input_tokens",
676
+ help: "Observed input (context) tokens per model response, by provider.",
677
+ buckets: MODEL_INPUT_TOKENS_BUCKETS,
678
+ labels: { provider },
679
+ value: normalizedInputTokens
680
+ });
681
+ }
682
+ function recordContextCompaction(observability, trigger) {
683
+ observability.incrementCounter({
684
+ name: "opengeni_context_compactions_total",
685
+ help: "Total context compactions performed, by trigger.",
686
+ labels: { trigger }
687
+ });
688
+ }
689
+ var MODEL_CACHE_HIT_RATIO_BUCKETS = [
690
+ 0.05,
691
+ 0.1,
692
+ 0.2,
693
+ 0.3,
694
+ 0.4,
695
+ 0.5,
696
+ 0.6,
697
+ 0.7,
698
+ 0.8,
699
+ 0.9,
700
+ 0.95,
701
+ 0.99,
702
+ 1
703
+ ];
704
+ function recordModelCacheTokens(observability, provider, input) {
705
+ const cached = modelUsageTokenCountOrNull(input.cachedTokens);
706
+ const cacheWrite = modelUsageTokenCountOrNull(input.cacheWriteTokens);
707
+ const prompt = modelUsageTokenCountOrNull(input.promptTokens);
708
+ incrementBoundedModelCacheCounter(observability, {
709
+ name: "opengeni_model_cache_read_telemetry_total",
710
+ help: "Authoritative model calls with reported or missing cache-read telemetry, by provider.",
711
+ provider,
712
+ labels: { provider, status: cached === null ? "missing" : "reported" },
713
+ amount: 1
714
+ });
715
+ if (cached !== null && cached > 0) {
716
+ incrementBoundedModelCacheCounter(observability, {
717
+ name: "opengeni_model_cached_tokens_total",
718
+ help: "Total prompt tokens served from the provider's prompt cache, by provider.",
719
+ provider,
720
+ labels: { provider },
721
+ amount: cached
722
+ });
723
+ }
724
+ if (cacheWrite !== null && cacheWrite > 0) {
725
+ incrementBoundedModelCacheCounter(observability, {
726
+ name: "opengeni_model_cache_write_tokens_total",
727
+ help: "Total prompt tokens written to the provider's prompt cache, by provider.",
728
+ provider,
729
+ labels: { provider },
730
+ amount: cacheWrite
731
+ });
732
+ }
733
+ if (cached !== null && prompt !== null && prompt > 0) {
734
+ observability.observeHistogram({
735
+ name: "opengeni_model_cache_hit_ratio",
736
+ help: "Per-call prompt-cache hit ratio (cached/prompt tokens) by provider.",
737
+ buckets: MODEL_CACHE_HIT_RATIO_BUCKETS,
738
+ labels: { provider },
739
+ // Clamp to [0, 1]: a provider that (rarely) reports cached >= prompt must
740
+ // not skew the histogram past 1.0.
741
+ value: Math.min(1, cached / prompt)
742
+ });
743
+ }
744
+ }
745
+ function incrementBoundedModelCacheCounter(observability, input) {
746
+ const totals = modelCacheCounterTotals.get(observability) ?? /* @__PURE__ */ new Map();
747
+ if (!modelCacheCounterTotals.has(observability)) {
748
+ modelCacheCounterTotals.set(observability, totals);
749
+ }
750
+ const labelKey = Object.entries(input.labels).sort(([left], [right]) => left.localeCompare(right)).map(([key2, value]) => `${key2}=${value}`).join(",");
751
+ const key = `${input.name}:${labelKey}`;
752
+ const current = totals.get(key) ?? 0;
753
+ if (input.amount > MAX_MODEL_CACHE_COUNTER_TOTAL - current) {
754
+ observability.warn("model cache metric cumulative limit reached", {
755
+ provider: input.provider,
756
+ metric: input.name
757
+ });
758
+ return;
759
+ }
760
+ observability.incrementCounter({
761
+ name: input.name,
762
+ help: input.help,
763
+ labels: input.labels,
764
+ amount: input.amount
765
+ });
766
+ totals.set(key, current + input.amount);
767
+ }
768
+ function nonnegativeFinite(value) {
769
+ return Number.isFinite(value) && value > 0 ? value : 0;
770
+ }
771
+ function requiredTemporalNumber(value, field, options = {}) {
772
+ if (value === null || value === void 0) {
773
+ throw new Error(`Temporal stats omitted required ${field}`);
774
+ }
775
+ const parsed = Number(value);
776
+ if (!Number.isFinite(parsed) || parsed < 0 || options.integer && !Number.isSafeInteger(parsed)) {
777
+ throw new Error(`Temporal stats returned invalid ${field}: ${String(value)}`);
778
+ }
779
+ return parsed;
780
+ }
781
+ function optionalTemporalNumber(value, field, options = {}) {
782
+ return value === null || value === void 0 ? 0 : requiredTemporalNumber(value, field, options);
783
+ }
784
+ function stableAccountHash(credentialId) {
785
+ if (!credentialId) {
786
+ return "none";
787
+ }
788
+ return createHash("sha256").update(credentialId).digest("hex").slice(0, 12);
789
+ }
790
+ function modelCallAccountContext(input) {
791
+ const accountChangedFromPrevCall = input.isFirstCallOfTurn && input.servingCredentialId !== null && input.priorSessionCredentialId !== null && input.priorSessionCredentialId !== input.servingCredentialId;
792
+ return {
793
+ servingAccountHash: stableAccountHash(input.servingCredentialId),
794
+ accountChangedFromPrevCall
795
+ };
796
+ }
797
+
798
+ export {
799
+ observabilityEventLogger,
800
+ runtimeMetricsHooksForObservability,
801
+ makeMachineOpObserver,
802
+ turnLifecycleMetricsFor,
803
+ recordTurnsQueuedGauge,
804
+ normalizeTurnTaskQueueStats,
805
+ startTurnCapacityMonitor,
806
+ recordSandboxLeaseGauges,
807
+ recordSandboxInventoryProjectionSuccess,
808
+ recordSandboxInventoryProjectionFailure,
809
+ recordCreditBalanceGauges,
810
+ recordSandboxOrphansTerminated,
811
+ recordSandboxCheckpointArtifactGauges,
812
+ recordSandboxCheckpointArtifactOutcome,
813
+ recordSandboxDeadlineRotationsRequested,
814
+ recordSandboxRotationBacklogGauges,
815
+ recordRetainedProcessInventoryGauges,
816
+ recordExpiredDrainingSandboxLeaseGauges,
817
+ recordRetainedProcessReconciliation,
818
+ recordCreditMicros,
819
+ StreamTimingMetrics,
820
+ recordBatchFlush,
821
+ recordSessionEventAppendLatency,
822
+ recordSessionEventPublishLatency,
823
+ recordModelInputTokens,
824
+ recordContextCompaction,
825
+ recordModelCacheTokens,
826
+ modelCallAccountContext
827
+ };
828
+ //# sourceMappingURL=chunk-E3RI3I33.js.map