@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +66 -0
- package/dist/activities/image-generation-references.d.ts +2 -0
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
- package/dist/activities/knowledge-source-sync.d.ts +33 -1
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +16 -7
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12575
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/documents.ts +6 -4
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +938 -0
- package/src/activities/image-generation-references.ts +11 -2
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync-driver.ts +23 -0
- package/src/activities/knowledge-source-sync.ts +556 -40
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +128 -136
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-SMKF2OBE.js.map +0 -1
- package/dist/activities-turn-IDS3WEHA.js +0 -16532
- package/dist/activities-turn-IDS3WEHA.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -2,11 +2,228 @@
|
|
|
2
2
|
import { createHash } from "crypto";
|
|
3
3
|
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
4
4
|
import { SandboxBackend } from "@opengeni/contracts";
|
|
5
|
+
|
|
6
|
+
// src/opensandbox-kubernetes-inventory.ts
|
|
7
|
+
import { readFile } from "fs/promises";
|
|
8
|
+
import { request as httpsRequest } from "https";
|
|
9
|
+
var KUBERNETES_API_URL = "https://kubernetes.default.svc";
|
|
10
|
+
var KUBERNETES_TOKEN_FILE = "/var/run/secrets/opengeni.io/opensandbox-kubernetes-inventory/token";
|
|
11
|
+
var KUBERNETES_CA_FILE = "/var/run/secrets/opengeni.io/opensandbox-kubernetes-inventory/ca.crt";
|
|
12
|
+
var KUBERNETES_REQUEST_TIMEOUT_MS = 1e4;
|
|
13
|
+
var KUBERNETES_RESPONSE_MAX_BYTES = 64 * 1024 * 1024;
|
|
14
|
+
var KUBERNETES_LIST_PAGE_SIZE = 200;
|
|
15
|
+
var KUBERNETES_LIST_MAX_PAGES = 1e4;
|
|
16
|
+
var OPENSANDBOX_CLEANUP_STUCK_SECONDS = 300;
|
|
17
|
+
var OPENSANDBOX_EXPIRATION_OVERDUE_SECONDS = 180;
|
|
18
|
+
var OPENSANDBOX_BATCHSANDBOX_PHASES = [
|
|
19
|
+
"pending",
|
|
20
|
+
"running",
|
|
21
|
+
"pausing",
|
|
22
|
+
"paused",
|
|
23
|
+
"resuming",
|
|
24
|
+
"failed",
|
|
25
|
+
"unknown"
|
|
26
|
+
];
|
|
27
|
+
var OPENSANDBOX_WORKLOAD_POD_CONDITIONS = [
|
|
28
|
+
"pending",
|
|
29
|
+
"image_pull",
|
|
30
|
+
"unschedulable"
|
|
31
|
+
];
|
|
32
|
+
var IMAGE_PULL_REASONS = /* @__PURE__ */ new Set(["ErrImagePull", "ImagePullBackOff", "InvalidImageName"]);
|
|
33
|
+
async function inspectOpenSandboxKubernetesInventory(input) {
|
|
34
|
+
const namespace = kubernetesNamespace(input.namespace);
|
|
35
|
+
const nowMs = input.nowMs ?? Date.now();
|
|
36
|
+
const requestJson = input.requestJson ?? await authenticatedKubernetesJsonRequest({
|
|
37
|
+
tokenFile: input.tokenFile ?? KUBERNETES_TOKEN_FILE,
|
|
38
|
+
caFile: input.caFile ?? KUBERNETES_CA_FILE
|
|
39
|
+
});
|
|
40
|
+
const apiUrl = new URL(input.apiUrl ?? KUBERNETES_API_URL);
|
|
41
|
+
if (apiUrl.protocol !== "https:") {
|
|
42
|
+
throw new Error("OpenSandbox Kubernetes inventory API must use HTTPS");
|
|
43
|
+
}
|
|
44
|
+
const inventory = {
|
|
45
|
+
batchSandboxPhases: zeroRecord(OPENSANDBOX_BATCHSANDBOX_PHASES),
|
|
46
|
+
workloadPodConditions: zeroRecord(OPENSANDBOX_WORKLOAD_POD_CONDITIONS),
|
|
47
|
+
cleanupStuck: 0,
|
|
48
|
+
expirationOverdue: 0
|
|
49
|
+
};
|
|
50
|
+
await visitKubernetesList({
|
|
51
|
+
apiUrl,
|
|
52
|
+
requestJson,
|
|
53
|
+
path: `/apis/sandbox.opensandbox.io/v1alpha1/namespaces/${encodeURIComponent(namespace)}/batchsandboxes`,
|
|
54
|
+
visit: (item) => observeBatchSandbox(inventory, item, nowMs)
|
|
55
|
+
});
|
|
56
|
+
await visitKubernetesList({
|
|
57
|
+
apiUrl,
|
|
58
|
+
requestJson,
|
|
59
|
+
path: `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods`,
|
|
60
|
+
query: { labelSelector: "opensandbox.io/id" },
|
|
61
|
+
visit: (item) => observeWorkloadPod(inventory, item)
|
|
62
|
+
});
|
|
63
|
+
return inventory;
|
|
64
|
+
}
|
|
65
|
+
function observeBatchSandbox(inventory, item, nowMs) {
|
|
66
|
+
const resource = record(item);
|
|
67
|
+
const metadata = record(resource.metadata);
|
|
68
|
+
const spec = record(resource.spec);
|
|
69
|
+
const status = record(resource.status);
|
|
70
|
+
inventory.batchSandboxPhases[batchSandboxPhase(status.phase)] += 1;
|
|
71
|
+
const deletionTimestampMs = dateMilliseconds(metadata.deletionTimestamp);
|
|
72
|
+
const finalizers = Array.isArray(metadata.finalizers) ? metadata.finalizers : [];
|
|
73
|
+
if (deletionTimestampMs !== null && finalizers.length > 0 && nowMs - deletionTimestampMs >= OPENSANDBOX_CLEANUP_STUCK_SECONDS * 1e3) {
|
|
74
|
+
inventory.cleanupStuck += 1;
|
|
75
|
+
}
|
|
76
|
+
const expirationMs = dateMilliseconds(spec.expireTime);
|
|
77
|
+
if (expirationMs !== null && nowMs - expirationMs >= OPENSANDBOX_EXPIRATION_OVERDUE_SECONDS * 1e3) {
|
|
78
|
+
inventory.expirationOverdue += 1;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
function observeWorkloadPod(inventory, item) {
|
|
82
|
+
const pod = record(item);
|
|
83
|
+
const metadata = record(pod.metadata);
|
|
84
|
+
if (dateMilliseconds(metadata.deletionTimestamp) !== null) return;
|
|
85
|
+
const status = record(pod.status);
|
|
86
|
+
if (status.phase === "Pending") inventory.workloadPodConditions.pending += 1;
|
|
87
|
+
if (podHasImagePullFailure(status)) inventory.workloadPodConditions.image_pull += 1;
|
|
88
|
+
if (podIsUnschedulable(status)) inventory.workloadPodConditions.unschedulable += 1;
|
|
89
|
+
}
|
|
90
|
+
function podHasImagePullFailure(status) {
|
|
91
|
+
for (const key of ["initContainerStatuses", "containerStatuses", "ephemeralContainerStatuses"]) {
|
|
92
|
+
const statuses = Array.isArray(status[key]) ? status[key] : [];
|
|
93
|
+
for (const value of statuses) {
|
|
94
|
+
const waiting = record(record(record(value).state).waiting);
|
|
95
|
+
if (typeof waiting.reason === "string" && IMAGE_PULL_REASONS.has(waiting.reason)) return true;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
function podIsUnschedulable(status) {
|
|
101
|
+
const conditions = Array.isArray(status.conditions) ? status.conditions : [];
|
|
102
|
+
return conditions.some((value) => {
|
|
103
|
+
const condition = record(value);
|
|
104
|
+
return condition.type === "PodScheduled" && condition.status === "False" && condition.reason === "Unschedulable";
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
function batchSandboxPhase(value) {
|
|
108
|
+
switch (value) {
|
|
109
|
+
case "Pending":
|
|
110
|
+
return "pending";
|
|
111
|
+
case "Succeed":
|
|
112
|
+
case "Running":
|
|
113
|
+
return "running";
|
|
114
|
+
case "Pausing":
|
|
115
|
+
return "pausing";
|
|
116
|
+
case "Paused":
|
|
117
|
+
return "paused";
|
|
118
|
+
case "Resuming":
|
|
119
|
+
return "resuming";
|
|
120
|
+
case "Failed":
|
|
121
|
+
return "failed";
|
|
122
|
+
default:
|
|
123
|
+
return "unknown";
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
async function visitKubernetesList(input) {
|
|
127
|
+
let continuation = "";
|
|
128
|
+
for (let page = 0; page < KUBERNETES_LIST_MAX_PAGES; page += 1) {
|
|
129
|
+
const url = new URL(input.path, input.apiUrl);
|
|
130
|
+
url.searchParams.set("limit", String(KUBERNETES_LIST_PAGE_SIZE));
|
|
131
|
+
for (const [key, value] of Object.entries(input.query ?? {})) {
|
|
132
|
+
url.searchParams.set(key, value);
|
|
133
|
+
}
|
|
134
|
+
if (continuation) url.searchParams.set("continue", continuation);
|
|
135
|
+
const payload = record(await input.requestJson(url));
|
|
136
|
+
if (!Array.isArray(payload.items)) {
|
|
137
|
+
throw new Error(`Kubernetes list ${input.path} returned no items array`);
|
|
138
|
+
}
|
|
139
|
+
for (const item of payload.items) input.visit(item);
|
|
140
|
+
const next = record(payload.metadata).continue;
|
|
141
|
+
if (typeof next !== "string" || next.length === 0) return;
|
|
142
|
+
if (next === continuation) {
|
|
143
|
+
throw new Error(`Kubernetes list ${input.path} repeated its continuation token`);
|
|
144
|
+
}
|
|
145
|
+
continuation = next;
|
|
146
|
+
}
|
|
147
|
+
throw new Error(`Kubernetes list ${input.path} exceeded its page safety bound`);
|
|
148
|
+
}
|
|
149
|
+
async function authenticatedKubernetesJsonRequest(input) {
|
|
150
|
+
const [tokenText, ca] = await Promise.all([
|
|
151
|
+
readFile(input.tokenFile, "utf8"),
|
|
152
|
+
readFile(input.caFile)
|
|
153
|
+
]);
|
|
154
|
+
const token = tokenText.trim();
|
|
155
|
+
if (!token) throw new Error("OpenSandbox Kubernetes inventory token is empty");
|
|
156
|
+
return async (url) => await new Promise((resolve, reject) => {
|
|
157
|
+
const request = httpsRequest(
|
|
158
|
+
url,
|
|
159
|
+
{
|
|
160
|
+
method: "GET",
|
|
161
|
+
ca,
|
|
162
|
+
headers: {
|
|
163
|
+
accept: "application/json",
|
|
164
|
+
authorization: `Bearer ${token}`
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
(response) => {
|
|
168
|
+
const chunks = [];
|
|
169
|
+
let bytes = 0;
|
|
170
|
+
response.on("data", (chunk) => {
|
|
171
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
172
|
+
bytes += buffer.length;
|
|
173
|
+
if (bytes > KUBERNETES_RESPONSE_MAX_BYTES) {
|
|
174
|
+
request.destroy(new Error("Kubernetes inventory response exceeded its byte limit"));
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
chunks.push(buffer);
|
|
178
|
+
});
|
|
179
|
+
response.on("end", () => {
|
|
180
|
+
const statusCode = response.statusCode ?? 0;
|
|
181
|
+
if (statusCode < 200 || statusCode >= 300) {
|
|
182
|
+
reject(new Error(`Kubernetes inventory request failed with HTTP ${statusCode}`));
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
try {
|
|
186
|
+
resolve(JSON.parse(Buffer.concat(chunks).toString("utf8")));
|
|
187
|
+
} catch {
|
|
188
|
+
reject(new Error("Kubernetes inventory response was not valid JSON"));
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
response.on("error", reject);
|
|
192
|
+
}
|
|
193
|
+
);
|
|
194
|
+
request.setTimeout(KUBERNETES_REQUEST_TIMEOUT_MS, () => {
|
|
195
|
+
request.destroy(new Error("Kubernetes inventory request timed out"));
|
|
196
|
+
});
|
|
197
|
+
request.on("error", reject);
|
|
198
|
+
request.end();
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
function kubernetesNamespace(value) {
|
|
202
|
+
const namespace = value.trim();
|
|
203
|
+
if (namespace.length === 0 || namespace.length > 63 || !/^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/u.test(namespace)) {
|
|
204
|
+
throw new Error("OpenSandbox Kubernetes inventory namespace is invalid");
|
|
205
|
+
}
|
|
206
|
+
return namespace;
|
|
207
|
+
}
|
|
208
|
+
function dateMilliseconds(value) {
|
|
209
|
+
if (typeof value !== "string" || value.length === 0) return null;
|
|
210
|
+
const milliseconds = Date.parse(value);
|
|
211
|
+
return Number.isFinite(milliseconds) ? milliseconds : null;
|
|
212
|
+
}
|
|
213
|
+
function record(value) {
|
|
214
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
215
|
+
}
|
|
216
|
+
function zeroRecord(keys) {
|
|
217
|
+
return Object.fromEntries(keys.map((key) => [key, 0]));
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// src/observability-metrics.ts
|
|
5
221
|
import {
|
|
6
222
|
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
|
|
7
223
|
modelUsageTokenCountOrNull
|
|
8
224
|
} from "@opengeni/runtime";
|
|
9
225
|
var turnTrackers = /* @__PURE__ */ new WeakMap();
|
|
226
|
+
var modelRequestTrackers = /* @__PURE__ */ new WeakMap();
|
|
10
227
|
var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
|
|
11
228
|
var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
|
|
12
229
|
var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
|
|
@@ -71,6 +288,47 @@ function runtimeMetricsHooksForObservability(observability) {
|
|
|
71
288
|
labels: { backend, stage }
|
|
72
289
|
});
|
|
73
290
|
},
|
|
291
|
+
onSandboxProviderApiThrottle: ({ backend, operation }) => {
|
|
292
|
+
observability.incrementCounter({
|
|
293
|
+
name: "opengeni_sandbox_provider_api_throttles_total",
|
|
294
|
+
help: "Total sandbox provider API throttle responses.",
|
|
295
|
+
labels: { backend, operation }
|
|
296
|
+
});
|
|
297
|
+
},
|
|
298
|
+
onSandboxTtlRenewal: ({ backend, outcome }) => {
|
|
299
|
+
observability.incrementCounter({
|
|
300
|
+
name: "opengeni_sandbox_ttl_renewals_total",
|
|
301
|
+
help: "Total renewable sandbox provider TTL refresh attempts.",
|
|
302
|
+
labels: { backend, outcome }
|
|
303
|
+
});
|
|
304
|
+
},
|
|
305
|
+
onOpenSandboxSignedEndpoint: ({ outcome, port }) => {
|
|
306
|
+
observability.incrementCounter({
|
|
307
|
+
name: "opengeni_opensandbox_signed_endpoint_total",
|
|
308
|
+
help: "OpenSandbox signed Channel B mint and host-fetch outcomes.",
|
|
309
|
+
labels: { outcome, port: String(port) }
|
|
310
|
+
});
|
|
311
|
+
},
|
|
312
|
+
onWorkspaceArchiveObject: ({ outcome, backend }) => {
|
|
313
|
+
observability.incrementCounter({
|
|
314
|
+
name: "opengeni_workspace_archive_object_total",
|
|
315
|
+
help: "Workspace archive object-storage put/delete outcomes.",
|
|
316
|
+
labels: { outcome, backend }
|
|
317
|
+
});
|
|
318
|
+
},
|
|
319
|
+
onMcpToolCall: ({ outcome, durationSeconds }) => {
|
|
320
|
+
observability.incrementCounter({
|
|
321
|
+
name: "opengeni_mcp_tool_calls_total",
|
|
322
|
+
help: "Total physical MCP tool calls by bounded structural outcome.",
|
|
323
|
+
labels: { outcome }
|
|
324
|
+
});
|
|
325
|
+
observability.observeHistogram({
|
|
326
|
+
name: "opengeni_mcp_tool_call_duration_seconds",
|
|
327
|
+
help: "MCP tool-call duration in seconds by bounded structural outcome.",
|
|
328
|
+
labels: { outcome },
|
|
329
|
+
value: durationSeconds
|
|
330
|
+
});
|
|
331
|
+
},
|
|
74
332
|
onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
|
|
75
333
|
observability.incrementCounter({
|
|
76
334
|
name: "opengeni_machine_op_total",
|
|
@@ -263,6 +521,89 @@ var TurnLifecycleMetrics = class {
|
|
|
263
521
|
return this.options.now?.() ?? Date.now();
|
|
264
522
|
}
|
|
265
523
|
};
|
|
524
|
+
function modelRequestLifecycleMetricsFor(observability) {
|
|
525
|
+
const existing = modelRequestTrackers.get(observability);
|
|
526
|
+
if (existing) return existing;
|
|
527
|
+
const tracker = new ModelRequestLifecycleMetrics(observability);
|
|
528
|
+
modelRequestTrackers.set(observability, tracker);
|
|
529
|
+
return tracker;
|
|
530
|
+
}
|
|
531
|
+
var ModelRequestLifecycleMetrics = class {
|
|
532
|
+
constructor(observability, options = {}) {
|
|
533
|
+
this.observability = observability;
|
|
534
|
+
this.options = options;
|
|
535
|
+
}
|
|
536
|
+
requests = /* @__PURE__ */ new Map();
|
|
537
|
+
providers = /* @__PURE__ */ new Set();
|
|
538
|
+
timer = null;
|
|
539
|
+
start(key, provider) {
|
|
540
|
+
const now = this.now();
|
|
541
|
+
this.providers.add(provider);
|
|
542
|
+
this.requests.set(key, { provider, startedAt: now, lastEventAt: now });
|
|
543
|
+
this.ensureTimer();
|
|
544
|
+
this.refreshGauges();
|
|
545
|
+
}
|
|
546
|
+
event(key, interEventGapMs) {
|
|
547
|
+
const request = this.requests.get(key);
|
|
548
|
+
if (!request) return;
|
|
549
|
+
request.lastEventAt = this.now();
|
|
550
|
+
this.observability.incrementCounter({
|
|
551
|
+
name: "opengeni_model_request_stream_events_total",
|
|
552
|
+
help: "Complete, valid provider SSE data events by provider.",
|
|
553
|
+
labels: { provider: request.provider }
|
|
554
|
+
});
|
|
555
|
+
if (interEventGapMs !== void 0) {
|
|
556
|
+
this.observability.observeHistogram({
|
|
557
|
+
name: "opengeni_model_request_stream_event_gap_seconds",
|
|
558
|
+
help: "Seconds between complete, valid provider SSE data events.",
|
|
559
|
+
buckets: MODEL_REQUEST_PHASE_BUCKETS,
|
|
560
|
+
labels: { provider: request.provider },
|
|
561
|
+
value: Math.max(0, interEventGapMs / 1e3)
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
finish(key) {
|
|
566
|
+
this.requests.delete(key);
|
|
567
|
+
this.refreshGauges();
|
|
568
|
+
if (this.requests.size === 0) this.stopTimer();
|
|
569
|
+
}
|
|
570
|
+
refreshGauges() {
|
|
571
|
+
const now = this.now();
|
|
572
|
+
for (const provider of this.providers) {
|
|
573
|
+
const active = [...this.requests.values()].filter((request) => request.provider === provider);
|
|
574
|
+
this.observability.setGauge({
|
|
575
|
+
name: "opengeni_model_requests_inflight",
|
|
576
|
+
help: "Current in-flight provider model requests in this worker process.",
|
|
577
|
+
labels: { provider },
|
|
578
|
+
value: active.length
|
|
579
|
+
});
|
|
580
|
+
this.observability.setGauge({
|
|
581
|
+
name: "opengeni_model_request_oldest_no_event_age_seconds",
|
|
582
|
+
help: "Seconds since the least recently progressing in-flight provider request received a valid SSE event.",
|
|
583
|
+
labels: { provider },
|
|
584
|
+
value: active.length === 0 ? 0 : Math.max(0, (now - Math.min(...active.map((request) => request.lastEventAt))) / 1e3)
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
stop() {
|
|
589
|
+
this.requests.clear();
|
|
590
|
+
this.refreshGauges();
|
|
591
|
+
this.stopTimer();
|
|
592
|
+
}
|
|
593
|
+
ensureTimer() {
|
|
594
|
+
if (this.timer) return;
|
|
595
|
+
this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15e3);
|
|
596
|
+
this.timer.unref?.();
|
|
597
|
+
}
|
|
598
|
+
stopTimer() {
|
|
599
|
+
if (!this.timer) return;
|
|
600
|
+
clearInterval(this.timer);
|
|
601
|
+
this.timer = null;
|
|
602
|
+
}
|
|
603
|
+
now() {
|
|
604
|
+
return this.options.now?.() ?? Date.now();
|
|
605
|
+
}
|
|
606
|
+
};
|
|
266
607
|
function recordTurnsQueuedGauge(observability, value) {
|
|
267
608
|
observability.setGauge({
|
|
268
609
|
name: "opengeni_turns_queued",
|
|
@@ -399,6 +740,34 @@ function recordSandboxLeaseGauges(observability, counts) {
|
|
|
399
740
|
});
|
|
400
741
|
}
|
|
401
742
|
}
|
|
743
|
+
function recordOpenSandboxKubernetesInventoryGauges(observability, inventory) {
|
|
744
|
+
for (const phase of OPENSANDBOX_BATCHSANDBOX_PHASES) {
|
|
745
|
+
observability.setGauge({
|
|
746
|
+
name: "opengeni_opensandbox_batchsandboxes",
|
|
747
|
+
help: "Current OpenSandbox BatchSandboxes by bounded lifecycle phase.",
|
|
748
|
+
labels: { phase },
|
|
749
|
+
value: inventory.batchSandboxPhases[phase]
|
|
750
|
+
});
|
|
751
|
+
}
|
|
752
|
+
for (const condition of OPENSANDBOX_WORKLOAD_POD_CONDITIONS) {
|
|
753
|
+
observability.setGauge({
|
|
754
|
+
name: "opengeni_opensandbox_workload_pods",
|
|
755
|
+
help: "Current OpenSandbox workload Pods by bounded operational condition.",
|
|
756
|
+
labels: { condition },
|
|
757
|
+
value: inventory.workloadPodConditions[condition]
|
|
758
|
+
});
|
|
759
|
+
}
|
|
760
|
+
observability.setGauge({
|
|
761
|
+
name: "opengeni_opensandbox_cleanup_stuck",
|
|
762
|
+
help: "Current OpenSandbox BatchSandboxes deleting past the cleanup threshold with finalizers.",
|
|
763
|
+
value: inventory.cleanupStuck
|
|
764
|
+
});
|
|
765
|
+
observability.setGauge({
|
|
766
|
+
name: "opengeni_opensandbox_expiration_overdue",
|
|
767
|
+
help: "Current OpenSandbox BatchSandboxes still present past the provider-expiry threshold.",
|
|
768
|
+
value: inventory.expirationOverdue
|
|
769
|
+
});
|
|
770
|
+
}
|
|
402
771
|
function recordSandboxInventoryProjectionSuccess(observability, domain, timestampSeconds = Date.now() / 1e3) {
|
|
403
772
|
observability.setGauge({
|
|
404
773
|
name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
|
|
@@ -635,6 +1004,14 @@ var TURN_STARTUP_PHASE_BUCKETS = [
|
|
|
635
1004
|
60,
|
|
636
1005
|
120
|
|
637
1006
|
];
|
|
1007
|
+
var TURN_STARTUP_MILESTONE_BUCKETS = [
|
|
1008
|
+
...TURN_STARTUP_PHASE_BUCKETS,
|
|
1009
|
+
300,
|
|
1010
|
+
600,
|
|
1011
|
+
1800,
|
|
1012
|
+
3600,
|
|
1013
|
+
7200
|
|
1014
|
+
];
|
|
638
1015
|
function turnStartupCountBucket(count) {
|
|
639
1016
|
if (count === null || !Number.isFinite(count) || count < 0) return "unknown";
|
|
640
1017
|
if (count === 0) return "0";
|
|
@@ -670,6 +1047,76 @@ function recordTurnSandboxEstablishPolicy(observability, input) {
|
|
|
670
1047
|
}
|
|
671
1048
|
});
|
|
672
1049
|
}
|
|
1050
|
+
function recordSandboxProvisionAttempt(observability, input) {
|
|
1051
|
+
const labels = {
|
|
1052
|
+
backend: input.backend,
|
|
1053
|
+
stage: input.stage,
|
|
1054
|
+
category: input.category,
|
|
1055
|
+
outcome: input.outcome
|
|
1056
|
+
};
|
|
1057
|
+
try {
|
|
1058
|
+
observability.incrementCounter({
|
|
1059
|
+
name: "opengeni_sandbox_provision_attempts_total",
|
|
1060
|
+
help: "Internal physical attempts within logical sandbox provisions.",
|
|
1061
|
+
labels
|
|
1062
|
+
});
|
|
1063
|
+
observability.observeHistogram({
|
|
1064
|
+
name: "opengeni_sandbox_provision_attempt_duration_seconds",
|
|
1065
|
+
help: "Internal sandbox provision attempt duration by bounded structural outcome.",
|
|
1066
|
+
buckets: TURN_STARTUP_PHASE_BUCKETS,
|
|
1067
|
+
labels,
|
|
1068
|
+
value: Math.max(0, input.durationSeconds)
|
|
1069
|
+
});
|
|
1070
|
+
} catch {
|
|
1071
|
+
try {
|
|
1072
|
+
observability.incrementCounter({
|
|
1073
|
+
name: "opengeni_observability_observer_errors_total",
|
|
1074
|
+
help: "Observability observer failures isolated from product execution.",
|
|
1075
|
+
labels: { observer: "sandbox_provision" }
|
|
1076
|
+
});
|
|
1077
|
+
} catch {
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function recordSandboxLogicalProvision(observability, input) {
|
|
1082
|
+
const labels = {
|
|
1083
|
+
backend: input.backend,
|
|
1084
|
+
stage: input.stage,
|
|
1085
|
+
category: input.category,
|
|
1086
|
+
outcome: input.outcome,
|
|
1087
|
+
expected: input.expected ? "true" : "false"
|
|
1088
|
+
};
|
|
1089
|
+
try {
|
|
1090
|
+
observability.incrementCounter({
|
|
1091
|
+
name: "opengeni_sandbox_provisions_total",
|
|
1092
|
+
help: "Logical sandbox provisions by terminal outcome and bounded failure taxonomy.",
|
|
1093
|
+
labels
|
|
1094
|
+
});
|
|
1095
|
+
observability.observeHistogram({
|
|
1096
|
+
name: "opengeni_sandbox_provision_duration_seconds",
|
|
1097
|
+
help: "Logical sandbox provision duration including internal safe retries.",
|
|
1098
|
+
buckets: TURN_STARTUP_PHASE_BUCKETS,
|
|
1099
|
+
labels,
|
|
1100
|
+
value: Math.max(0, input.durationSeconds)
|
|
1101
|
+
});
|
|
1102
|
+
observability.observeHistogram({
|
|
1103
|
+
name: "opengeni_sandbox_provision_internal_attempts",
|
|
1104
|
+
help: "Internal physical attempts per terminal logical sandbox provision.",
|
|
1105
|
+
buckets: [1, 2, 3, 4, 5, 8],
|
|
1106
|
+
labels,
|
|
1107
|
+
value: Math.max(1, input.internalAttempts)
|
|
1108
|
+
});
|
|
1109
|
+
} catch {
|
|
1110
|
+
try {
|
|
1111
|
+
observability.incrementCounter({
|
|
1112
|
+
name: "opengeni_observability_observer_errors_total",
|
|
1113
|
+
help: "Observability observer failures isolated from product execution.",
|
|
1114
|
+
labels: { observer: "sandbox_provision" }
|
|
1115
|
+
});
|
|
1116
|
+
} catch {
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
673
1120
|
function recordTurnWorkerPreparationTotal(observability, input) {
|
|
674
1121
|
observability.observeHistogram({
|
|
675
1122
|
name: "opengeni_turn_worker_preparation_duration_seconds",
|
|
@@ -683,6 +1130,20 @@ function recordTurnWorkerPreparationTotal(observability, input) {
|
|
|
683
1130
|
value: Math.max(0, input.durationSeconds)
|
|
684
1131
|
});
|
|
685
1132
|
}
|
|
1133
|
+
function recordTurnStartupMilestone(observability, input) {
|
|
1134
|
+
observability.observeHistogram({
|
|
1135
|
+
name: "opengeni_turn_startup_milestone_duration_seconds",
|
|
1136
|
+
help: "Cumulative seconds from durable turn queueing to a bounded startup milestone.",
|
|
1137
|
+
buckets: TURN_STARTUP_MILESTONE_BUCKETS,
|
|
1138
|
+
labels: {
|
|
1139
|
+
milestone: input.milestone,
|
|
1140
|
+
provider: input.provider,
|
|
1141
|
+
backend: input.backend,
|
|
1142
|
+
outcome: input.outcome
|
|
1143
|
+
},
|
|
1144
|
+
value: Math.max(0, input.durationSeconds)
|
|
1145
|
+
});
|
|
1146
|
+
}
|
|
686
1147
|
function contentDeltaClass(type) {
|
|
687
1148
|
if (type === "agent.message.delta") {
|
|
688
1149
|
return "message";
|
|
@@ -937,14 +1398,17 @@ function modelCallAccountContext(input) {
|
|
|
937
1398
|
}
|
|
938
1399
|
|
|
939
1400
|
export {
|
|
1401
|
+
inspectOpenSandboxKubernetesInventory,
|
|
940
1402
|
observabilityEventLogger,
|
|
941
1403
|
runtimeMetricsHooksForObservability,
|
|
942
1404
|
makeMachineOpObserver,
|
|
943
1405
|
turnLifecycleMetricsFor,
|
|
1406
|
+
modelRequestLifecycleMetricsFor,
|
|
944
1407
|
recordTurnsQueuedGauge,
|
|
945
1408
|
normalizeTurnTaskQueueStats,
|
|
946
1409
|
startTurnCapacityMonitor,
|
|
947
1410
|
recordSandboxLeaseGauges,
|
|
1411
|
+
recordOpenSandboxKubernetesInventoryGauges,
|
|
948
1412
|
recordSandboxInventoryProjectionSuccess,
|
|
949
1413
|
recordSandboxInventoryProjectionFailure,
|
|
950
1414
|
recordCreditBalanceGauges,
|
|
@@ -960,7 +1424,10 @@ export {
|
|
|
960
1424
|
recordModelRequestPhase,
|
|
961
1425
|
recordTurnStartupPhase,
|
|
962
1426
|
recordTurnSandboxEstablishPolicy,
|
|
1427
|
+
recordSandboxProvisionAttempt,
|
|
1428
|
+
recordSandboxLogicalProvision,
|
|
963
1429
|
recordTurnWorkerPreparationTotal,
|
|
1430
|
+
recordTurnStartupMilestone,
|
|
964
1431
|
StreamTimingMetrics,
|
|
965
1432
|
recordBatchFlush,
|
|
966
1433
|
recordSessionEventAppendLatency,
|
|
@@ -971,4 +1438,4 @@ export {
|
|
|
971
1438
|
recordModelCacheTokens,
|
|
972
1439
|
modelCallAccountContext
|
|
973
1440
|
};
|
|
974
|
-
//# sourceMappingURL=chunk-
|
|
1441
|
+
//# sourceMappingURL=chunk-CE7TPKBS.js.map
|