@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +17 -4
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync.d.ts +4 -0
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +9 -10
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12701
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +147 -49
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync.ts +178 -37
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +102 -142
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-BJSDRSIP.js.map +0 -1
- package/dist/activities-turn-J5YO7XXX.js +0 -17243
- package/dist/activities-turn-J5YO7XXX.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
package/src/http.ts
CHANGED
|
@@ -118,12 +118,22 @@ export function dbReadyCheck(
|
|
|
118
118
|
db: Database,
|
|
119
119
|
posture?: RuntimeDatabasePostureOptions,
|
|
120
120
|
): () => Promise<void> {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
121
|
+
let inFlight: Promise<void> | undefined;
|
|
122
|
+
return () => {
|
|
123
|
+
// The HTTP timeout cannot cancel a driver query. Reuse an outstanding
|
|
124
|
+
// attempt so repeated kubelet probes cannot queue an unbounded tail behind
|
|
125
|
+
// one slow or disconnected database connection.
|
|
126
|
+
if (inFlight) return inFlight;
|
|
127
|
+
inFlight = (async () => {
|
|
128
|
+
if (posture) {
|
|
129
|
+
await assertRuntimeDatabasePosture(db, posture);
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
await db.execute(dbSql`select 1`);
|
|
133
|
+
})().finally(() => {
|
|
134
|
+
inFlight = undefined;
|
|
135
|
+
});
|
|
136
|
+
return inFlight;
|
|
127
137
|
};
|
|
128
138
|
}
|
|
129
139
|
|
package/src/index.ts
CHANGED
|
@@ -742,8 +742,13 @@ export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependen
|
|
|
742
742
|
* in the same deployment owns them.
|
|
743
743
|
*/
|
|
744
744
|
internalSchedules?: "role-default" | "none";
|
|
745
|
-
/**
|
|
746
|
-
|
|
745
|
+
/**
|
|
746
|
+
* Set false only when the host exposes equivalent lifecycle endpoints itself.
|
|
747
|
+
* A dedicated readiness database prevents ordinary activity-pool saturation
|
|
748
|
+
* from making an otherwise healthy Temporal worker fail Kubernetes readiness.
|
|
749
|
+
* The embedding host retains ownership of this handle.
|
|
750
|
+
*/
|
|
751
|
+
http?: false | { readinessTimeoutMs?: number; readinessDb?: Database };
|
|
747
752
|
};
|
|
748
753
|
|
|
749
754
|
export type OpenGeniWorkerService = {
|
|
@@ -894,7 +899,10 @@ export async function createOpenGeniWorkerService(
|
|
|
894
899
|
settings,
|
|
895
900
|
observability,
|
|
896
901
|
checks: {
|
|
897
|
-
db: dbReadyCheck(
|
|
902
|
+
db: dbReadyCheck(
|
|
903
|
+
options.http?.readinessDb ?? options.activityDependencies.db,
|
|
904
|
+
options.databasePosture,
|
|
905
|
+
),
|
|
898
906
|
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
899
907
|
temporal: temporalReadyCheck(workerBundle.connection),
|
|
900
908
|
},
|
|
@@ -1027,10 +1035,22 @@ export async function startWorker() {
|
|
|
1027
1035
|
...(searchPath ? { searchPath } : {}),
|
|
1028
1036
|
rlsStrategy: settings.rlsStrategy,
|
|
1029
1037
|
});
|
|
1038
|
+
// Readiness must prove PostgreSQL independently of activity demand. Sharing
|
|
1039
|
+
// the default application pool makes the probe wait behind live turns when
|
|
1040
|
+
// all ten slots are occupied, even while PostgreSQL itself is healthy. Since
|
|
1041
|
+
// Kubernetes readiness cannot stop Temporal polling, that false negative
|
|
1042
|
+
// only wedges rollouts and service health; reserve one probe connection.
|
|
1043
|
+
const readinessDbClient = createDb(settings.databaseUrl, {
|
|
1044
|
+
...(searchPath ? { searchPath } : {}),
|
|
1045
|
+
rlsStrategy: settings.rlsStrategy,
|
|
1046
|
+
max: 1,
|
|
1047
|
+
});
|
|
1030
1048
|
const databasePosture = {
|
|
1031
1049
|
rlsStrategy: settings.rlsStrategy,
|
|
1032
1050
|
expectedRole: settings.runtimeDatabaseRole,
|
|
1033
1051
|
targetSchema: settings.dbSchema.trim() || "public",
|
|
1052
|
+
organizationTenancyCanonicalActivationEnabled:
|
|
1053
|
+
settings.organizationTenancyCanonicalActivationEnabled,
|
|
1034
1054
|
} as const;
|
|
1035
1055
|
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
1036
1056
|
let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
|
|
@@ -1056,6 +1076,7 @@ export async function startWorker() {
|
|
|
1056
1076
|
role,
|
|
1057
1077
|
settings,
|
|
1058
1078
|
databasePosture,
|
|
1079
|
+
http: { readinessDb: readinessDbClient.db },
|
|
1059
1080
|
activityDependencies: {
|
|
1060
1081
|
observability,
|
|
1061
1082
|
db: dbClient.db,
|
|
@@ -1063,10 +1084,24 @@ export async function startWorker() {
|
|
|
1063
1084
|
},
|
|
1064
1085
|
});
|
|
1065
1086
|
} finally {
|
|
1066
|
-
await Promise.allSettled([bus?.close(), dbClient.close()]);
|
|
1087
|
+
await Promise.allSettled([bus?.close(), readinessDbClient.close(), dbClient.close()]);
|
|
1067
1088
|
}
|
|
1068
1089
|
}
|
|
1069
1090
|
|
|
1070
1091
|
if (import.meta.main) {
|
|
1071
|
-
|
|
1092
|
+
// `bun --watch` keeps its supervisor alive after the application consumes
|
|
1093
|
+
// SIGINT. Once the worker, NATS, DB, and Temporal resources have all closed,
|
|
1094
|
+
// explicitly terminate that supervisor too; otherwise a later file edit can
|
|
1095
|
+
// resurrect this stopped process as a duplicate worker with stale env/code.
|
|
1096
|
+
let interrupted = false;
|
|
1097
|
+
const markInterrupted = () => {
|
|
1098
|
+
interrupted = true;
|
|
1099
|
+
};
|
|
1100
|
+
process.once("SIGINT", markInterrupted);
|
|
1101
|
+
try {
|
|
1102
|
+
await startWorker();
|
|
1103
|
+
} finally {
|
|
1104
|
+
process.off("SIGINT", markInterrupted);
|
|
1105
|
+
if (interrupted) process.exit(0);
|
|
1106
|
+
}
|
|
1072
1107
|
}
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import type {
|
|
2
|
+
ModelContextContributionSource,
|
|
3
|
+
ModelContextContributionSummary,
|
|
2
4
|
PreferenceRegistrySnapshot,
|
|
3
5
|
ResolvedCompanyProfileSnapshot,
|
|
4
6
|
ResolvedWorkspaceInstructionPolicySnapshot,
|
|
5
|
-
|
|
7
|
+
SandboxBackend,
|
|
8
|
+
HistoricalMemoryPromptMode,
|
|
6
9
|
} from "@opengeni/contracts";
|
|
7
10
|
import { composeRuntimeSkills, type RuntimeSkillActivation } from "@opengeni/runtime";
|
|
8
11
|
|
|
@@ -14,13 +17,7 @@ export type CompanyBrainContributionCategory =
|
|
|
14
17
|
|
|
15
18
|
export type CompanyBrainContribution = Readonly<{
|
|
16
19
|
category: CompanyBrainContributionCategory;
|
|
17
|
-
source:
|
|
18
|
-
| "workspace_instruction_policy"
|
|
19
|
-
| "legacy_workspace_instructions"
|
|
20
|
-
| "preference_registry_descriptor"
|
|
21
|
-
| "company_profile"
|
|
22
|
-
| "legacy_memory_v1"
|
|
23
|
-
| "runtime_skill_catalog";
|
|
20
|
+
source: ModelContextContributionSource;
|
|
24
21
|
inclusionReason:
|
|
25
22
|
| "active_instruction_policy"
|
|
26
23
|
| "legacy_instruction_fallback"
|
|
@@ -33,11 +30,35 @@ export type CompanyBrainContribution = Readonly<{
|
|
|
33
30
|
estimatedTokens: number;
|
|
34
31
|
}>;
|
|
35
32
|
|
|
33
|
+
export function summarizeCompanyBrainContributions(
|
|
34
|
+
receipt: CompanyBrainContributionReceipt,
|
|
35
|
+
): readonly ModelContextContributionSummary[] {
|
|
36
|
+
const bySource = new Map<ModelContextContributionSource, ModelContextContributionSummary>();
|
|
37
|
+
for (const entry of receipt.contributions) {
|
|
38
|
+
const current = bySource.get(entry.source);
|
|
39
|
+
bySource.set(entry.source, {
|
|
40
|
+
source: entry.source,
|
|
41
|
+
items: (current?.items ?? 0) + 1,
|
|
42
|
+
utf8Bytes: (current?.utf8Bytes ?? 0) + entry.utf8Bytes,
|
|
43
|
+
estimatedTokens: (current?.estimatedTokens ?? 0) + entry.estimatedTokens,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
return Object.freeze([...bySource.values()].map((summary) => Object.freeze(summary)));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function modelVisibleCompanyBrainSkillActivations(
|
|
50
|
+
sandboxBackend: SandboxBackend,
|
|
51
|
+
skillActivations: readonly RuntimeSkillActivation[],
|
|
52
|
+
): readonly RuntimeSkillActivation[] {
|
|
53
|
+
return sandboxBackend === "none" ? [] : skillActivations;
|
|
54
|
+
}
|
|
55
|
+
|
|
36
56
|
export type CompanyBrainContributionReceipt = Readonly<{
|
|
57
|
+
contextSelectionReceiptId: string;
|
|
37
58
|
attemptId: string;
|
|
38
59
|
turnId: string;
|
|
39
60
|
sessionRole: "root" | "child";
|
|
40
|
-
memoryPromptMode:
|
|
61
|
+
memoryPromptMode: HistoricalMemoryPromptMode;
|
|
41
62
|
instructionPolicySnapshotId: string;
|
|
42
63
|
preferenceSnapshotId: string | null;
|
|
43
64
|
companyProfileSnapshotId: string;
|
|
@@ -51,10 +72,11 @@ export type CompanyBrainContributionReceipt = Readonly<{
|
|
|
51
72
|
* of truth for billing and context pressure.
|
|
52
73
|
*/
|
|
53
74
|
export function buildCompanyBrainContributionReceipt(input: {
|
|
75
|
+
contextSelectionReceiptId: string;
|
|
54
76
|
attemptId: string;
|
|
55
77
|
turnId: string;
|
|
56
78
|
nestedAgentDepth: number;
|
|
57
|
-
memoryPromptMode:
|
|
79
|
+
memoryPromptMode: HistoricalMemoryPromptMode;
|
|
58
80
|
instructionPolicy: ResolvedWorkspaceInstructionPolicySnapshot;
|
|
59
81
|
workspaceAgentInstructions: string | null;
|
|
60
82
|
preferences: PreferenceRegistrySnapshot | null;
|
|
@@ -131,6 +153,7 @@ export function buildCompanyBrainContributionReceipt(input: {
|
|
|
131
153
|
);
|
|
132
154
|
}
|
|
133
155
|
return Object.freeze({
|
|
156
|
+
contextSelectionReceiptId: input.contextSelectionReceiptId,
|
|
134
157
|
attemptId: input.attemptId,
|
|
135
158
|
turnId: input.turnId,
|
|
136
159
|
sessionRole: input.nestedAgentDepth === 0 ? "root" : "child",
|
|
@@ -4,6 +4,11 @@ import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
|
|
|
4
4
|
import type { EventLogger } from "@opengeni/events";
|
|
5
5
|
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
6
6
|
import type { CompanyBrainContributionReceipt } from "./model-context-contributions";
|
|
7
|
+
import {
|
|
8
|
+
OPENSANDBOX_BATCHSANDBOX_PHASES,
|
|
9
|
+
OPENSANDBOX_WORKLOAD_POD_CONDITIONS,
|
|
10
|
+
type OpenSandboxKubernetesInventory,
|
|
11
|
+
} from "./opensandbox-kubernetes-inventory";
|
|
7
12
|
import {
|
|
8
13
|
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
|
|
9
14
|
modelUsageTokenCountOrNull,
|
|
@@ -35,6 +40,7 @@ export type TemporalTurnTaskQueueStats = {
|
|
|
35
40
|
} | null;
|
|
36
41
|
|
|
37
42
|
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
43
|
+
const modelRequestTrackers = new WeakMap<Observability, ModelRequestLifecycleMetrics>();
|
|
38
44
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
39
45
|
const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
|
|
40
46
|
|
|
@@ -116,6 +122,47 @@ export function runtimeMetricsHooksForObservability(
|
|
|
116
122
|
labels: { backend, stage },
|
|
117
123
|
});
|
|
118
124
|
},
|
|
125
|
+
onSandboxProviderApiThrottle: ({ backend, operation }) => {
|
|
126
|
+
observability.incrementCounter({
|
|
127
|
+
name: "opengeni_sandbox_provider_api_throttles_total",
|
|
128
|
+
help: "Total sandbox provider API throttle responses.",
|
|
129
|
+
labels: { backend, operation },
|
|
130
|
+
});
|
|
131
|
+
},
|
|
132
|
+
onSandboxTtlRenewal: ({ backend, outcome }) => {
|
|
133
|
+
observability.incrementCounter({
|
|
134
|
+
name: "opengeni_sandbox_ttl_renewals_total",
|
|
135
|
+
help: "Total renewable sandbox provider TTL refresh attempts.",
|
|
136
|
+
labels: { backend, outcome },
|
|
137
|
+
});
|
|
138
|
+
},
|
|
139
|
+
onOpenSandboxSignedEndpoint: ({ outcome, port }) => {
|
|
140
|
+
observability.incrementCounter({
|
|
141
|
+
name: "opengeni_opensandbox_signed_endpoint_total",
|
|
142
|
+
help: "OpenSandbox signed Channel B mint and host-fetch outcomes.",
|
|
143
|
+
labels: { outcome, port: String(port) },
|
|
144
|
+
});
|
|
145
|
+
},
|
|
146
|
+
onWorkspaceArchiveObject: ({ outcome, backend }) => {
|
|
147
|
+
observability.incrementCounter({
|
|
148
|
+
name: "opengeni_workspace_archive_object_total",
|
|
149
|
+
help: "Workspace archive object-storage put/delete outcomes.",
|
|
150
|
+
labels: { outcome, backend },
|
|
151
|
+
});
|
|
152
|
+
},
|
|
153
|
+
onMcpToolCall: ({ outcome, durationSeconds }) => {
|
|
154
|
+
observability.incrementCounter({
|
|
155
|
+
name: "opengeni_mcp_tool_calls_total",
|
|
156
|
+
help: "Total physical MCP tool calls by bounded structural outcome.",
|
|
157
|
+
labels: { outcome },
|
|
158
|
+
});
|
|
159
|
+
observability.observeHistogram({
|
|
160
|
+
name: "opengeni_mcp_tool_call_duration_seconds",
|
|
161
|
+
help: "MCP tool-call duration in seconds by bounded structural outcome.",
|
|
162
|
+
labels: { outcome },
|
|
163
|
+
value: durationSeconds,
|
|
164
|
+
});
|
|
165
|
+
},
|
|
119
166
|
onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
|
|
120
167
|
observability.incrementCounter({
|
|
121
168
|
name: "opengeni_machine_op_total",
|
|
@@ -370,6 +417,111 @@ export class TurnLifecycleMetrics {
|
|
|
370
417
|
}
|
|
371
418
|
}
|
|
372
419
|
|
|
420
|
+
export function modelRequestLifecycleMetricsFor(
|
|
421
|
+
observability: Observability,
|
|
422
|
+
): ModelRequestLifecycleMetrics {
|
|
423
|
+
const existing = modelRequestTrackers.get(observability);
|
|
424
|
+
if (existing) return existing;
|
|
425
|
+
const tracker = new ModelRequestLifecycleMetrics(observability);
|
|
426
|
+
modelRequestTrackers.set(observability, tracker);
|
|
427
|
+
return tracker;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
export class ModelRequestLifecycleMetrics {
|
|
431
|
+
private readonly requests = new Map<
|
|
432
|
+
string,
|
|
433
|
+
{ provider: string; startedAt: number; lastEventAt: number }
|
|
434
|
+
>();
|
|
435
|
+
private readonly providers = new Set<string>();
|
|
436
|
+
private timer: ReturnType<typeof setInterval> | null = null;
|
|
437
|
+
|
|
438
|
+
constructor(
|
|
439
|
+
private readonly observability: Observability,
|
|
440
|
+
private readonly options: {
|
|
441
|
+
now?: () => number;
|
|
442
|
+
refreshIntervalMs?: number;
|
|
443
|
+
} = {},
|
|
444
|
+
) {}
|
|
445
|
+
|
|
446
|
+
start(key: string, provider: string): void {
|
|
447
|
+
const now = this.now();
|
|
448
|
+
this.providers.add(provider);
|
|
449
|
+
this.requests.set(key, { provider, startedAt: now, lastEventAt: now });
|
|
450
|
+
this.ensureTimer();
|
|
451
|
+
this.refreshGauges();
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
event(key: string, interEventGapMs?: number): void {
|
|
455
|
+
const request = this.requests.get(key);
|
|
456
|
+
if (!request) return;
|
|
457
|
+
request.lastEventAt = this.now();
|
|
458
|
+
this.observability.incrementCounter({
|
|
459
|
+
name: "opengeni_model_request_stream_events_total",
|
|
460
|
+
help: "Complete, valid provider SSE data events by provider.",
|
|
461
|
+
labels: { provider: request.provider },
|
|
462
|
+
});
|
|
463
|
+
if (interEventGapMs !== undefined) {
|
|
464
|
+
this.observability.observeHistogram({
|
|
465
|
+
name: "opengeni_model_request_stream_event_gap_seconds",
|
|
466
|
+
help: "Seconds between complete, valid provider SSE data events.",
|
|
467
|
+
buckets: MODEL_REQUEST_PHASE_BUCKETS,
|
|
468
|
+
labels: { provider: request.provider },
|
|
469
|
+
value: Math.max(0, interEventGapMs / 1000),
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
finish(key: string): void {
|
|
475
|
+
this.requests.delete(key);
|
|
476
|
+
this.refreshGauges();
|
|
477
|
+
if (this.requests.size === 0) this.stopTimer();
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
refreshGauges(): void {
|
|
481
|
+
const now = this.now();
|
|
482
|
+
for (const provider of this.providers) {
|
|
483
|
+
const active = [...this.requests.values()].filter((request) => request.provider === provider);
|
|
484
|
+
this.observability.setGauge({
|
|
485
|
+
name: "opengeni_model_requests_inflight",
|
|
486
|
+
help: "Current in-flight provider model requests in this worker process.",
|
|
487
|
+
labels: { provider },
|
|
488
|
+
value: active.length,
|
|
489
|
+
});
|
|
490
|
+
this.observability.setGauge({
|
|
491
|
+
name: "opengeni_model_request_oldest_no_event_age_seconds",
|
|
492
|
+
help: "Seconds since the least recently progressing in-flight provider request received a valid SSE event.",
|
|
493
|
+
labels: { provider },
|
|
494
|
+
value:
|
|
495
|
+
active.length === 0
|
|
496
|
+
? 0
|
|
497
|
+
: Math.max(0, (now - Math.min(...active.map((request) => request.lastEventAt))) / 1000),
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
stop(): void {
|
|
503
|
+
this.requests.clear();
|
|
504
|
+
this.refreshGauges();
|
|
505
|
+
this.stopTimer();
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
private ensureTimer(): void {
|
|
509
|
+
if (this.timer) return;
|
|
510
|
+
this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15_000);
|
|
511
|
+
this.timer.unref?.();
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
private stopTimer(): void {
|
|
515
|
+
if (!this.timer) return;
|
|
516
|
+
clearInterval(this.timer);
|
|
517
|
+
this.timer = null;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
private now(): number {
|
|
521
|
+
return this.options.now?.() ?? Date.now();
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
373
525
|
export function recordTurnsQueuedGauge(observability: Observability, value: number): void {
|
|
374
526
|
observability.setGauge({
|
|
375
527
|
name: "opengeni_turns_queued",
|
|
@@ -538,12 +690,45 @@ export function recordSandboxLeaseGauges(
|
|
|
538
690
|
}
|
|
539
691
|
}
|
|
540
692
|
|
|
693
|
+
export function recordOpenSandboxKubernetesInventoryGauges(
|
|
694
|
+
observability: Observability,
|
|
695
|
+
inventory: OpenSandboxKubernetesInventory,
|
|
696
|
+
): void {
|
|
697
|
+
for (const phase of OPENSANDBOX_BATCHSANDBOX_PHASES) {
|
|
698
|
+
observability.setGauge({
|
|
699
|
+
name: "opengeni_opensandbox_batchsandboxes",
|
|
700
|
+
help: "Current OpenSandbox BatchSandboxes by bounded lifecycle phase.",
|
|
701
|
+
labels: { phase },
|
|
702
|
+
value: inventory.batchSandboxPhases[phase],
|
|
703
|
+
});
|
|
704
|
+
}
|
|
705
|
+
for (const condition of OPENSANDBOX_WORKLOAD_POD_CONDITIONS) {
|
|
706
|
+
observability.setGauge({
|
|
707
|
+
name: "opengeni_opensandbox_workload_pods",
|
|
708
|
+
help: "Current OpenSandbox workload Pods by bounded operational condition.",
|
|
709
|
+
labels: { condition },
|
|
710
|
+
value: inventory.workloadPodConditions[condition],
|
|
711
|
+
});
|
|
712
|
+
}
|
|
713
|
+
observability.setGauge({
|
|
714
|
+
name: "opengeni_opensandbox_cleanup_stuck",
|
|
715
|
+
help: "Current OpenSandbox BatchSandboxes deleting past the cleanup threshold with finalizers.",
|
|
716
|
+
value: inventory.cleanupStuck,
|
|
717
|
+
});
|
|
718
|
+
observability.setGauge({
|
|
719
|
+
name: "opengeni_opensandbox_expiration_overdue",
|
|
720
|
+
help: "Current OpenSandbox BatchSandboxes still present past the provider-expiry threshold.",
|
|
721
|
+
value: inventory.expirationOverdue,
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
|
|
541
725
|
export const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [
|
|
542
726
|
"leases",
|
|
543
727
|
"checkpoint_artifacts",
|
|
544
728
|
"rotation_backlog",
|
|
545
729
|
"retained_processes",
|
|
546
730
|
"expired_drains",
|
|
731
|
+
"opensandbox_kubernetes",
|
|
547
732
|
] as const;
|
|
548
733
|
|
|
549
734
|
export type SandboxInventoryProjectionDomain =
|
|
@@ -921,6 +1106,17 @@ export type TurnStartupPhase =
|
|
|
921
1106
|
| "model_prepare_sandbox_client_delete"
|
|
922
1107
|
| "model_prepare_sandbox_client_state_serialize"
|
|
923
1108
|
| "model_prepare_sandbox_client_reuse_check"
|
|
1109
|
+
| "model_prepare_sandbox_workspace_mutation_admission"
|
|
1110
|
+
| "model_prepare_sandbox_workspace_mutation_provider"
|
|
1111
|
+
| "model_prepare_sandbox_workspace_mutation_settlement"
|
|
1112
|
+
| "model_prepare_sandbox_first_routed_resolution_other"
|
|
1113
|
+
| "model_prepare_sandbox_first_routed_mutation_admission"
|
|
1114
|
+
| "model_prepare_sandbox_first_routed_provider_operation"
|
|
1115
|
+
| "model_prepare_sandbox_first_routed_mutation_settlement"
|
|
1116
|
+
| "model_prepare_sandbox_first_routed_other"
|
|
1117
|
+
| "model_prepare_sandbox_snapshot_wait"
|
|
1118
|
+
| "model_prepare_runner_before_first_sandbox_operation"
|
|
1119
|
+
| "model_prepare_sdk_after_first_sandbox_operation"
|
|
924
1120
|
| "model_prepare_runner_before_mcp_tools"
|
|
925
1121
|
| "model_prepare_mcp_tools_snapshot"
|
|
926
1122
|
| "model_prepare_mcp_tools_before_input_filter"
|
|
@@ -938,6 +1134,7 @@ export type TurnStartupPhase =
|
|
|
938
1134
|
| "stream_bootstrap";
|
|
939
1135
|
|
|
940
1136
|
export type TurnStartupOutcome = "completed" | "failed";
|
|
1137
|
+
export type TurnStartupMilestone = "queue" | "provider_dispatch" | "first_byte";
|
|
941
1138
|
export type TurnStartupCache = "hit" | "miss" | "disabled" | "none";
|
|
942
1139
|
export type TurnStartupCountBucket = "0" | "1" | "2-5" | "6-20" | "21+" | "unknown";
|
|
943
1140
|
export type TurnSandboxEstablishPolicy = "eager" | "on-demand";
|
|
@@ -947,12 +1144,45 @@ export type TurnSandboxEstablishReason =
|
|
|
947
1144
|
| "machine_primary"
|
|
948
1145
|
| "backend_none"
|
|
949
1146
|
| "initial_run_credentials"
|
|
1147
|
+
| "initial_run_credentials_deferred"
|
|
950
1148
|
| "generated_video_files"
|
|
951
1149
|
| "signed_file_resources";
|
|
1150
|
+
export type SandboxLogicalProvisionCategory =
|
|
1151
|
+
| "create"
|
|
1152
|
+
| "resume"
|
|
1153
|
+
| "exec_readiness"
|
|
1154
|
+
| "sibling_warming"
|
|
1155
|
+
| "lease_superseded"
|
|
1156
|
+
| "drain_capture_wait"
|
|
1157
|
+
| "archive_recovery"
|
|
1158
|
+
| "provider_transport"
|
|
1159
|
+
| "configuration"
|
|
1160
|
+
| "unknown";
|
|
1161
|
+
export type SandboxLogicalProvisionStage =
|
|
1162
|
+
| "create"
|
|
1163
|
+
| "resume"
|
|
1164
|
+
| "exec_readiness"
|
|
1165
|
+
| "sibling_warming"
|
|
1166
|
+
| "lease_admission"
|
|
1167
|
+
| "lifecycle_wait"
|
|
1168
|
+
| "archive_recovery"
|
|
1169
|
+
| "configuration"
|
|
1170
|
+
| "provider_transport"
|
|
1171
|
+
| "unknown";
|
|
1172
|
+
export type SandboxLogicalProvisionOutcome = "completed" | "expected_transition" | "failed";
|
|
1173
|
+
export type SandboxProvisionAttemptOutcome = "completed" | "retrying" | "failed";
|
|
952
1174
|
|
|
953
1175
|
const TURN_STARTUP_PHASE_BUCKETS = [
|
|
954
1176
|
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120,
|
|
955
1177
|
];
|
|
1178
|
+
const TURN_STARTUP_MILESTONE_BUCKETS = [
|
|
1179
|
+
...TURN_STARTUP_PHASE_BUCKETS,
|
|
1180
|
+
300,
|
|
1181
|
+
600,
|
|
1182
|
+
1_800,
|
|
1183
|
+
3_600,
|
|
1184
|
+
7_200,
|
|
1185
|
+
];
|
|
956
1186
|
|
|
957
1187
|
export function turnStartupCountBucket(count: number | null): TurnStartupCountBucket {
|
|
958
1188
|
if (count === null || !Number.isFinite(count) || count < 0) return "unknown";
|
|
@@ -1020,6 +1250,103 @@ export function recordTurnSandboxEstablishPolicy(
|
|
|
1020
1250
|
});
|
|
1021
1251
|
}
|
|
1022
1252
|
|
|
1253
|
+
/** Physical establish/resume attempts inside one logical provision. Safe
|
|
1254
|
+
* lifecycle retries are counted here, never as another user-visible failure. */
|
|
1255
|
+
export function recordSandboxProvisionAttempt(
|
|
1256
|
+
observability: Observability,
|
|
1257
|
+
input: {
|
|
1258
|
+
backend: string;
|
|
1259
|
+
stage: SandboxLogicalProvisionStage;
|
|
1260
|
+
category: SandboxLogicalProvisionCategory;
|
|
1261
|
+
outcome: SandboxProvisionAttemptOutcome;
|
|
1262
|
+
durationSeconds: number;
|
|
1263
|
+
},
|
|
1264
|
+
): void {
|
|
1265
|
+
const labels = {
|
|
1266
|
+
backend: input.backend,
|
|
1267
|
+
stage: input.stage,
|
|
1268
|
+
category: input.category,
|
|
1269
|
+
outcome: input.outcome,
|
|
1270
|
+
};
|
|
1271
|
+
try {
|
|
1272
|
+
observability.incrementCounter({
|
|
1273
|
+
name: "opengeni_sandbox_provision_attempts_total",
|
|
1274
|
+
help: "Internal physical attempts within logical sandbox provisions.",
|
|
1275
|
+
labels,
|
|
1276
|
+
});
|
|
1277
|
+
observability.observeHistogram({
|
|
1278
|
+
name: "opengeni_sandbox_provision_attempt_duration_seconds",
|
|
1279
|
+
help: "Internal sandbox provision attempt duration by bounded structural outcome.",
|
|
1280
|
+
buckets: TURN_STARTUP_PHASE_BUCKETS,
|
|
1281
|
+
labels,
|
|
1282
|
+
value: Math.max(0, input.durationSeconds),
|
|
1283
|
+
});
|
|
1284
|
+
} catch {
|
|
1285
|
+
try {
|
|
1286
|
+
observability.incrementCounter({
|
|
1287
|
+
name: "opengeni_observability_observer_errors_total",
|
|
1288
|
+
help: "Observability observer failures isolated from product execution.",
|
|
1289
|
+
labels: { observer: "sandbox_provision" },
|
|
1290
|
+
});
|
|
1291
|
+
} catch {
|
|
1292
|
+
// The registry itself is unhealthy; provisioning remains authoritative.
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
/** One terminal observation for one correlation-qualified logical provision. */
|
|
1298
|
+
export function recordSandboxLogicalProvision(
|
|
1299
|
+
observability: Observability,
|
|
1300
|
+
input: {
|
|
1301
|
+
backend: string;
|
|
1302
|
+
stage: SandboxLogicalProvisionStage;
|
|
1303
|
+
category: SandboxLogicalProvisionCategory | "none";
|
|
1304
|
+
outcome: SandboxLogicalProvisionOutcome;
|
|
1305
|
+
expected: boolean;
|
|
1306
|
+
internalAttempts: number;
|
|
1307
|
+
durationSeconds: number;
|
|
1308
|
+
},
|
|
1309
|
+
): void {
|
|
1310
|
+
const labels = {
|
|
1311
|
+
backend: input.backend,
|
|
1312
|
+
stage: input.stage,
|
|
1313
|
+
category: input.category,
|
|
1314
|
+
outcome: input.outcome,
|
|
1315
|
+
expected: input.expected ? "true" : "false",
|
|
1316
|
+
};
|
|
1317
|
+
try {
|
|
1318
|
+
observability.incrementCounter({
|
|
1319
|
+
name: "opengeni_sandbox_provisions_total",
|
|
1320
|
+
help: "Logical sandbox provisions by terminal outcome and bounded failure taxonomy.",
|
|
1321
|
+
labels,
|
|
1322
|
+
});
|
|
1323
|
+
observability.observeHistogram({
|
|
1324
|
+
name: "opengeni_sandbox_provision_duration_seconds",
|
|
1325
|
+
help: "Logical sandbox provision duration including internal safe retries.",
|
|
1326
|
+
buckets: TURN_STARTUP_PHASE_BUCKETS,
|
|
1327
|
+
labels,
|
|
1328
|
+
value: Math.max(0, input.durationSeconds),
|
|
1329
|
+
});
|
|
1330
|
+
observability.observeHistogram({
|
|
1331
|
+
name: "opengeni_sandbox_provision_internal_attempts",
|
|
1332
|
+
help: "Internal physical attempts per terminal logical sandbox provision.",
|
|
1333
|
+
buckets: [1, 2, 3, 4, 5, 8],
|
|
1334
|
+
labels,
|
|
1335
|
+
value: Math.max(1, input.internalAttempts),
|
|
1336
|
+
});
|
|
1337
|
+
} catch {
|
|
1338
|
+
try {
|
|
1339
|
+
observability.incrementCounter({
|
|
1340
|
+
name: "opengeni_observability_observer_errors_total",
|
|
1341
|
+
help: "Observability observer failures isolated from product execution.",
|
|
1342
|
+
labels: { observer: "sandbox_provision" },
|
|
1343
|
+
});
|
|
1344
|
+
} catch {
|
|
1345
|
+
// The registry itself is unhealthy; provisioning remains authoritative.
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1023
1350
|
export function recordTurnWorkerPreparationTotal(
|
|
1024
1351
|
observability: Observability,
|
|
1025
1352
|
input: {
|
|
@@ -1042,6 +1369,36 @@ export function recordTurnWorkerPreparationTotal(
|
|
|
1042
1369
|
});
|
|
1043
1370
|
}
|
|
1044
1371
|
|
|
1372
|
+
/**
|
|
1373
|
+
* Measure cumulative user-visible startup latency from the durable turn queue
|
|
1374
|
+
* timestamp to a bounded milestone. Unlike the per-phase histogram above,
|
|
1375
|
+
* these samples are end-to-end and can therefore back real queue/dispatch/TTFB
|
|
1376
|
+
* SLOs without adding turn or session identifiers to Prometheus.
|
|
1377
|
+
*/
|
|
1378
|
+
export function recordTurnStartupMilestone(
|
|
1379
|
+
observability: Observability,
|
|
1380
|
+
input: {
|
|
1381
|
+
milestone: TurnStartupMilestone;
|
|
1382
|
+
provider: string;
|
|
1383
|
+
backend: string;
|
|
1384
|
+
outcome: TurnStartupOutcome;
|
|
1385
|
+
durationSeconds: number;
|
|
1386
|
+
},
|
|
1387
|
+
): void {
|
|
1388
|
+
observability.observeHistogram({
|
|
1389
|
+
name: "opengeni_turn_startup_milestone_duration_seconds",
|
|
1390
|
+
help: "Cumulative seconds from durable turn queueing to a bounded startup milestone.",
|
|
1391
|
+
buckets: TURN_STARTUP_MILESTONE_BUCKETS,
|
|
1392
|
+
labels: {
|
|
1393
|
+
milestone: input.milestone,
|
|
1394
|
+
provider: input.provider,
|
|
1395
|
+
backend: input.backend,
|
|
1396
|
+
outcome: input.outcome,
|
|
1397
|
+
},
|
|
1398
|
+
value: Math.max(0, input.durationSeconds),
|
|
1399
|
+
});
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1045
1402
|
// ── Streaming SLIs ────────────────────────────────────────────────────────────
|
|
1046
1403
|
// Instruments the token-streaming pipeline so "streaming is sluggish" is a number,
|
|
1047
1404
|
// not a vibe. Split across the three attributable stages so an operator can tell
|