@opengeni/worker-bundle 0.16.29 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +110 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +6 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +17 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +21 -36
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +7 -5
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +87 -5
- package/dist/activities/video-generation-admission.d.ts +43 -0
- package/dist/activities/video-generation-credential.d.ts +24 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +55 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/activities-control-XURE4VKN.js +7221 -0
- package/dist/activities-control-XURE4VKN.js.map +1 -0
- package/dist/activities-control.d.ts +297 -0
- package/dist/activities-turn-6EBXGGLS.js +16036 -0
- package/dist/activities-turn-6EBXGGLS.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +138 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-5GRJIP5H.js +3092 -0
- package/dist/chunk-5GRJIP5H.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-TYNPCG4U.js +906 -0
- package/dist/chunk-TYNPCG4U.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +189 -10
- package/dist/index.js +715 -17380
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +23 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +53 -1
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1094 -370
- package/dist/workflows/activities.d.ts +8 -2
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +10 -7
- package/package.json +35 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +2600 -340
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +54 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +54 -45
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +8 -3
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +75 -152
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +104 -74
- package/src/activities/session-state.ts +22 -4
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +104 -5
- package/src/activities/video-generation-admission.ts +303 -0
- package/src/activities/video-generation-credential.ts +112 -0
- package/src/activities/video-generation-reconciliation.ts +771 -0
- package/src/activities/video-media-validation.ts +332 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +350 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +58 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +53 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +328 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +123 -0
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +297 -13
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +32 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +31 -0
- package/src/workflows/file-upload-reaper.ts +9 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/session.ts +20 -6
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +18 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
PreferenceRegistrySnapshot,
|
|
3
|
+
ResolvedCompanyProfileSnapshot,
|
|
4
|
+
ResolvedWorkspaceInstructionPolicySnapshot,
|
|
5
|
+
WorkspaceMemoryPromptMode,
|
|
6
|
+
} from "@opengeni/contracts";
|
|
7
|
+
import { composeRuntimeSkills, type RuntimeSkillActivation } from "@opengeni/runtime";
|
|
8
|
+
|
|
9
|
+
export type CompanyBrainContributionCategory =
|
|
10
|
+
| "mandatory_rule"
|
|
11
|
+
| "skill_guide_descriptor"
|
|
12
|
+
| "retrieved_knowledge"
|
|
13
|
+
| "task_note";
|
|
14
|
+
|
|
15
|
+
export type CompanyBrainContribution = Readonly<{
|
|
16
|
+
category: CompanyBrainContributionCategory;
|
|
17
|
+
source:
|
|
18
|
+
| "workspace_instruction_policy"
|
|
19
|
+
| "legacy_workspace_instructions"
|
|
20
|
+
| "preference_registry_descriptor"
|
|
21
|
+
| "company_profile"
|
|
22
|
+
| "legacy_memory_v1"
|
|
23
|
+
| "runtime_skill_catalog";
|
|
24
|
+
inclusionReason:
|
|
25
|
+
| "active_instruction_policy"
|
|
26
|
+
| "legacy_instruction_fallback"
|
|
27
|
+
| "active_preference_descriptor"
|
|
28
|
+
| "active_company_profile_root"
|
|
29
|
+
| "legacy_standing_working_set"
|
|
30
|
+
| "authorized_skill_descriptor";
|
|
31
|
+
authorityScope: "organization" | "workspace" | "user" | "session";
|
|
32
|
+
utf8Bytes: number;
|
|
33
|
+
estimatedTokens: number;
|
|
34
|
+
}>;
|
|
35
|
+
|
|
36
|
+
export type CompanyBrainContributionReceipt = Readonly<{
|
|
37
|
+
attemptId: string;
|
|
38
|
+
turnId: string;
|
|
39
|
+
sessionRole: "root" | "child";
|
|
40
|
+
memoryPromptMode: WorkspaceMemoryPromptMode;
|
|
41
|
+
instructionPolicySnapshotId: string;
|
|
42
|
+
preferenceSnapshotId: string | null;
|
|
43
|
+
companyProfileSnapshotId: string;
|
|
44
|
+
contributions: readonly CompanyBrainContribution[];
|
|
45
|
+
}>;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Build a content-free exact-attempt receipt for the Company Brain material
|
|
49
|
+
* selected before inference. Token counts are deliberately estimates (UTF-8
|
|
50
|
+
* bytes / 4); provider-accounted total input tokens remain the separate source
|
|
51
|
+
* of truth for billing and context pressure.
|
|
52
|
+
*/
|
|
53
|
+
export function buildCompanyBrainContributionReceipt(input: {
|
|
54
|
+
attemptId: string;
|
|
55
|
+
turnId: string;
|
|
56
|
+
nestedAgentDepth: number;
|
|
57
|
+
memoryPromptMode: WorkspaceMemoryPromptMode;
|
|
58
|
+
instructionPolicy: ResolvedWorkspaceInstructionPolicySnapshot;
|
|
59
|
+
workspaceAgentInstructions: string | null;
|
|
60
|
+
preferences: PreferenceRegistrySnapshot | null;
|
|
61
|
+
companyProfile: ResolvedCompanyProfileSnapshot;
|
|
62
|
+
companyProfileIncluded: boolean;
|
|
63
|
+
workspaceMemory: string | null;
|
|
64
|
+
skillActivations: readonly RuntimeSkillActivation[];
|
|
65
|
+
}): CompanyBrainContributionReceipt {
|
|
66
|
+
const contributions: CompanyBrainContribution[] = [];
|
|
67
|
+
for (const entry of input.instructionPolicy.entries) {
|
|
68
|
+
contributions.push(
|
|
69
|
+
contribution({
|
|
70
|
+
category: "mandatory_rule",
|
|
71
|
+
source: "workspace_instruction_policy",
|
|
72
|
+
inclusionReason: "active_instruction_policy",
|
|
73
|
+
authorityScope: "workspace",
|
|
74
|
+
text: entry.content,
|
|
75
|
+
}),
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
if (input.instructionPolicy.entries.length === 0 && input.workspaceAgentInstructions) {
|
|
79
|
+
contributions.push(
|
|
80
|
+
contribution({
|
|
81
|
+
category: "mandatory_rule",
|
|
82
|
+
source: "legacy_workspace_instructions",
|
|
83
|
+
inclusionReason: "legacy_instruction_fallback",
|
|
84
|
+
authorityScope: "workspace",
|
|
85
|
+
text: input.workspaceAgentInstructions,
|
|
86
|
+
}),
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
for (const descriptor of input.preferences?.descriptors ?? []) {
|
|
90
|
+
contributions.push(
|
|
91
|
+
contribution({
|
|
92
|
+
category: "skill_guide_descriptor",
|
|
93
|
+
source: "preference_registry_descriptor",
|
|
94
|
+
inclusionReason: "active_preference_descriptor",
|
|
95
|
+
authorityScope: descriptor.scope,
|
|
96
|
+
text: JSON.stringify(descriptor),
|
|
97
|
+
}),
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
if (input.companyProfileIncluded && input.companyProfile.profile) {
|
|
101
|
+
contributions.push(
|
|
102
|
+
contribution({
|
|
103
|
+
category: "retrieved_knowledge",
|
|
104
|
+
source: "company_profile",
|
|
105
|
+
inclusionReason: "active_company_profile_root",
|
|
106
|
+
authorityScope: "organization",
|
|
107
|
+
text: JSON.stringify(input.companyProfile.profile.profile),
|
|
108
|
+
}),
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
if (input.workspaceMemory?.trim()) {
|
|
112
|
+
contributions.push(
|
|
113
|
+
contribution({
|
|
114
|
+
category: "retrieved_knowledge",
|
|
115
|
+
source: "legacy_memory_v1",
|
|
116
|
+
inclusionReason: "legacy_standing_working_set",
|
|
117
|
+
authorityScope: "workspace",
|
|
118
|
+
text: input.workspaceMemory,
|
|
119
|
+
}),
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
for (const descriptor of composeRuntimeSkills(input.skillActivations).configuredDescriptors) {
|
|
123
|
+
contributions.push(
|
|
124
|
+
contribution({
|
|
125
|
+
category: "skill_guide_descriptor",
|
|
126
|
+
source: "runtime_skill_catalog",
|
|
127
|
+
inclusionReason: "authorized_skill_descriptor",
|
|
128
|
+
authorityScope: descriptor.source === "session" ? "session" : "workspace",
|
|
129
|
+
text: descriptor.description,
|
|
130
|
+
}),
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
return Object.freeze({
|
|
134
|
+
attemptId: input.attemptId,
|
|
135
|
+
turnId: input.turnId,
|
|
136
|
+
sessionRole: input.nestedAgentDepth === 0 ? "root" : "child",
|
|
137
|
+
memoryPromptMode: input.memoryPromptMode,
|
|
138
|
+
instructionPolicySnapshotId: input.instructionPolicy.id,
|
|
139
|
+
preferenceSnapshotId: input.preferences?.id ?? null,
|
|
140
|
+
companyProfileSnapshotId: input.companyProfile.id,
|
|
141
|
+
contributions: Object.freeze(contributions),
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function contribution(
|
|
146
|
+
input: Omit<CompanyBrainContribution, "utf8Bytes" | "estimatedTokens"> & {
|
|
147
|
+
text: string;
|
|
148
|
+
},
|
|
149
|
+
): CompanyBrainContribution {
|
|
150
|
+
const utf8Bytes = Buffer.byteLength(input.text, "utf8");
|
|
151
|
+
return Object.freeze({
|
|
152
|
+
category: input.category,
|
|
153
|
+
source: input.source,
|
|
154
|
+
inclusionReason: input.inclusionReason,
|
|
155
|
+
authorityScope: input.authorityScope,
|
|
156
|
+
utf8Bytes,
|
|
157
|
+
estimatedTokens: Math.ceil(utf8Bytes / 4),
|
|
158
|
+
});
|
|
159
|
+
}
|
|
@@ -3,6 +3,7 @@ import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
|
3
3
|
import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
|
|
4
4
|
import type { EventLogger } from "@opengeni/events";
|
|
5
5
|
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
6
|
+
import type { CompanyBrainContributionReceipt } from "./model-context-contributions";
|
|
6
7
|
import {
|
|
7
8
|
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
|
|
8
9
|
modelUsageTokenCountOrNull,
|
|
@@ -15,6 +16,23 @@ export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
|
|
|
15
16
|
export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
|
|
16
17
|
export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
|
|
17
18
|
export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
19
|
+
export type TurnTaskQueueStats = {
|
|
20
|
+
/** Temporal activity tasks ready for a turn worker to accept. */
|
|
21
|
+
eligibleBacklog: number;
|
|
22
|
+
oldestBacklogAgeSeconds: number;
|
|
23
|
+
tasksAddRate: number;
|
|
24
|
+
tasksDispatchRate: number;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export type TemporalTurnTaskQueueStats = {
|
|
28
|
+
approximateBacklogCount?: unknown;
|
|
29
|
+
approximateBacklogAge?: {
|
|
30
|
+
seconds?: unknown;
|
|
31
|
+
nanos?: unknown;
|
|
32
|
+
} | null;
|
|
33
|
+
tasksAddRate?: unknown;
|
|
34
|
+
tasksDispatchRate?: unknown;
|
|
35
|
+
} | null;
|
|
18
36
|
|
|
19
37
|
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
20
38
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
@@ -78,23 +96,24 @@ export function runtimeMetricsHooksForObservability(
|
|
|
78
96
|
value: durationSeconds,
|
|
79
97
|
});
|
|
80
98
|
},
|
|
81
|
-
onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
|
|
99
|
+
onSandboxCreate: ({ backend, imageSource, outcome, durationSeconds }) => {
|
|
82
100
|
observability.incrementCounter({
|
|
83
101
|
name: "opengeni_sandbox_creates_total",
|
|
84
102
|
help: "Total sandbox create attempts by backend and outcome.",
|
|
85
|
-
labels: { backend, outcome },
|
|
103
|
+
labels: { backend, image_source: imageSource, outcome },
|
|
86
104
|
});
|
|
87
105
|
observability.observeHistogram({
|
|
88
106
|
name: "opengeni_sandbox_create_duration_seconds",
|
|
89
107
|
help: "Sandbox create duration in seconds by backend.",
|
|
90
|
-
labels: { backend },
|
|
108
|
+
labels: { backend, image_source: imageSource },
|
|
91
109
|
value: durationSeconds,
|
|
92
110
|
});
|
|
93
111
|
},
|
|
94
|
-
onSandboxWarmingTimeout: () => {
|
|
112
|
+
onSandboxWarmingTimeout: ({ backend, stage }) => {
|
|
95
113
|
observability.incrementCounter({
|
|
96
114
|
name: "opengeni_sandbox_warming_timeouts_total",
|
|
97
115
|
help: "Total sandbox warming timeouts.",
|
|
116
|
+
labels: { backend, stage },
|
|
98
117
|
});
|
|
99
118
|
},
|
|
100
119
|
onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
|
|
@@ -230,7 +249,7 @@ export function turnLifecycleMetricsFor(observability: Observability): TurnLifec
|
|
|
230
249
|
}
|
|
231
250
|
|
|
232
251
|
export class TurnLifecycleMetrics {
|
|
233
|
-
private readonly
|
|
252
|
+
private readonly turns = new Map<string, { startedAt: number; lastProgressAt: number }>();
|
|
234
253
|
private timer: ReturnType<typeof setInterval> | null = null;
|
|
235
254
|
|
|
236
255
|
constructor(
|
|
@@ -239,15 +258,22 @@ export class TurnLifecycleMetrics {
|
|
|
239
258
|
) {}
|
|
240
259
|
|
|
241
260
|
start(turnId: string): void {
|
|
242
|
-
|
|
261
|
+
const now = this.now();
|
|
262
|
+
this.turns.set(turnId, { startedAt: now, lastProgressAt: now });
|
|
243
263
|
this.ensureTimer();
|
|
244
264
|
this.refreshGauges();
|
|
245
265
|
}
|
|
246
266
|
|
|
267
|
+
progress(turnId: string): void {
|
|
268
|
+
const turn = this.turns.get(turnId);
|
|
269
|
+
if (!turn) return;
|
|
270
|
+
turn.lastProgressAt = this.now();
|
|
271
|
+
}
|
|
272
|
+
|
|
247
273
|
finish(turnId: string, outcome: TurnOutcome | null, durationSeconds?: number): void {
|
|
248
|
-
const startedAt = this.
|
|
274
|
+
const startedAt = this.turns.get(turnId)?.startedAt;
|
|
249
275
|
if (startedAt !== undefined) {
|
|
250
|
-
this.
|
|
276
|
+
this.turns.delete(turnId);
|
|
251
277
|
}
|
|
252
278
|
if (outcome) {
|
|
253
279
|
const observedDuration =
|
|
@@ -266,7 +292,7 @@ export class TurnLifecycleMetrics {
|
|
|
266
292
|
});
|
|
267
293
|
}
|
|
268
294
|
this.refreshGauges();
|
|
269
|
-
if (this.
|
|
295
|
+
if (this.turns.size === 0) {
|
|
270
296
|
this.stopTimer();
|
|
271
297
|
}
|
|
272
298
|
}
|
|
@@ -275,32 +301,46 @@ export class TurnLifecycleMetrics {
|
|
|
275
301
|
this.observability.setGauge({
|
|
276
302
|
name: "opengeni_turns_inflight",
|
|
277
303
|
help: "Current number of in-flight agent turns in this worker process.",
|
|
278
|
-
value: this.
|
|
304
|
+
value: this.turns.size,
|
|
279
305
|
});
|
|
280
306
|
this.observability.setGauge({
|
|
281
307
|
name: "opengeni_turn_oldest_inflight_age_seconds",
|
|
282
308
|
help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
|
|
283
309
|
value: this.oldestInflightAgeSeconds(),
|
|
284
310
|
});
|
|
311
|
+
this.observability.setGauge({
|
|
312
|
+
name: "opengeni_turn_oldest_no_progress_age_seconds",
|
|
313
|
+
help: "Seconds since durable progress for the least recently progressing in-flight turn.",
|
|
314
|
+
value: this.oldestNoProgressAgeSeconds(),
|
|
315
|
+
});
|
|
285
316
|
}
|
|
286
317
|
|
|
287
318
|
stop(): void {
|
|
288
|
-
this.
|
|
319
|
+
this.turns.clear();
|
|
289
320
|
this.refreshGauges();
|
|
290
321
|
this.stopTimer();
|
|
291
322
|
}
|
|
292
323
|
|
|
293
324
|
private oldestInflightAgeSeconds(): number {
|
|
294
|
-
if (this.
|
|
325
|
+
if (this.turns.size === 0) {
|
|
295
326
|
return 0;
|
|
296
327
|
}
|
|
297
328
|
let oldest = Number.POSITIVE_INFINITY;
|
|
298
|
-
for (const startedAt of this.
|
|
329
|
+
for (const { startedAt } of this.turns.values()) {
|
|
299
330
|
oldest = Math.min(oldest, startedAt);
|
|
300
331
|
}
|
|
301
332
|
return Math.max(0, (this.now() - oldest) / 1000);
|
|
302
333
|
}
|
|
303
334
|
|
|
335
|
+
private oldestNoProgressAgeSeconds(): number {
|
|
336
|
+
if (this.turns.size === 0) return 0;
|
|
337
|
+
let leastRecentProgress = Number.POSITIVE_INFINITY;
|
|
338
|
+
for (const { lastProgressAt } of this.turns.values()) {
|
|
339
|
+
leastRecentProgress = Math.min(leastRecentProgress, lastProgressAt);
|
|
340
|
+
}
|
|
341
|
+
return Math.max(0, (this.now() - leastRecentProgress) / 1000);
|
|
342
|
+
}
|
|
343
|
+
|
|
304
344
|
private ensureTimer(): void {
|
|
305
345
|
if (this.timer) {
|
|
306
346
|
return;
|
|
@@ -330,6 +370,152 @@ export function recordTurnsQueuedGauge(observability: Observability, value: numb
|
|
|
330
370
|
});
|
|
331
371
|
}
|
|
332
372
|
|
|
373
|
+
/**
|
|
374
|
+
* Record the authoritative turn activity queue rather than aggregate Postgres
|
|
375
|
+
* prompts. A paused human prompt is durable queue truth but is not runnable and
|
|
376
|
+
* never reaches this Temporal task queue. DescribeTaskQueue's approximate
|
|
377
|
+
* backlog count and age are explicitly documented by Temporal as autoscaling
|
|
378
|
+
* signals.
|
|
379
|
+
*/
|
|
380
|
+
export function recordTurnTaskQueueStats(
|
|
381
|
+
observability: Observability,
|
|
382
|
+
stats: TurnTaskQueueStats,
|
|
383
|
+
): void {
|
|
384
|
+
observability.setGauge({
|
|
385
|
+
name: "opengeni_turn_eligible_backlog",
|
|
386
|
+
help: "Temporal runAgentTurn activity tasks eligible for immediate worker admission.",
|
|
387
|
+
value: nonnegativeFinite(stats.eligibleBacklog),
|
|
388
|
+
});
|
|
389
|
+
observability.setGauge({
|
|
390
|
+
name: "opengeni_turn_eligible_backlog_oldest_age_seconds",
|
|
391
|
+
help: "Approximate age of the oldest eligible runAgentTurn activity task.",
|
|
392
|
+
value: nonnegativeFinite(stats.oldestBacklogAgeSeconds),
|
|
393
|
+
});
|
|
394
|
+
observability.setGauge({
|
|
395
|
+
name: "opengeni_turn_eligible_tasks_add_rate",
|
|
396
|
+
help: "Temporal runAgentTurn tasks added per second over its rolling window.",
|
|
397
|
+
value: nonnegativeFinite(stats.tasksAddRate),
|
|
398
|
+
});
|
|
399
|
+
observability.setGauge({
|
|
400
|
+
name: "opengeni_turn_eligible_tasks_dispatch_rate",
|
|
401
|
+
help: "Temporal runAgentTurn tasks dispatched per second over its rolling window.",
|
|
402
|
+
value: nonnegativeFinite(stats.tasksDispatchRate),
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
export function normalizeTurnTaskQueueStats(
|
|
407
|
+
stats: TemporalTurnTaskQueueStats | undefined,
|
|
408
|
+
): TurnTaskQueueStats {
|
|
409
|
+
if (!stats) {
|
|
410
|
+
throw new Error("Temporal DescribeTaskQueue response omitted required stats");
|
|
411
|
+
}
|
|
412
|
+
const eligibleBacklog = requiredTemporalNumber(
|
|
413
|
+
stats.approximateBacklogCount,
|
|
414
|
+
"approximateBacklogCount",
|
|
415
|
+
{ integer: true },
|
|
416
|
+
);
|
|
417
|
+
let oldestBacklogAgeSeconds = 0;
|
|
418
|
+
if (stats.approximateBacklogAge) {
|
|
419
|
+
const seconds = optionalTemporalNumber(
|
|
420
|
+
stats.approximateBacklogAge.seconds,
|
|
421
|
+
"approximateBacklogAge.seconds",
|
|
422
|
+
{ integer: true },
|
|
423
|
+
);
|
|
424
|
+
const nanos = optionalTemporalNumber(
|
|
425
|
+
stats.approximateBacklogAge.nanos,
|
|
426
|
+
"approximateBacklogAge.nanos",
|
|
427
|
+
{ integer: true },
|
|
428
|
+
);
|
|
429
|
+
if (nanos >= 1_000_000_000) {
|
|
430
|
+
throw new Error("Temporal approximateBacklogAge.nanos must be less than one second");
|
|
431
|
+
}
|
|
432
|
+
oldestBacklogAgeSeconds = seconds + nanos / 1_000_000_000;
|
|
433
|
+
} else if (eligibleBacklog > 0) {
|
|
434
|
+
throw new Error("Temporal stats omitted approximateBacklogAge for a nonzero backlog");
|
|
435
|
+
}
|
|
436
|
+
return {
|
|
437
|
+
eligibleBacklog,
|
|
438
|
+
oldestBacklogAgeSeconds,
|
|
439
|
+
tasksAddRate: optionalTemporalNumber(stats.tasksAddRate, "tasksAddRate"),
|
|
440
|
+
tasksDispatchRate: optionalTemporalNumber(stats.tasksDispatchRate, "tasksDispatchRate"),
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
export function startTurnCapacityMonitor(input: {
|
|
445
|
+
observability: Observability;
|
|
446
|
+
read: () => Promise<TurnTaskQueueStats>;
|
|
447
|
+
intervalMs?: number;
|
|
448
|
+
now?: () => number;
|
|
449
|
+
}): { close: () => Promise<void> } {
|
|
450
|
+
const intervalMs = input.intervalMs ?? 15_000;
|
|
451
|
+
const now = input.now ?? Date.now;
|
|
452
|
+
const startedAt = now();
|
|
453
|
+
let lastSuccessAt: number | null = null;
|
|
454
|
+
let lastReadSucceeded = false;
|
|
455
|
+
let stopped = false;
|
|
456
|
+
let running: Promise<void> | null = null;
|
|
457
|
+
const recordStatus = () => {
|
|
458
|
+
const observedAt = now();
|
|
459
|
+
const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
|
|
460
|
+
const set = (name: string, help: string, value: number) =>
|
|
461
|
+
input.observability.setGauge({ name, help, value });
|
|
462
|
+
set(
|
|
463
|
+
"opengeni_turn_capacity_monitor_last_read_success",
|
|
464
|
+
"Whether the latest Temporal turn-queue capacity read completed successfully.",
|
|
465
|
+
lastReadSucceeded ? 1 : 0,
|
|
466
|
+
);
|
|
467
|
+
set(
|
|
468
|
+
"opengeni_turn_capacity_monitor_last_success_timestamp_seconds",
|
|
469
|
+
"Unix timestamp of the latest successful Temporal turn-queue capacity read, or zero before one succeeds.",
|
|
470
|
+
lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
|
|
471
|
+
);
|
|
472
|
+
set(
|
|
473
|
+
"opengeni_turn_capacity_monitor_last_success_age_seconds",
|
|
474
|
+
"Age of the latest successful Temporal turn-queue capacity read, or monitor age before one succeeds.",
|
|
475
|
+
Math.max(0, successAgeMs / 1_000),
|
|
476
|
+
);
|
|
477
|
+
set(
|
|
478
|
+
"opengeni_turn_capacity_monitor_fresh",
|
|
479
|
+
"Whether eligible-backlog gauges have a successful Temporal read within three monitor intervals.",
|
|
480
|
+
lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
|
|
481
|
+
);
|
|
482
|
+
};
|
|
483
|
+
const refresh = () => {
|
|
484
|
+
// Advance freshness age even while a Temporal read is hung. Old backlog
|
|
485
|
+
// values remain observable for diagnosis but cease to be authoritative.
|
|
486
|
+
recordStatus();
|
|
487
|
+
if (stopped || running) return;
|
|
488
|
+
running = input
|
|
489
|
+
.read()
|
|
490
|
+
.then((stats) => {
|
|
491
|
+
recordTurnTaskQueueStats(input.observability, stats);
|
|
492
|
+
lastSuccessAt = now();
|
|
493
|
+
lastReadSucceeded = true;
|
|
494
|
+
recordStatus();
|
|
495
|
+
})
|
|
496
|
+
.catch((error) => {
|
|
497
|
+
lastReadSucceeded = false;
|
|
498
|
+
recordStatus();
|
|
499
|
+
input.observability.warn("turn capacity monitor: Temporal task-queue stats failed", {
|
|
500
|
+
error: error instanceof Error ? error.message : String(error),
|
|
501
|
+
});
|
|
502
|
+
})
|
|
503
|
+
.finally(() => {
|
|
504
|
+
running = null;
|
|
505
|
+
});
|
|
506
|
+
};
|
|
507
|
+
refresh();
|
|
508
|
+
const timer = setInterval(refresh, intervalMs);
|
|
509
|
+
timer.unref?.();
|
|
510
|
+
return {
|
|
511
|
+
close: async () => {
|
|
512
|
+
stopped = true;
|
|
513
|
+
clearInterval(timer);
|
|
514
|
+
await running;
|
|
515
|
+
},
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
|
|
333
519
|
export function recordSandboxLeaseGauges(
|
|
334
520
|
observability: Observability,
|
|
335
521
|
counts: Partial<Record<SandboxLeaseLiveness, number>>,
|
|
@@ -642,6 +828,39 @@ export function recordCreditMicros(
|
|
|
642
828
|
});
|
|
643
829
|
}
|
|
644
830
|
|
|
831
|
+
const MODEL_REQUEST_PHASE_BUCKETS = [
|
|
832
|
+
0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5, 10, 30, 60, 300,
|
|
833
|
+
];
|
|
834
|
+
|
|
835
|
+
/**
|
|
836
|
+
* Record provider lifecycle diagnostics from the synchronous transport observer.
|
|
837
|
+
* Provider ids come from the resolved provider registry; phase/outcome are closed
|
|
838
|
+
* enums. This function never performs I/O outside the in-memory metrics registry.
|
|
839
|
+
*/
|
|
840
|
+
export function recordModelRequestPhase(
|
|
841
|
+
observability: Observability,
|
|
842
|
+
input: {
|
|
843
|
+
provider: string;
|
|
844
|
+
phase: "headers" | "first_byte" | "terminal";
|
|
845
|
+
outcome?: "completed" | "failed" | "timed_out";
|
|
846
|
+
durationSeconds: number;
|
|
847
|
+
},
|
|
848
|
+
): void {
|
|
849
|
+
const outcome = input.outcome ?? "";
|
|
850
|
+
observability.incrementCounter({
|
|
851
|
+
name: "opengeni_model_request_phases_total",
|
|
852
|
+
help: "Provider model-request lifecycle phases by bounded phase and outcome.",
|
|
853
|
+
labels: { provider: input.provider, phase: input.phase, outcome },
|
|
854
|
+
});
|
|
855
|
+
observability.observeHistogram({
|
|
856
|
+
name: "opengeni_model_request_phase_duration_seconds",
|
|
857
|
+
help: "Monotonic seconds from provider dispatch to a model-request lifecycle phase.",
|
|
858
|
+
buckets: MODEL_REQUEST_PHASE_BUCKETS,
|
|
859
|
+
labels: { provider: input.provider, phase: input.phase },
|
|
860
|
+
value: Math.max(0, input.durationSeconds),
|
|
861
|
+
});
|
|
862
|
+
}
|
|
863
|
+
|
|
645
864
|
// ── Streaming SLIs ────────────────────────────────────────────────────────────
|
|
646
865
|
// Instruments the token-streaming pipeline so "streaming is sluggish" is a number,
|
|
647
866
|
// not a vibe. Split across the three attributable stages so an operator can tell
|
|
@@ -828,6 +1047,40 @@ export function recordContextCompaction(observability: Observability, trigger: s
|
|
|
828
1047
|
});
|
|
829
1048
|
}
|
|
830
1049
|
|
|
1050
|
+
const MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2_048, 8_192];
|
|
1051
|
+
|
|
1052
|
+
/**
|
|
1053
|
+
* Metadata-only Company Brain exposure telemetry. Labels are closed enums and
|
|
1054
|
+
* never include tenant ids, record ids, titles, descriptions, or content.
|
|
1055
|
+
*/
|
|
1056
|
+
export function recordCompanyBrainContributions(
|
|
1057
|
+
observability: Observability,
|
|
1058
|
+
receipt: CompanyBrainContributionReceipt,
|
|
1059
|
+
): void {
|
|
1060
|
+
for (const contribution of receipt.contributions) {
|
|
1061
|
+
const labels = {
|
|
1062
|
+
category: contribution.category,
|
|
1063
|
+
source: contribution.source,
|
|
1064
|
+
inclusion_reason: contribution.inclusionReason,
|
|
1065
|
+
authority_scope: contribution.authorityScope,
|
|
1066
|
+
session_role: receipt.sessionRole,
|
|
1067
|
+
memory_prompt_mode: receipt.memoryPromptMode,
|
|
1068
|
+
};
|
|
1069
|
+
observability.incrementCounter({
|
|
1070
|
+
name: "opengeni_model_context_contributions_total",
|
|
1071
|
+
help: "Company Brain contributions exposed to a model, by bounded reason and authority.",
|
|
1072
|
+
labels,
|
|
1073
|
+
});
|
|
1074
|
+
observability.observeHistogram({
|
|
1075
|
+
name: "opengeni_model_context_contribution_estimated_tokens",
|
|
1076
|
+
help: "Estimated tokens per model-visible Company Brain contribution.",
|
|
1077
|
+
buckets: MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS,
|
|
1078
|
+
labels,
|
|
1079
|
+
value: contribution.estimatedTokens,
|
|
1080
|
+
});
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
|
|
831
1084
|
// ── Prompt-cache efficiency ─────────────────────────────────────────────────
|
|
832
1085
|
// Per model-call prompt-cache signal, provider-labelled only (bounded
|
|
833
1086
|
// cardinality — never a session id or account). `cached_tokens` is the slice of
|
|
@@ -948,6 +1201,37 @@ function incrementBoundedModelCacheCounter(
|
|
|
948
1201
|
totals.set(key, current + input.amount);
|
|
949
1202
|
}
|
|
950
1203
|
|
|
1204
|
+
function nonnegativeFinite(value: number): number {
|
|
1205
|
+
return Number.isFinite(value) && value > 0 ? value : 0;
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
function requiredTemporalNumber(
|
|
1209
|
+
value: unknown,
|
|
1210
|
+
field: string,
|
|
1211
|
+
options: { integer?: boolean } = {},
|
|
1212
|
+
): number {
|
|
1213
|
+
if (value === null || value === undefined) {
|
|
1214
|
+
throw new Error(`Temporal stats omitted required ${field}`);
|
|
1215
|
+
}
|
|
1216
|
+
const parsed = Number(value);
|
|
1217
|
+
if (
|
|
1218
|
+
!Number.isFinite(parsed) ||
|
|
1219
|
+
parsed < 0 ||
|
|
1220
|
+
(options.integer && !Number.isSafeInteger(parsed))
|
|
1221
|
+
) {
|
|
1222
|
+
throw new Error(`Temporal stats returned invalid ${field}: ${String(value)}`);
|
|
1223
|
+
}
|
|
1224
|
+
return parsed;
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
function optionalTemporalNumber(
|
|
1228
|
+
value: unknown,
|
|
1229
|
+
field: string,
|
|
1230
|
+
options: { integer?: boolean } = {},
|
|
1231
|
+
): number {
|
|
1232
|
+
return value === null || value === undefined ? 0 : requiredTemporalNumber(value, field, options);
|
|
1233
|
+
}
|
|
1234
|
+
|
|
951
1235
|
/**
|
|
952
1236
|
* An opaque, stable, non-reversible tag for a codex credential (account) — the
|
|
953
1237
|
* per-call log dimension the account-switch hypothesis correlates against. It is
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
export const SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS = 5 * 60_000;
|
|
2
|
+
export const SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS = 5 * 60_000;
|
|
3
|
+
export const SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS = 65 * 60_000;
|
|
4
|
+
export const SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS = 60_000;
|
|
5
|
+
export const SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS = 20_000;
|
|
6
|
+
export const SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS = 5_000;
|
|
7
|
+
// Match the DB inventory's fair 500-row bound so one sweep never discovers a
|
|
8
|
+
// due box and then needlessly defers it for another schedule period. Five
|
|
9
|
+
// hundred small StartChild commands remain far below Temporal history limits;
|
|
10
|
+
// actual provider I/O stays bounded by worker activity concurrency.
|
|
11
|
+
export const SANDBOX_REAPER_CHILD_DISPATCH_LIMIT = 500;
|
|
12
|
+
export const SANDBOX_REAPER_V2_WORKFLOW_ID = "opengeni-sandbox-lease-reaper-v2";
|
|
13
|
+
export const SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID = "opengeni-sandbox-lease-maintenance-v1";
|
|
14
|
+
|
|
15
|
+
// A protocol-specific queue is the rolling-deploy compatibility boundary for
|
|
16
|
+
// the split reaper activities. An old control worker polls only the base queue,
|
|
17
|
+
// so it can replay already-started legacy workflows but can never receive a new
|
|
18
|
+
// activity type that its binary does not register.
|
|
19
|
+
export const SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX = "-sandbox-lifecycle-v1";
|
|
20
|
+
|
|
21
|
+
export function sandboxLifecycleTaskQueue(baseTaskQueue: string): string {
|
|
22
|
+
return baseTaskQueue.endsWith(SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX)
|
|
23
|
+
? baseTaskQueue
|
|
24
|
+
: `${baseTaskQueue}${SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX}`;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export type SandboxDrainTimeoutClass = "fast" | "extended";
|
|
28
|
+
|
|
29
|
+
export type SandboxDrainTarget = {
|
|
30
|
+
workspaceId: string;
|
|
31
|
+
sandboxGroupId: string;
|
|
32
|
+
instanceId: string | null;
|
|
33
|
+
leaseEpoch: number;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export type SandboxDrainActivityInput = {
|
|
37
|
+
target: SandboxDrainTarget;
|
|
38
|
+
timeoutClass: SandboxDrainTimeoutClass;
|
|
39
|
+
/** Provider operation budget frozen when the durable child is created. A
|
|
40
|
+
* rolling config change cannot make an existing fast child retry forever. */
|
|
41
|
+
snapshotTimeoutMs: number;
|
|
42
|
+
captureTimeoutMs: number;
|
|
43
|
+
/** Stable logical teardown operation; every Temporal activity retry keeps it
|
|
44
|
+
* while receiving a distinct exact capture-attempt id. */
|
|
45
|
+
operationId: string;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export type SandboxDrainWorkflowInput = Omit<SandboxDrainActivityInput, "operationId">;
|
|
49
|
+
|
|
50
|
+
export type SandboxLeaseSweepMaintenanceInput = {
|
|
51
|
+
examined: number;
|
|
52
|
+
started: number;
|
|
53
|
+
alreadyRunning: number;
|
|
54
|
+
startFailed: number;
|
|
55
|
+
rotationsRequested: number;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export function sandboxDrainTimeoutClass(captureTimeoutMs: number): SandboxDrainTimeoutClass {
|
|
59
|
+
return captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS <
|
|
60
|
+
SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS
|
|
61
|
+
? "fast"
|
|
62
|
+
: "extended";
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function sandboxDrainActivityTimeoutMs(timeoutClass: SandboxDrainTimeoutClass): number {
|
|
66
|
+
return timeoutClass === "fast"
|
|
67
|
+
? SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS
|
|
68
|
+
: SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function sandboxDrainWorkflowId(target: SandboxDrainTarget): string {
|
|
72
|
+
return `sandbox-drain:${target.workspaceId}:${target.sandboxGroupId}:${target.leaseEpoch}`;
|
|
73
|
+
}
|