@opengeni/worker-bundle 1.0.0-canary.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +3 -1
- package/dist/activities/agent-turn/admission.d.ts +24 -3
- package/dist/activities/agent-turn/agent-build.d.ts +4 -0
- package/dist/activities/agent-turn/claude-usage-observer.d.ts +58 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +13 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex.d.ts +3 -2
- package/dist/activities/agent-turn/compaction-prep.d.ts +21 -5
- package/dist/activities/agent-turn/credential-leases.d.ts +19 -33
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +82 -11
- package/dist/activities/agent-turn/failure-settlement.d.ts +6 -3
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +17 -0
- package/dist/activities/agent-turn/finalization.d.ts +4 -2
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +9 -2
- package/dist/activities/agent-turn/history-sink.d.ts +2 -0
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +41 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +6 -0
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/quiescence.d.ts +7 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +9 -0
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +8 -3
- package/dist/activities/agent-turn/sandbox-establish.d.ts +2 -1
- package/dist/activities/agent-turn/sandbox-provision.d.ts +3 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -20
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +2 -2
- package/dist/activities/agent-turn/session-title.d.ts +34 -2
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-remove.d.ts +14 -0
- package/dist/activities/agent-turn/skill-search.d.ts +3 -1
- package/dist/activities/agent-turn/skill-tools.d.ts +15 -2
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -3
- package/dist/activities/agent-turn/subscription-lease.d.ts +38 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +56 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +43 -0
- package/dist/activities/agent-turn/turn-context.d.ts +29 -0
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +3 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -1
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +29 -0
- package/dist/activities/codemode-dispatcher.d.ts +9 -2
- package/dist/activities/codex-capacity.d.ts +18 -2
- package/dist/activities/codex-rotation.d.ts +7 -1
- package/dist/activities/context-compaction.d.ts +16 -2
- package/dist/activities/gmail-files.d.ts +32 -0
- package/dist/activities/goals.d.ts +8 -0
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +3 -0
- package/dist/activities/mcp-credentials.d.ts +12 -13
- package/dist/activities/mcp-operation-authority.d.ts +0 -5
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +13 -0
- package/dist/activities/parent-wake.d.ts +6 -12
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/rig-verification.d.ts +2 -3
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +15 -3
- package/dist/activities/sandbox-images.d.ts +1 -1
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-state.d.ts +70 -3
- package/dist/activities/types.d.ts +55 -6
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +46 -0
- package/dist/{activities-control-HMFGSKC5.js → activities-control-BNPTRTOE.js} +1896 -418
- package/dist/activities-control-BNPTRTOE.js.map +1 -0
- package/dist/activities-control.d.ts +182 -4
- package/dist/{activities-turn-AP67JADS.js → activities-turn-5LGZOSVU.js} +4476 -1327
- package/dist/activities-turn-5LGZOSVU.js.map +1 -0
- package/dist/activities.d.ts +182 -4
- package/dist/artifact-materializer-entry.js +5 -2
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +5 -2
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-A4XQWVZK.js → chunk-3ZTRWDMN.js} +307 -4
- package/dist/chunk-3ZTRWDMN.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-CYITU5UH.js → chunk-LJX7Z2KP.js} +1445 -343
- package/dist/chunk-LJX7Z2KP.js.map +1 -0
- package/dist/editable-artifact-materializer-service.d.ts +2 -2
- package/dist/editable-artifact-outbox-service.d.ts +2 -2
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +95 -3
- package/dist/index.js +137 -43
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +118 -8
- package/dist/retained-process-diagnostics.d.ts +4 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-resume.d.ts +55 -13
- package/dist/sandbox-routing.d.ts +5 -3
- package/dist/sandbox-snapshot-diagnostics.d.ts +20 -0
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/warm-capture-holder.d.ts +3 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +273 -41
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +9 -0
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +21 -19
- package/src/activities/agent-run-admission.ts +17 -1
- package/src/activities/agent-turn/admission.ts +66 -3
- package/src/activities/agent-turn/agent-build.ts +84 -10
- package/src/activities/agent-turn/claim.ts +186 -43
- package/src/activities/agent-turn/claude-usage-observer.ts +221 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +172 -8
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex.ts +13 -5
- package/src/activities/agent-turn/compaction-prep.ts +170 -102
- package/src/activities/agent-turn/credential-leases.ts +122 -216
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +647 -48
- package/src/activities/agent-turn/failure-settlement.ts +682 -130
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +120 -0
- package/src/activities/agent-turn/finalization.ts +146 -43
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +69 -28
- package/src/activities/agent-turn/history-sink.ts +16 -2
- package/src/activities/agent-turn/history.ts +38 -11
- package/src/activities/agent-turn/input-wait-reply.ts +103 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/media-artifacts.ts +2 -1
- package/src/activities/agent-turn/model-call-admission.ts +94 -0
- package/src/activities/agent-turn/model-usage.ts +64 -3
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/quiescence.ts +10 -7
- package/src/activities/agent-turn/recovery-warning.ts +16 -0
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +116 -55
- package/src/activities/agent-turn/run.ts +320 -93
- package/src/activities/agent-turn/sandbox-establish.ts +84 -8
- package/src/activities/agent-turn/sandbox-provision.ts +3 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -21
- package/src/activities/agent-turn/sandbox-runtime.ts +343 -18
- package/src/activities/agent-turn/session-title.ts +81 -5
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-remove.ts +46 -0
- package/src/activities/agent-turn/skill-save.ts +1 -1
- package/src/activities/agent-turn/skill-search.ts +10 -1
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +112 -18
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +313 -77
- package/src/activities/agent-turn/subscription-lease.ts +115 -0
- package/src/activities/agent-turn/tool-environment.ts +316 -167
- package/src/activities/agent-turn/tool-policy.ts +80 -1
- package/src/activities/agent-turn/turn-context.ts +35 -0
- package/src/activities/agent-turn/turn-sandbox-access.ts +7 -1
- package/src/activities/agent-turn/xai-capacity.ts +94 -51
- package/src/activities/automations.ts +76 -4
- package/src/activities/browser-deadline-checkpoint.ts +279 -0
- package/src/activities/capabilities.ts +108 -11
- package/src/activities/codemode-dispatcher.ts +36 -8
- package/src/activities/codex-capacity.ts +60 -25
- package/src/activities/codex-rotation.ts +39 -17
- package/src/activities/context-compaction.ts +75 -18
- package/src/activities/gmail-files.ts +88 -0
- package/src/activities/goals.ts +116 -121
- package/src/activities/google-drive-publication.ts +1 -3
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/knowledge-indexing.ts +269 -15
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +17 -0
- package/src/activities/mcp-credentials.ts +83 -194
- package/src/activities/mcp-operation-authority.ts +16 -41
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +15 -2
- package/src/activities/parent-wake.ts +46 -21
- package/src/activities/retained-screenshots.ts +82 -11
- package/src/activities/rig-verification.ts +38 -37
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +59 -17
- package/src/activities/run-input.ts +25 -16
- package/src/activities/sandbox-images.ts +0 -1
- package/src/activities/sandbox-lease.ts +798 -82
- package/src/activities/scheduled-tasks.ts +550 -178
- package/src/activities/session-state.ts +245 -6
- package/src/activities/site-auth-maintenance.ts +3 -0
- package/src/activities/types.ts +72 -5
- package/src/activities/workflow-wake.ts +2 -17
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +283 -0
- package/src/activities-control.ts +4 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +13 -0
- package/src/activity-services.ts +16 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +7 -3
- package/src/editable-artifact-outbox-service.ts +4 -4
- package/src/github-rest-mcp.ts +1 -2
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +3 -0
- package/src/http.ts +4 -2
- package/src/index.ts +99 -37
- package/src/observability-metrics.ts +510 -5
- package/src/personal-github-git-credentials.ts +2 -1
- package/src/pr-review-credentials.ts +1 -3
- package/src/retained-process-diagnostics.ts +50 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-resume.ts +388 -101
- package/src/sandbox-routing.ts +150 -25
- package/src/sandbox-snapshot-diagnostics.ts +112 -0
- package/src/session-event-fanout.ts +23 -0
- package/src/warm-capture-holder.ts +15 -0
- package/src/worker-service-lifecycle.ts +53 -0
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +50 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session.ts +242 -22
- package/src/workflows.ts +4 -0
- package/dist/activities/agent-turn/knowledge-sources.d.ts +0 -20
- package/dist/activities/packs.d.ts +0 -55
- package/dist/activities-control-HMFGSKC5.js.map +0 -1
- package/dist/activities-turn-AP67JADS.js.map +0 -1
- package/dist/chunk-A4XQWVZK.js.map +0 -1
- package/dist/chunk-CYITU5UH.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/knowledge-sources.ts +0 -86
- package/src/activities/packs.ts +0 -166
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
import {
|
|
9
9
|
INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
|
|
10
10
|
MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
11
|
+
agentRunAdmissionDenial,
|
|
11
12
|
assertReusableSessionRevivable,
|
|
12
13
|
captureConnectedCommandOutput,
|
|
13
14
|
createCodexCapacityActivities,
|
|
@@ -23,59 +24,147 @@ import {
|
|
|
23
24
|
notifyParentOfChildIdle,
|
|
24
25
|
persistWorkspaceArchiveCandidate,
|
|
25
26
|
providerRecoveryCountFromMetadata,
|
|
27
|
+
publishDurableSessionEvents,
|
|
26
28
|
putVersion1TarArchiveOrInline,
|
|
27
29
|
reconcileAutomaticSessionTitleFanout,
|
|
28
30
|
reconcilePendingParentSystemUpdates,
|
|
29
31
|
reconcilePendingSessionWorkflowWakes,
|
|
30
32
|
resolveIncidentTelemetryResponderMetadata,
|
|
31
|
-
resolveWorkspacePackRuntime,
|
|
32
33
|
rigProviderImageSourceImage,
|
|
33
34
|
scheduledUserMessagePayload,
|
|
35
|
+
warnDrainSnapshotFailure,
|
|
34
36
|
withFirstPartyTools,
|
|
35
37
|
workflowIdForSession
|
|
36
|
-
} from "./chunk-
|
|
38
|
+
} from "./chunk-LJX7Z2KP.js";
|
|
37
39
|
import {
|
|
38
40
|
inspectOpenSandboxKubernetesInventory,
|
|
41
|
+
recordActiveUserGauges,
|
|
39
42
|
recordCreditBalanceGauges,
|
|
43
|
+
recordCreditGrantGauges,
|
|
40
44
|
recordCreditMicros,
|
|
41
45
|
recordExpiredDrainingSandboxLeaseGauges,
|
|
46
|
+
recordManagedAuthNewSignupsSwitchGauge,
|
|
42
47
|
recordOpenSandboxKubernetesInventoryGauges,
|
|
43
48
|
recordRetainedProcessInventoryGauges,
|
|
44
49
|
recordRetainedProcessReconciliation,
|
|
45
50
|
recordSandboxCheckpointArtifactGauges,
|
|
46
51
|
recordSandboxCheckpointArtifactOutcome,
|
|
52
|
+
recordSandboxCommandContainment,
|
|
47
53
|
recordSandboxDeadlineRotationsRequested,
|
|
48
54
|
recordSandboxInventoryProjectionFailure,
|
|
49
55
|
recordSandboxInventoryProjectionSuccess,
|
|
50
56
|
recordSandboxLeaseGauges,
|
|
51
57
|
recordSandboxOrphansTerminated,
|
|
58
|
+
recordSandboxProviderMissingBeforeCapture,
|
|
59
|
+
recordSandboxRecoveryObservationGauges,
|
|
52
60
|
recordSandboxRotationBacklogGauges,
|
|
53
61
|
recordTurnsQueuedGauge,
|
|
62
|
+
recordVerifiedSignupTrialDeploymentFlagGauge,
|
|
63
|
+
recordVerifiedSignupTrialSwitchGauge,
|
|
54
64
|
recordWorkerDeathRecoveryMetrics,
|
|
55
65
|
runtimeMetricsHooksForObservability
|
|
56
|
-
} from "./chunk-
|
|
66
|
+
} from "./chunk-3ZTRWDMN.js";
|
|
57
67
|
import "./chunk-PZ5AY32C.js";
|
|
58
68
|
|
|
59
69
|
// src/activities/knowledge-indexing.ts
|
|
60
70
|
import { configuredStaticUsageLimits } from "@opengeni/config";
|
|
71
|
+
import { documentEmbeddingCostMicros, paidDocumentEmbedding } from "@opengeni/core";
|
|
61
72
|
import {
|
|
73
|
+
applyCreditDebitAfterUse,
|
|
62
74
|
appendKnowledgeIndexChunks,
|
|
63
75
|
claimKnowledgeIndexJobs,
|
|
76
|
+
checkWorkspaceAllowance,
|
|
64
77
|
completeKnowledgeIndexJob,
|
|
65
78
|
continueKnowledgeIndexJob,
|
|
79
|
+
creditDebitAttributionMetadata,
|
|
66
80
|
deferKnowledgeIndexJob,
|
|
81
|
+
freezeKnowledgeIndexBillingMode,
|
|
67
82
|
getBillingBalance,
|
|
83
|
+
guardPaidKnowledgeIndexPublication,
|
|
84
|
+
knowledgeIndexBillingActivationTime,
|
|
68
85
|
readKnowledgeIndexSource,
|
|
69
86
|
recordUsageEvent,
|
|
70
87
|
sumUsageQuantity,
|
|
71
|
-
withWorkspaceUsageLock
|
|
88
|
+
withWorkspaceUsageLock,
|
|
89
|
+
waitKnowledgeIndexForFunding
|
|
72
90
|
} from "@opengeni/db";
|
|
91
|
+
var KnowledgeIndexUsageLimitError = class extends Error {
|
|
92
|
+
constructor() {
|
|
93
|
+
super("monthly document indexing limit reached");
|
|
94
|
+
this.name = "KnowledgeIndexUsageLimitError";
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
function knowledgeIndexFailureDiagnostic(stage, error) {
|
|
98
|
+
if (error instanceof KnowledgeIndexUsageLimitError) {
|
|
99
|
+
return {
|
|
100
|
+
errorClass: "KnowledgeIndexOperationError",
|
|
101
|
+
errorCode: "knowledge_index_usage_limit_reached",
|
|
102
|
+
origin: "worker"
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
if (stage === "embedding") {
|
|
106
|
+
const status = ownValue(error, "status");
|
|
107
|
+
return {
|
|
108
|
+
errorClass: "KnowledgeIndexOperationError",
|
|
109
|
+
errorCode: "knowledge_index_embedding_failed",
|
|
110
|
+
origin: "worker",
|
|
111
|
+
...typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599 ? { status } : {}
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
const postgres = postgresErrorState(error);
|
|
115
|
+
if (!postgres) {
|
|
116
|
+
return {
|
|
117
|
+
errorClass: "KnowledgeIndexOperationError",
|
|
118
|
+
errorCode: "knowledge_index_failed",
|
|
119
|
+
origin: "worker"
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
return {
|
|
123
|
+
errorClass: "KnowledgeIndexOperationError",
|
|
124
|
+
errorCode: "knowledge_index_persistence_failed",
|
|
125
|
+
origin: "db",
|
|
126
|
+
...postgres.sqlState ? { sqlState: postgres.sqlState } : {}
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
function ownValue(value, key) {
|
|
130
|
+
try {
|
|
131
|
+
if (!value || typeof value !== "object") return void 0;
|
|
132
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
133
|
+
return descriptor && "value" in descriptor ? descriptor.value : void 0;
|
|
134
|
+
} catch {
|
|
135
|
+
return void 0;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
function postgresErrorState(error) {
|
|
139
|
+
let current = error;
|
|
140
|
+
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
|
|
141
|
+
if (ownValue(current, "name") === "PostgresError") {
|
|
142
|
+
const code = ownValue(current, "code");
|
|
143
|
+
return typeof code === "string" && /^[0-9A-Z]{5}$/.test(code) ? { sqlState: code } : {};
|
|
144
|
+
}
|
|
145
|
+
current = ownValue(current, "cause");
|
|
146
|
+
}
|
|
147
|
+
return void 0;
|
|
148
|
+
}
|
|
73
149
|
function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
150
|
+
let activationTime;
|
|
74
151
|
return {
|
|
75
152
|
indexKnowledge: async () => {
|
|
76
153
|
const result = { completed: 0, advanced: 0, deferred: 0, unavailable: 0 };
|
|
77
154
|
if (!resolveDocumentServices) return result;
|
|
78
155
|
const { db, settings, observability } = await services();
|
|
156
|
+
let policyActivatedAt;
|
|
157
|
+
if (paidDocumentEmbedding(settings)) {
|
|
158
|
+
if (!settings.documentEmbeddingCreditsActivatedAt)
|
|
159
|
+
throw new Error("paid Knowledge embedding requires an activation cutoff");
|
|
160
|
+
policyActivatedAt = new Date(settings.documentEmbeddingCreditsActivatedAt);
|
|
161
|
+
} else {
|
|
162
|
+
activationTime ??= knowledgeIndexBillingActivationTime(db).catch((error) => {
|
|
163
|
+
activationTime = void 0;
|
|
164
|
+
throw error;
|
|
165
|
+
});
|
|
166
|
+
policyActivatedAt = await activationTime;
|
|
167
|
+
}
|
|
79
168
|
const { embedder } = await resolveDocumentServices();
|
|
80
169
|
const { knowledgeIndexChunks } = await import("@opengeni/documents");
|
|
81
170
|
const claims = await claimKnowledgeIndexJobs(db, {
|
|
@@ -84,6 +173,7 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
84
173
|
limit: 2
|
|
85
174
|
});
|
|
86
175
|
for (const claim of claims) {
|
|
176
|
+
let stage = "processing";
|
|
87
177
|
try {
|
|
88
178
|
const source = await readKnowledgeIndexSource(db, claim);
|
|
89
179
|
if (!source) {
|
|
@@ -107,9 +197,57 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
107
197
|
chunks.push(chunk);
|
|
108
198
|
}
|
|
109
199
|
if (chunks.length) {
|
|
110
|
-
|
|
200
|
+
const frozenPolicy = await freezeKnowledgeIndexBillingMode(
|
|
201
|
+
lockedDb,
|
|
202
|
+
claim,
|
|
203
|
+
// Deterministic embeddings incur no provider charge; a valid
|
|
204
|
+
// credits-mode config with zero tariff must still index them.
|
|
205
|
+
// OpenAI shadow keeps its price snapshot for internal estimates.
|
|
206
|
+
settings.documentEmbeddingProvider === "openai" ? settings.documentEmbeddingBillingMode ?? "usage_only" : "usage_only",
|
|
207
|
+
policyActivatedAt,
|
|
208
|
+
settings.documentEmbeddingRateMicrosPerMillionBytes ?? 0
|
|
209
|
+
);
|
|
210
|
+
if (frozenPolicy.mode === "awaiting_review") {
|
|
211
|
+
result.deferred++;
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
if (frozenPolicy.mode === "obsolete") {
|
|
215
|
+
result.unavailable++;
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (frozenPolicy.mode === "credits" && !paidDocumentEmbedding(settings)) {
|
|
219
|
+
await deferKnowledgeIndexJob(lockedDb, claim);
|
|
220
|
+
result.deferred++;
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
const paid = frozenPolicy.mode === "credits";
|
|
224
|
+
if (paid && claim.billingAttribution.kind === "unknown") {
|
|
225
|
+
observability.warn("Paid Knowledge indexing awaits initiating attribution", {
|
|
226
|
+
errorCode: "knowledge_index_attribution_unavailable"
|
|
227
|
+
});
|
|
228
|
+
await deferKnowledgeIndexJob(lockedDb, claim);
|
|
229
|
+
result.deferred++;
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
if (paid && current.nextIndex === 0) {
|
|
111
233
|
const balance = await getBillingBalance(lockedDb, claim.accountId);
|
|
112
|
-
if (balance.balanceMicros <= 0)
|
|
234
|
+
if (balance.balanceMicros <= 0) {
|
|
235
|
+
await waitKnowledgeIndexForFunding(lockedDb, claim);
|
|
236
|
+
result.deferred++;
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
if (paid) {
|
|
241
|
+
const refusal = await checkWorkspaceAllowance(lockedDb, {
|
|
242
|
+
accountId: claim.accountId,
|
|
243
|
+
workspaceId: current.billingWorkspaceId,
|
|
244
|
+
subjectId: claim.billingAttribution.kind === "turn" || claim.billingAttribution.kind === "human" ? claim.billingAttribution.initiatingHumanSubjectId : null
|
|
245
|
+
});
|
|
246
|
+
if (refusal) {
|
|
247
|
+
await deferKnowledgeIndexJob(lockedDb, claim);
|
|
248
|
+
result.deferred++;
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
113
251
|
}
|
|
114
252
|
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
115
253
|
const limit = configuredStaticUsageLimits(settings).maxDocumentIndexedChunksPerWorkspace;
|
|
@@ -120,13 +258,27 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
120
258
|
eventType: "document.indexed",
|
|
121
259
|
since: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1))
|
|
122
260
|
});
|
|
123
|
-
if (used + chunks.length > limit)
|
|
124
|
-
throw new Error("monthly document indexing limit reached");
|
|
261
|
+
if (used + chunks.length > limit) throw new KnowledgeIndexUsageLimitError();
|
|
125
262
|
}
|
|
126
263
|
}
|
|
127
|
-
const
|
|
264
|
+
const inputs = chunks.map((chunk) => chunk.embeddingInput);
|
|
265
|
+
const bytes = inputs.reduce(
|
|
266
|
+
(sum, input) => sum + Buffer.byteLength(input, "utf8"),
|
|
267
|
+
0
|
|
268
|
+
);
|
|
269
|
+
stage = "embedding";
|
|
270
|
+
const vectors = await embedder.embedMany(inputs);
|
|
128
271
|
if (vectors.length !== chunks.length)
|
|
129
272
|
throw new Error("Incomplete Knowledge embeddings");
|
|
273
|
+
stage = "processing";
|
|
274
|
+
if (paid) {
|
|
275
|
+
const publication = await guardPaidKnowledgeIndexPublication(lockedDb, claim);
|
|
276
|
+
if (publication !== "published") {
|
|
277
|
+
if (publication === "obsolete") result.unavailable++;
|
|
278
|
+
else result.deferred++;
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
130
282
|
const appended = await appendKnowledgeIndexChunks(
|
|
131
283
|
lockedDb,
|
|
132
284
|
claim,
|
|
@@ -147,6 +299,62 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
147
299
|
sourceResourceId: claim.revisionId,
|
|
148
300
|
idempotencyKey: `knowledge.indexed:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
149
301
|
});
|
|
302
|
+
await recordUsageEvent(lockedDb, {
|
|
303
|
+
accountId: claim.accountId,
|
|
304
|
+
workspaceId: current.billingWorkspaceId,
|
|
305
|
+
eventType: "document.embedding_bytes",
|
|
306
|
+
quantity: bytes,
|
|
307
|
+
unit: "byte",
|
|
308
|
+
sourceResourceType: "knowledge_revision",
|
|
309
|
+
sourceResourceId: claim.revisionId,
|
|
310
|
+
idempotencyKey: `knowledge.embedding_bytes:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
311
|
+
});
|
|
312
|
+
if (frozenPolicy.mode === "shadow" && frozenPolicy.rateMicrosPerMillionBytes > 0) {
|
|
313
|
+
const estimate = documentEmbeddingCostMicros(
|
|
314
|
+
{
|
|
315
|
+
...settings,
|
|
316
|
+
documentEmbeddingRateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
317
|
+
},
|
|
318
|
+
bytes
|
|
319
|
+
);
|
|
320
|
+
if (estimate > 0)
|
|
321
|
+
await recordUsageEvent(lockedDb, {
|
|
322
|
+
accountId: claim.accountId,
|
|
323
|
+
workspaceId: current.billingWorkspaceId,
|
|
324
|
+
eventType: "document.embedding_shadow_estimate",
|
|
325
|
+
quantity: estimate,
|
|
326
|
+
unit: "micro_usd",
|
|
327
|
+
sourceResourceType: "knowledge_revision",
|
|
328
|
+
sourceResourceId: claim.revisionId,
|
|
329
|
+
idempotencyKey: `knowledge.embedding_shadow:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
if (paid) {
|
|
333
|
+
const cost = documentEmbeddingCostMicros(
|
|
334
|
+
{
|
|
335
|
+
...settings,
|
|
336
|
+
documentEmbeddingRateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
337
|
+
},
|
|
338
|
+
bytes
|
|
339
|
+
);
|
|
340
|
+
if (cost > 0)
|
|
341
|
+
await applyCreditDebitAfterUse(lockedDb, {
|
|
342
|
+
accountId: claim.accountId,
|
|
343
|
+
workspaceId: current.billingWorkspaceId,
|
|
344
|
+
type: "document_embedding_debit",
|
|
345
|
+
amountMicros: cost,
|
|
346
|
+
sourceType: "knowledge_revision",
|
|
347
|
+
sourceId: claim.revisionId,
|
|
348
|
+
idempotencyKey: `knowledge.embedding:${claim.revisionId}:${claim.generation}:${current.nextIndex}`,
|
|
349
|
+
metadata: {
|
|
350
|
+
...creditDebitAttributionMetadata(claim.billingAttribution),
|
|
351
|
+
model: claim.model,
|
|
352
|
+
bytes,
|
|
353
|
+
chunks: chunks.length,
|
|
354
|
+
rateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
355
|
+
}
|
|
356
|
+
});
|
|
357
|
+
}
|
|
150
358
|
}
|
|
151
359
|
if (more) {
|
|
152
360
|
await continueKnowledgeIndexJob(lockedDb, claim);
|
|
@@ -161,14 +369,19 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
161
369
|
else result.unavailable++;
|
|
162
370
|
}
|
|
163
371
|
});
|
|
164
|
-
} catch {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
372
|
+
} catch (error) {
|
|
373
|
+
observability.warn(
|
|
374
|
+
"Knowledge indexing batch deferred",
|
|
375
|
+
knowledgeIndexFailureDiagnostic(stage, error)
|
|
376
|
+
);
|
|
377
|
+
await deferKnowledgeIndexJob(db, claim).catch((deferError) => {
|
|
378
|
+
const deferDiagnostic = knowledgeIndexFailureDiagnostic("processing", deferError);
|
|
379
|
+
observability.warn("Knowledge indexing batch deferral failed", {
|
|
380
|
+
...deferDiagnostic,
|
|
381
|
+
errorCode: "knowledge_index_defer_failed"
|
|
382
|
+
});
|
|
171
383
|
});
|
|
384
|
+
result.deferred++;
|
|
172
385
|
}
|
|
173
386
|
}
|
|
174
387
|
return result;
|
|
@@ -260,6 +473,240 @@ function cleanupErrorCategory(error) {
|
|
|
260
473
|
return typeof code === "string" && SAFE_PROVIDER_ERROR_CODES.has(code) ? code : "unknown";
|
|
261
474
|
}
|
|
262
475
|
|
|
476
|
+
// src/activities/browser-deadline-checkpoint.ts
|
|
477
|
+
import { Context } from "@temporalio/activity";
|
|
478
|
+
import {
|
|
479
|
+
environmentsEncryptionKeyBytes,
|
|
480
|
+
resolveFirstPartyDelegationSecret
|
|
481
|
+
} from "@opengeni/config";
|
|
482
|
+
import { BROWSER_STATE_ARTIFACT_CONTENT_TYPE } from "@opengeni/contracts";
|
|
483
|
+
import {
|
|
484
|
+
browserDeadlineCheckpoint,
|
|
485
|
+
clearSuspendedBrowserSessionController,
|
|
486
|
+
commitBrowserSessionSuspension,
|
|
487
|
+
dispatchBrowserSessionOperation,
|
|
488
|
+
failBrowserSessionSuspension,
|
|
489
|
+
listBrowserDeadlineCheckpoints,
|
|
490
|
+
readLease,
|
|
491
|
+
releaseLeaseHolder,
|
|
492
|
+
withRlsContext
|
|
493
|
+
} from "@opengeni/db";
|
|
494
|
+
import { deriveBrowserControllerAdminToken } from "@opengeni/interaction/browser-controller-authority";
|
|
495
|
+
import {
|
|
496
|
+
browserStateArtifactAad,
|
|
497
|
+
browserStateManifestDigest,
|
|
498
|
+
browserStateObjectKey,
|
|
499
|
+
deriveBrowserStateDataKey,
|
|
500
|
+
wrapBrowserStateDataKey
|
|
501
|
+
} from "@opengeni/interaction/browser-state-authority";
|
|
502
|
+
import {
|
|
503
|
+
BrowserControlClient,
|
|
504
|
+
BrowserControlRequestError,
|
|
505
|
+
createSandboxClientForBackend,
|
|
506
|
+
deserializeSandboxSessionStateEnvelope,
|
|
507
|
+
resumeExactSandboxSession
|
|
508
|
+
} from "@opengeni/runtime";
|
|
509
|
+
var UPLOAD_CLEANUP_GRACE_MS = 24 * 60 * 6e4;
|
|
510
|
+
function heartbeat() {
|
|
511
|
+
try {
|
|
512
|
+
Context.current().heartbeat();
|
|
513
|
+
} catch {
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
async function withActivityHeartbeat(run) {
|
|
517
|
+
heartbeat();
|
|
518
|
+
const timer = setInterval(heartbeat, 5e3);
|
|
519
|
+
try {
|
|
520
|
+
return await run();
|
|
521
|
+
} finally {
|
|
522
|
+
clearInterval(timer);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
async function resumeCheckpointController({
|
|
526
|
+
target,
|
|
527
|
+
lease,
|
|
528
|
+
settings,
|
|
529
|
+
controllerRoot
|
|
530
|
+
}) {
|
|
531
|
+
const provider = createSandboxClientForBackend("modal", settings);
|
|
532
|
+
const envelope = lease.resumeState;
|
|
533
|
+
const state = await deserializeSandboxSessionStateEnvelope(
|
|
534
|
+
provider,
|
|
535
|
+
envelope.sessionState ?? lease.resumeState,
|
|
536
|
+
target.instanceId
|
|
537
|
+
);
|
|
538
|
+
const resumed = await resumeExactSandboxSession(provider, "modal", state, target.instanceId);
|
|
539
|
+
return new BrowserControlClient(resumed.session, {
|
|
540
|
+
adminToken: deriveBrowserControllerAdminToken({
|
|
541
|
+
rootSecret: controllerRoot,
|
|
542
|
+
accountId: target.accountId,
|
|
543
|
+
workspaceId: target.workspaceId,
|
|
544
|
+
placement: { kind: "sandbox_group", sandboxGroupId: target.sandboxGroupId },
|
|
545
|
+
placementInstanceId: target.instanceId
|
|
546
|
+
})
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
function createBrowserDeadlineCheckpointActivities(services, connectController = resumeCheckpointController) {
|
|
550
|
+
return {
|
|
551
|
+
async listDueBrowserCheckpoints() {
|
|
552
|
+
return await withActivityHeartbeat(async () => {
|
|
553
|
+
const { db, settings, objectStorage } = await services();
|
|
554
|
+
if (!settings.sandboxOwnershipEnabled || !objectStorage || !resolveFirstPartyDelegationSecret(settings))
|
|
555
|
+
return [];
|
|
556
|
+
const root = environmentsEncryptionKeyBytes(settings);
|
|
557
|
+
if (!root) return [];
|
|
558
|
+
root.fill(0);
|
|
559
|
+
return await listBrowserDeadlineCheckpoints(db);
|
|
560
|
+
});
|
|
561
|
+
},
|
|
562
|
+
async checkpointBrowserBeforeDeadline(target) {
|
|
563
|
+
return await withActivityHeartbeat(async () => {
|
|
564
|
+
const { db, settings, objectStorage } = await services();
|
|
565
|
+
const rootKey = environmentsEncryptionKeyBytes(settings);
|
|
566
|
+
const controllerRoot = resolveFirstPartyDelegationSecret(settings);
|
|
567
|
+
if (!rootKey || !objectStorage || !controllerRoot) {
|
|
568
|
+
rootKey?.fill(0);
|
|
569
|
+
throw new Error("browser deadline checkpoint storage or encryption is unavailable");
|
|
570
|
+
}
|
|
571
|
+
let dataKey = null;
|
|
572
|
+
let aad = null;
|
|
573
|
+
let timer = null;
|
|
574
|
+
try {
|
|
575
|
+
const claim = await browserDeadlineCheckpoint(db, target, { prepare: true, touch: true });
|
|
576
|
+
if (!claim) return { status: "skipped" };
|
|
577
|
+
let pulseRunning = false;
|
|
578
|
+
let authorityLost = false;
|
|
579
|
+
const pulse = async () => {
|
|
580
|
+
if (pulseRunning) return;
|
|
581
|
+
pulseRunning = true;
|
|
582
|
+
try {
|
|
583
|
+
if (!await browserDeadlineCheckpoint(db, target, { touch: true }))
|
|
584
|
+
authorityLost = true;
|
|
585
|
+
} finally {
|
|
586
|
+
pulseRunning = false;
|
|
587
|
+
}
|
|
588
|
+
};
|
|
589
|
+
timer = setInterval(() => {
|
|
590
|
+
void pulse().catch(() => void 0);
|
|
591
|
+
}, 5e3);
|
|
592
|
+
const lease = await readLease(db, target.workspaceId, target.sandboxGroupId);
|
|
593
|
+
if (!lease || lease.instanceId !== target.instanceId || lease.leaseEpoch !== target.leaseEpoch || !lease.resumeState) {
|
|
594
|
+
throw new Error("browser deadline checkpoint placement changed");
|
|
595
|
+
}
|
|
596
|
+
const placement = {
|
|
597
|
+
kind: "sandbox_group",
|
|
598
|
+
sandboxGroupId: target.sandboxGroupId
|
|
599
|
+
};
|
|
600
|
+
const client = await connectController({ target, lease, settings, controllerRoot });
|
|
601
|
+
if (authorityLost || !await browserDeadlineCheckpoint(db, target, { touch: true })) {
|
|
602
|
+
throw new Error("browser deadline checkpoint authority changed");
|
|
603
|
+
}
|
|
604
|
+
if (claim.state !== "completed") {
|
|
605
|
+
const scope = { ...target, operationId: claim.operationId };
|
|
606
|
+
const objectKey = browserStateObjectKey(target.workspaceId, claim.operationId);
|
|
607
|
+
dataKey = deriveBrowserStateDataKey(rootKey, { ...scope, objectKey });
|
|
608
|
+
aad = browserStateArtifactAad({ ...scope, objectKey });
|
|
609
|
+
await dispatchBrowserSessionOperation(db, {
|
|
610
|
+
...scope,
|
|
611
|
+
deadlineTarget: target,
|
|
612
|
+
stateUpload: {
|
|
613
|
+
objectKey,
|
|
614
|
+
cleanupAfter: new Date(Date.now() + UPLOAD_CLEANUP_GRACE_MS)
|
|
615
|
+
}
|
|
616
|
+
});
|
|
617
|
+
const signed = await objectStorage.createPutUrl({
|
|
618
|
+
key: objectKey,
|
|
619
|
+
contentType: BROWSER_STATE_ARTIFACT_CONTENT_TYPE,
|
|
620
|
+
audience: "sandbox"
|
|
621
|
+
});
|
|
622
|
+
const receipt = await client.captureState({
|
|
623
|
+
...scope,
|
|
624
|
+
objectKey,
|
|
625
|
+
afterCapture: "stop",
|
|
626
|
+
dataKey,
|
|
627
|
+
aad,
|
|
628
|
+
upload: {
|
|
629
|
+
url: signed.url,
|
|
630
|
+
requiredHeaders: signed.requiredHeaders,
|
|
631
|
+
expiresAt: signed.expiresAt.toISOString()
|
|
632
|
+
}
|
|
633
|
+
}).catch(async (error) => {
|
|
634
|
+
if (error instanceof BrowserControlRequestError && (error.error.code === "outcome_unknown" || !error.error.retryable)) {
|
|
635
|
+
await failBrowserSessionSuspension(db, { ...scope, error: error.error });
|
|
636
|
+
}
|
|
637
|
+
throw error;
|
|
638
|
+
});
|
|
639
|
+
if (authorityLost) throw new Error("browser deadline checkpoint authority changed");
|
|
640
|
+
const portable = receipt.manifest.profileCrypto !== "platform_bound";
|
|
641
|
+
await commitBrowserSessionSuspension(db, {
|
|
642
|
+
...scope,
|
|
643
|
+
deadlineTarget: target,
|
|
644
|
+
artifact: {
|
|
645
|
+
kind: "chromium_profile",
|
|
646
|
+
format: receipt.format,
|
|
647
|
+
objectKey,
|
|
648
|
+
artifactDigest: receipt.artifactDigest,
|
|
649
|
+
contentDigest: receipt.contentDigest,
|
|
650
|
+
manifestDigest: browserStateManifestDigest(receipt.manifest),
|
|
651
|
+
sizeBytes: receipt.sizeBytes,
|
|
652
|
+
encryptedDataKey: wrapBrowserStateDataKey(rootKey, dataKey, {
|
|
653
|
+
...scope,
|
|
654
|
+
objectKey,
|
|
655
|
+
artifactDigest: receipt.artifactDigest,
|
|
656
|
+
contentDigest: receipt.contentDigest
|
|
657
|
+
}),
|
|
658
|
+
materialization: {
|
|
659
|
+
portability: portable ? "portable" : "placement_bound",
|
|
660
|
+
reason: portable ? null : "Profile encryption depends on the source operating-system credential store.",
|
|
661
|
+
platform: receipt.manifest.platform,
|
|
662
|
+
architecture: receipt.manifest.architecture,
|
|
663
|
+
engine: receipt.manifest.engine,
|
|
664
|
+
engineVersion: receipt.manifest.engineVersion,
|
|
665
|
+
driverId: receipt.manifest.driverId,
|
|
666
|
+
driverSchemaVersion: receipt.manifest.driverSchemaVersion,
|
|
667
|
+
profileCrypto: receipt.manifest.profileCrypto,
|
|
668
|
+
providerId: null,
|
|
669
|
+
placement: portable ? null : placement
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
});
|
|
673
|
+
}
|
|
674
|
+
await client.endSession(target, { removeState: true }).catch((error) => {
|
|
675
|
+
if (!(error instanceof BrowserControlRequestError && error.status === 404)) throw error;
|
|
676
|
+
});
|
|
677
|
+
await withRlsContext(
|
|
678
|
+
db,
|
|
679
|
+
target,
|
|
680
|
+
async (tx) => {
|
|
681
|
+
if ((await browserDeadlineCheckpoint(tx, target))?.state !== "completed") {
|
|
682
|
+
throw new Error("browser deadline checkpoint cleanup authority changed");
|
|
683
|
+
}
|
|
684
|
+
await clearSuspendedBrowserSessionController(tx, {
|
|
685
|
+
...target,
|
|
686
|
+
expectedControllerGeneration: target.controllerGeneration
|
|
687
|
+
});
|
|
688
|
+
await releaseLeaseHolder(tx, {
|
|
689
|
+
...target,
|
|
690
|
+
kind: "interaction",
|
|
691
|
+
holderId: `browser-session:${target.browserSessionId}`,
|
|
692
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
693
|
+
});
|
|
694
|
+
},
|
|
695
|
+
void 0,
|
|
696
|
+
"none"
|
|
697
|
+
);
|
|
698
|
+
return { status: "suspended" };
|
|
699
|
+
} finally {
|
|
700
|
+
if (timer) clearInterval(timer);
|
|
701
|
+
dataKey?.fill(0);
|
|
702
|
+
aad?.fill(0);
|
|
703
|
+
rootKey.fill(0);
|
|
704
|
+
}
|
|
705
|
+
});
|
|
706
|
+
}
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
|
|
263
710
|
// src/activities/automations.ts
|
|
264
711
|
import {
|
|
265
712
|
resolveTurnExecutionPolicyV1,
|
|
@@ -267,88 +714,30 @@ import {
|
|
|
267
714
|
WORKSPACE_OPENROUTER_MODEL_ID_PREFIX
|
|
268
715
|
} from "@opengeni/config";
|
|
269
716
|
import {
|
|
717
|
+
applySessionAgentConfigWriteThrough,
|
|
270
718
|
assertWorkspaceModelPolicyAllows,
|
|
271
719
|
canonicalConfiguredModel,
|
|
720
|
+
resolveSessionAgentConfigForCreate,
|
|
272
721
|
createAndStartSessionWithOutcome,
|
|
273
722
|
recordWorkspaceUsage,
|
|
274
723
|
resolveCatalogSettings,
|
|
275
|
-
resolveWorkspaceCatalogSettings
|
|
724
|
+
resolveWorkspaceCatalogSettings,
|
|
725
|
+
isDeveloperSetupDelegatedPermissionAllowed
|
|
276
726
|
} from "@opengeni/core";
|
|
277
727
|
import {
|
|
278
728
|
AutomationAuthorityRevokedError,
|
|
279
729
|
assertAutomationRunAuthorityInTransaction,
|
|
280
730
|
claimAutomationRun,
|
|
281
|
-
settleAutomationRun
|
|
731
|
+
settleAutomationRun,
|
|
732
|
+
requireWorkspace
|
|
282
733
|
} from "@opengeni/db";
|
|
283
734
|
import { SessionSkills } from "@opengeni/contracts";
|
|
284
|
-
|
|
285
|
-
// src/activities/agent-run-admission.ts
|
|
286
|
-
import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
|
|
287
|
-
import { modelFundingForAdmission } from "@opengeni/core";
|
|
288
|
-
import { getBillingBalance as getBillingBalance2, isCodexBilledTurn, sumUsageQuantity as sumUsageQuantity2 } from "@opengeni/db";
|
|
289
|
-
async function agentRunAdmissionDenial(services, input) {
|
|
290
|
-
const codexBilled = await isCodexBilledTurn({
|
|
291
|
-
db: services.db,
|
|
292
|
-
settings: services.settings,
|
|
293
|
-
workspaceId: input.workspaceId,
|
|
294
|
-
model: input.model
|
|
295
|
-
});
|
|
296
|
-
const externallyBilled = modelFundingForAdmission(
|
|
297
|
-
services.settings,
|
|
298
|
-
input.model,
|
|
299
|
-
codexBilled
|
|
300
|
-
).fundedWithoutCredits;
|
|
301
|
-
if (!externallyBilled && (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")) {
|
|
302
|
-
if (services.entitlements) {
|
|
303
|
-
const decision = await services.entitlements.admitRun({
|
|
304
|
-
accountId: input.accountId,
|
|
305
|
-
workspaceId: input.workspaceId,
|
|
306
|
-
action: "agent_run:create",
|
|
307
|
-
quantity: input.requestedAgentRuns
|
|
308
|
-
});
|
|
309
|
-
if (!decision.allowed) return "insufficient_credits";
|
|
310
|
-
} else {
|
|
311
|
-
const balance = await getBillingBalance2(services.db, input.accountId);
|
|
312
|
-
if (balance.balanceMicros <= 0) return "insufficient_credits";
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
if (services.settings.usageLimitsMode !== "static" && services.settings.usageLimitsMode !== "managed") {
|
|
316
|
-
return null;
|
|
317
|
-
}
|
|
318
|
-
const limits = configuredStaticUsageLimits2(services.settings);
|
|
319
|
-
if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
|
|
320
|
-
const used = await sumUsageQuantity2(services.db, {
|
|
321
|
-
accountId: input.accountId,
|
|
322
|
-
eventType: "model.cost",
|
|
323
|
-
since: startOfUtcMonth()
|
|
324
|
-
});
|
|
325
|
-
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
326
|
-
return "monthly_model_cost_limit";
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
330
|
-
const used = await sumUsageQuantity2(services.db, {
|
|
331
|
-
workspaceId: input.workspaceId,
|
|
332
|
-
eventType: "agent_run.created",
|
|
333
|
-
since: startOfUtcMonth()
|
|
334
|
-
});
|
|
335
|
-
if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
336
|
-
return "monthly_agent_run_limit";
|
|
337
|
-
}
|
|
338
|
-
}
|
|
339
|
-
return null;
|
|
340
|
-
}
|
|
341
|
-
function startOfUtcMonth() {
|
|
342
|
-
const now = /* @__PURE__ */ new Date();
|
|
343
|
-
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
// src/activities/automations.ts
|
|
347
735
|
function createAutomationActivities(services, options = {}) {
|
|
348
736
|
const claim = options.claim ?? claimAutomationRun;
|
|
349
737
|
const settle = options.settle ?? settleAutomationRun;
|
|
350
738
|
const assertAuthority = options.assertAuthority ?? assertAutomationRunAuthorityInTransaction;
|
|
351
739
|
const createSession2 = options.createSession ?? createAndStartSessionWithOutcome;
|
|
740
|
+
const readWorkspace = options.readWorkspace ?? requireWorkspace;
|
|
352
741
|
const admit = options.admit ?? agentRunAdmissionDenial;
|
|
353
742
|
const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows;
|
|
354
743
|
const recordUsage = options.recordUsage ?? recordWorkspaceUsage;
|
|
@@ -393,6 +782,17 @@ function createAutomationActivities(services, options = {}) {
|
|
|
393
782
|
return { action: "skipped", reason: "accepted_execution_mismatch" };
|
|
394
783
|
}
|
|
395
784
|
const template = accepted.sessionTemplate;
|
|
785
|
+
if (template.credentialRestriction === "developer_setup" && template.firstPartyMcpPermissions.some(
|
|
786
|
+
(permission) => !isDeveloperSetupDelegatedPermissionAllowed(permission)
|
|
787
|
+
)) {
|
|
788
|
+
await settle(service.db, {
|
|
789
|
+
workspaceId: input.workspaceId,
|
|
790
|
+
runId: run.id,
|
|
791
|
+
status: "failed",
|
|
792
|
+
errorCode: "credential_restriction_violation"
|
|
793
|
+
});
|
|
794
|
+
return { action: "failed", reason: "credential_restriction_violation" };
|
|
795
|
+
}
|
|
396
796
|
const requestedModel = template.model ?? deploymentCatalogSettings.openaiModel;
|
|
397
797
|
const catalogSettings = requestedModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) ? (await resolveWorkspaceCatalogSettings(service.db, catalogSourceSettings, {
|
|
398
798
|
accountId: input.accountId,
|
|
@@ -445,7 +845,42 @@ function createAutomationActivities(services, options = {}) {
|
|
|
445
845
|
reason: "interactive_compute_not_allowed"
|
|
446
846
|
};
|
|
447
847
|
}
|
|
448
|
-
|
|
848
|
+
let agentWriteThrough;
|
|
849
|
+
try {
|
|
850
|
+
const workspaceSettings = (await readWorkspace(service.db, input.workspaceId)).settings;
|
|
851
|
+
const resolution = resolveSessionAgentConfigForCreate({
|
|
852
|
+
settings: catalogSettings,
|
|
853
|
+
creator: "automation",
|
|
854
|
+
request: template.agent,
|
|
855
|
+
instructions: template.instructions ?? void 0,
|
|
856
|
+
workspaceSettings,
|
|
857
|
+
parent: null,
|
|
858
|
+
goal: false
|
|
859
|
+
});
|
|
860
|
+
const written = applySessionAgentConfigWriteThrough({
|
|
861
|
+
config: resolution.config,
|
|
862
|
+
firstPartyMcpTools: template.firstPartyMcpTools,
|
|
863
|
+
tools: template.tools,
|
|
864
|
+
toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
|
|
865
|
+
productServerIds: template.tools.map((tool) => tool.id)
|
|
866
|
+
});
|
|
867
|
+
agentWriteThrough = {
|
|
868
|
+
config: resolution.config,
|
|
869
|
+
instructions: resolution.instructions,
|
|
870
|
+
firstPartyMcpTools: written.firstPartyMcpTools,
|
|
871
|
+
tools: written.tools
|
|
872
|
+
};
|
|
873
|
+
} catch (error) {
|
|
874
|
+
if (!isUnprocessableEntity(error)) throw error;
|
|
875
|
+
await settle(service.db, {
|
|
876
|
+
workspaceId: input.workspaceId,
|
|
877
|
+
runId: run.id,
|
|
878
|
+
status: "failed",
|
|
879
|
+
errorCode: "dispatch_failed"
|
|
880
|
+
});
|
|
881
|
+
return { action: "failed", reason: "dispatch_failed" };
|
|
882
|
+
}
|
|
883
|
+
const resolvedTurnExecutionPolicy = resolveTurnExecutionPolicyV1(catalogSettings, {
|
|
449
884
|
modelId: model,
|
|
450
885
|
requestedModelId: template.model,
|
|
451
886
|
modelSource: template.model ? "explicit" : "deployment",
|
|
@@ -454,6 +889,10 @@ function createAutomationActivities(services, options = {}) {
|
|
|
454
889
|
latencyMode: "standard",
|
|
455
890
|
latencyModeSource: "deployment"
|
|
456
891
|
});
|
|
892
|
+
const turnExecutionPolicy = template.credentialRestriction ? {
|
|
893
|
+
...resolvedTurnExecutionPolicy,
|
|
894
|
+
credentialRestriction: template.credentialRestriction
|
|
895
|
+
} : resolvedTurnExecutionPolicy;
|
|
457
896
|
const created = await createSession2({
|
|
458
897
|
db: service.db,
|
|
459
898
|
bus: service.bus,
|
|
@@ -476,10 +915,12 @@ function createAutomationActivities(services, options = {}) {
|
|
|
476
915
|
accountId: input.accountId,
|
|
477
916
|
workspaceId: input.workspaceId,
|
|
478
917
|
initialMessage: accepted.initialMessage,
|
|
918
|
+
surface: "automation",
|
|
919
|
+
metrics: service.observability,
|
|
479
920
|
resources: template.resources,
|
|
480
921
|
skills: SessionSkills.parse(template.skills),
|
|
481
922
|
bundledSkillIds: template.bundledSkillIds,
|
|
482
|
-
tools:
|
|
923
|
+
tools: agentWriteThrough.tools,
|
|
483
924
|
toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
|
|
484
925
|
model,
|
|
485
926
|
reasoningEffort,
|
|
@@ -498,10 +939,11 @@ function createAutomationActivities(services, options = {}) {
|
|
|
498
939
|
label: accepted.serviceLabel
|
|
499
940
|
},
|
|
500
941
|
createdByContext: accepted.provenance,
|
|
501
|
-
instructions: template.instructions,
|
|
942
|
+
instructions: agentWriteThrough.instructions ?? template.instructions,
|
|
502
943
|
policyRole: template.policyRole,
|
|
503
944
|
firstPartyMcpPermissions: template.firstPartyMcpPermissions,
|
|
504
|
-
firstPartyMcpTools:
|
|
945
|
+
firstPartyMcpTools: agentWriteThrough.firstPartyMcpTools,
|
|
946
|
+
...agentWriteThrough.config ? { agentConfig: agentWriteThrough.config } : {},
|
|
505
947
|
retainWorkspaceCustomModel: model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX),
|
|
506
948
|
createIdempotencyKey: `automation-run:${run.id}`,
|
|
507
949
|
subjectId: accepted.serviceSubjectId
|
|
@@ -716,6 +1158,85 @@ function createFileUploadReaperActivities(services, options = {}) {
|
|
|
716
1158
|
return { reapExpiredFileUploads, recoverVideoGenerationWorkflows };
|
|
717
1159
|
}
|
|
718
1160
|
|
|
1161
|
+
// src/activities/model-call-fact-reconciler.ts
|
|
1162
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
1163
|
+
import { listWorkspaceIdsAfter, reconcileModelCallFacts } from "@opengeni/db";
|
|
1164
|
+
var MODEL_CALL_FACT_RECONCILE_LOOKBACK_MS = 2 * 60 * 60 * 1e3;
|
|
1165
|
+
var MODEL_CALL_FACT_RECONCILE_SETTLE_MS = 10 * 60 * 1e3;
|
|
1166
|
+
var MODEL_CALL_FACT_RECONCILE_WORKSPACE_LIMIT = 500;
|
|
1167
|
+
var MODEL_CALL_FACT_RECONCILE_RUN_BUDGET_MS = 3 * 60 * 1e3;
|
|
1168
|
+
var WORKSPACE_PAGE_SIZE = 200;
|
|
1169
|
+
function createModelCallFactReconcilerActivities(services, options = {}) {
|
|
1170
|
+
const now = options.now ?? Date.now;
|
|
1171
|
+
const runBudgetMs = options.runBudgetMs ?? MODEL_CALL_FACT_RECONCILE_RUN_BUDGET_MS;
|
|
1172
|
+
const startAfterWorkspaceId = options.startAfterWorkspaceId ?? randomUUID2;
|
|
1173
|
+
const reconcile = options.reconcile ?? reconcileModelCallFacts;
|
|
1174
|
+
const listWorkspaces = options.listWorkspaces ?? listWorkspaceIdsAfter;
|
|
1175
|
+
async function reconcileRecentModelCallFacts() {
|
|
1176
|
+
const { db, observability } = await services();
|
|
1177
|
+
const startedAt = now();
|
|
1178
|
+
const until = new Date(startedAt - MODEL_CALL_FACT_RECONCILE_SETTLE_MS);
|
|
1179
|
+
const since = new Date(until.getTime() - MODEL_CALL_FACT_RECONCILE_LOOKBACK_MS);
|
|
1180
|
+
const result = {
|
|
1181
|
+
workspaces: 0,
|
|
1182
|
+
missing: 0,
|
|
1183
|
+
repaired: 0,
|
|
1184
|
+
unrepaired: 0,
|
|
1185
|
+
truncatedWorkspaces: 0,
|
|
1186
|
+
failedWorkspaces: 0,
|
|
1187
|
+
budgetExhausted: false
|
|
1188
|
+
};
|
|
1189
|
+
const start = startAfterWorkspaceId();
|
|
1190
|
+
let cursor = start;
|
|
1191
|
+
let wrapped = false;
|
|
1192
|
+
while (true) {
|
|
1193
|
+
const page = await listWorkspaces(db, { afterId: cursor, limit: WORKSPACE_PAGE_SIZE });
|
|
1194
|
+
const ids = wrapped ? page.filter((id) => id <= start) : page;
|
|
1195
|
+
for (const workspaceId of ids) {
|
|
1196
|
+
if (now() - startedAt >= runBudgetMs) {
|
|
1197
|
+
result.budgetExhausted = true;
|
|
1198
|
+
break;
|
|
1199
|
+
}
|
|
1200
|
+
result.workspaces += 1;
|
|
1201
|
+
try {
|
|
1202
|
+
const outcome = await reconcile(db, {
|
|
1203
|
+
workspaceId,
|
|
1204
|
+
since,
|
|
1205
|
+
until,
|
|
1206
|
+
limit: MODEL_CALL_FACT_RECONCILE_WORKSPACE_LIMIT
|
|
1207
|
+
});
|
|
1208
|
+
result.missing += outcome.missing;
|
|
1209
|
+
result.repaired += outcome.repaired;
|
|
1210
|
+
result.unrepaired += outcome.unrepaired;
|
|
1211
|
+
if (outcome.truncated) result.truncatedWorkspaces += 1;
|
|
1212
|
+
} catch (error) {
|
|
1213
|
+
result.failedWorkspaces += 1;
|
|
1214
|
+
observability.warn("model call fact reconciliation failed for a workspace", {
|
|
1215
|
+
workspaceId,
|
|
1216
|
+
errorName: error instanceof Error ? error.name : "unknown"
|
|
1217
|
+
});
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
if (result.budgetExhausted) break;
|
|
1221
|
+
if (wrapped && ids.length < page.length) break;
|
|
1222
|
+
if (page.length < WORKSPACE_PAGE_SIZE) {
|
|
1223
|
+
if (wrapped) break;
|
|
1224
|
+
wrapped = true;
|
|
1225
|
+
cursor = null;
|
|
1226
|
+
continue;
|
|
1227
|
+
}
|
|
1228
|
+
cursor = page[page.length - 1];
|
|
1229
|
+
}
|
|
1230
|
+
if (result.failedWorkspaces > 0 || result.truncatedWorkspaces > 0 || result.budgetExhausted) {
|
|
1231
|
+
observability.warn("model call fact reconciliation incomplete", { ...result });
|
|
1232
|
+
} else if (result.missing > 0) {
|
|
1233
|
+
observability.info("model call fact reconciliation", { ...result });
|
|
1234
|
+
}
|
|
1235
|
+
return result;
|
|
1236
|
+
}
|
|
1237
|
+
return { reconcileRecentModelCallFacts };
|
|
1238
|
+
}
|
|
1239
|
+
|
|
719
1240
|
// src/activities/retained-screenshot-reaper.ts
|
|
720
1241
|
import {
|
|
721
1242
|
claimRetainedScreenshotMaintenance,
|
|
@@ -884,7 +1405,7 @@ import {
|
|
|
884
1405
|
markSandboxCheckpointArtifactDeletePending,
|
|
885
1406
|
recordWarmingSandboxCreated,
|
|
886
1407
|
registerSandboxCheckpointArtifact,
|
|
887
|
-
releaseLeaseHolder,
|
|
1408
|
+
releaseLeaseHolder as releaseLeaseHolder2,
|
|
888
1409
|
touchLeaseHolder,
|
|
889
1410
|
updateRigChangeStatus,
|
|
890
1411
|
SandboxCheckpointArtifactRegistrationConflictError
|
|
@@ -978,13 +1499,13 @@ var RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 6e4;
|
|
|
978
1499
|
var RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 1e4;
|
|
979
1500
|
var RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS = 250;
|
|
980
1501
|
var RIG_VERIFICATION_OWNER_TTL_MS = 20 * 6e4;
|
|
981
|
-
var RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE = "
|
|
1502
|
+
var RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE = "Sandbox Environment verification lease ownership is disabled; refusing to create an unowned verifier sandbox";
|
|
982
1503
|
var RIG_VERIFICATION_HOLDER_TOUCH_INTERVAL_MS = 1e4;
|
|
983
1504
|
var RIG_VERIFICATION_RELEASE_GRACE_MS = 1;
|
|
984
1505
|
var RigVerificationActivityDeadlineError = class extends Error {
|
|
985
1506
|
constructor(startToCloseTimeoutMs, cleanupReserveMs) {
|
|
986
1507
|
super(
|
|
987
|
-
`
|
|
1508
|
+
`Sandbox Environment verification activity-local deadline reached with ${cleanupReserveMs}ms reserved for cleanup before the ${startToCloseTimeoutMs}ms Temporal start-to-close timeout`
|
|
988
1509
|
);
|
|
989
1510
|
this.startToCloseTimeoutMs = startToCloseTimeoutMs;
|
|
990
1511
|
this.cleanupReserveMs = cleanupReserveMs;
|
|
@@ -1003,7 +1524,9 @@ function createRigVerificationActivityLifecycle(context = currentActivityContext
|
|
|
1003
1524
|
const temporalSignal = context?.cancellationSignal;
|
|
1004
1525
|
const forwardTemporalCancellation = () => {
|
|
1005
1526
|
if (!controller.signal.aborted) {
|
|
1006
|
-
controller.abort(
|
|
1527
|
+
controller.abort(
|
|
1528
|
+
temporalSignal?.reason ?? new Error("Sandbox Environment verification cancelled")
|
|
1529
|
+
);
|
|
1007
1530
|
}
|
|
1008
1531
|
};
|
|
1009
1532
|
if (temporalSignal?.aborted) {
|
|
@@ -1030,7 +1553,7 @@ function createRigVerificationActivityLifecycle(context = currentActivityContext
|
|
|
1030
1553
|
Math.max(1, startToCloseTimeoutMs - reserveMs)
|
|
1031
1554
|
);
|
|
1032
1555
|
}
|
|
1033
|
-
const
|
|
1556
|
+
const heartbeat2 = () => {
|
|
1034
1557
|
if (!context) return;
|
|
1035
1558
|
try {
|
|
1036
1559
|
context.heartbeat({
|
|
@@ -1043,13 +1566,13 @@ function createRigVerificationActivityLifecycle(context = currentActivityContext
|
|
|
1043
1566
|
if (!controller.signal.aborted) controller.abort(error);
|
|
1044
1567
|
}
|
|
1045
1568
|
};
|
|
1046
|
-
|
|
1569
|
+
heartbeat2();
|
|
1047
1570
|
const heartbeatTimeoutMs = context?.info.heartbeatTimeoutMs;
|
|
1048
1571
|
const heartbeatIntervalMs = typeof heartbeatTimeoutMs === "number" && heartbeatTimeoutMs > 0 ? Math.min(
|
|
1049
1572
|
RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS,
|
|
1050
1573
|
Math.max(RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS, Math.floor(heartbeatTimeoutMs / 3))
|
|
1051
1574
|
) : RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS;
|
|
1052
|
-
const heartbeatTimer = context ? setInterval(
|
|
1575
|
+
const heartbeatTimer = context ? setInterval(heartbeat2, heartbeatIntervalMs) : null;
|
|
1053
1576
|
if (heartbeatTimer && "unref" in heartbeatTimer && typeof heartbeatTimer.unref === "function") {
|
|
1054
1577
|
heartbeatTimer.unref();
|
|
1055
1578
|
}
|
|
@@ -1065,7 +1588,7 @@ function createRigVerificationActivityLifecycle(context = currentActivityContext
|
|
|
1065
1588
|
};
|
|
1066
1589
|
}
|
|
1067
1590
|
function abortReason(signal) {
|
|
1068
|
-
return signal.reason ?? new Error("
|
|
1591
|
+
return signal.reason ?? new Error("Sandbox Environment verification cancelled");
|
|
1069
1592
|
}
|
|
1070
1593
|
function throwIfAborted(signal) {
|
|
1071
1594
|
if (signal.aborted) throw abortReason(signal);
|
|
@@ -1134,7 +1657,7 @@ var defaultOwnershipDependencies = {
|
|
|
1134
1657
|
recordCreated: recordWarmingSandboxCreated,
|
|
1135
1658
|
commitWarm: commitWarmingToWarm,
|
|
1136
1659
|
touchHolder: touchLeaseHolder,
|
|
1137
|
-
releaseHolder:
|
|
1660
|
+
releaseHolder: releaseLeaseHolder2,
|
|
1138
1661
|
failWarming: failWarmingToCold,
|
|
1139
1662
|
markWarmLost: markWarmLeaseInstanceLost,
|
|
1140
1663
|
serialize: serializeEstablishedSandboxEnvelope,
|
|
@@ -1145,7 +1668,9 @@ var defaultOwnershipDependencies = {
|
|
|
1145
1668
|
};
|
|
1146
1669
|
var RigVerificationLeaseUnavailableError = class extends Error {
|
|
1147
1670
|
constructor(sandboxGroupId, role) {
|
|
1148
|
-
super(
|
|
1671
|
+
super(
|
|
1672
|
+
`Sandbox Environment verification requires a new clean sandbox, but lease ${sandboxGroupId} was ${role}`
|
|
1673
|
+
);
|
|
1149
1674
|
this.sandboxGroupId = sandboxGroupId;
|
|
1150
1675
|
this.role = role;
|
|
1151
1676
|
}
|
|
@@ -1168,7 +1693,7 @@ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = def
|
|
|
1168
1693
|
let establishmentSettled = false;
|
|
1169
1694
|
let establishmentPromise = null;
|
|
1170
1695
|
const warnCleanupTimeout = (operation, instanceId) => {
|
|
1171
|
-
input.observability.warn("
|
|
1696
|
+
input.observability.warn("sandbox environment verifier: ownership cleanup timed out", {
|
|
1172
1697
|
holderId,
|
|
1173
1698
|
instanceId,
|
|
1174
1699
|
operation,
|
|
@@ -1176,7 +1701,7 @@ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = def
|
|
|
1176
1701
|
});
|
|
1177
1702
|
};
|
|
1178
1703
|
const warnCleanupFailure = (operation, instanceId, error) => {
|
|
1179
|
-
input.observability.warn("
|
|
1704
|
+
input.observability.warn("sandbox environment verifier: ownership cleanup failed", {
|
|
1180
1705
|
holderId,
|
|
1181
1706
|
instanceId,
|
|
1182
1707
|
operation,
|
|
@@ -1302,11 +1827,14 @@ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = def
|
|
|
1302
1827
|
);
|
|
1303
1828
|
} catch (error) {
|
|
1304
1829
|
if (!signal.aborted) {
|
|
1305
|
-
input.observability.warn(
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1830
|
+
input.observability.warn(
|
|
1831
|
+
"sandbox environment verifier: Modal ownership tag failed",
|
|
1832
|
+
{
|
|
1833
|
+
holderId,
|
|
1834
|
+
instanceId: created.instanceId,
|
|
1835
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1836
|
+
}
|
|
1837
|
+
);
|
|
1310
1838
|
}
|
|
1311
1839
|
}
|
|
1312
1840
|
}
|
|
@@ -1501,10 +2029,12 @@ function rigProviderImageContentMarkerCommand(contentHash, markerRoot = "/var/op
|
|
|
1501
2029
|
}
|
|
1502
2030
|
function rigProviderImageContentMarker(contentHash, markerRoot = "/var/opengeni") {
|
|
1503
2031
|
if (!/^sha256:[0-9a-f]{64}$/u.test(contentHash)) {
|
|
1504
|
-
throw new Error(
|
|
2032
|
+
throw new Error(
|
|
2033
|
+
"Sandbox Environment provider image content marker requires a canonical SHA-256 value"
|
|
2034
|
+
);
|
|
1505
2035
|
}
|
|
1506
2036
|
if (!/^\/[A-Za-z0-9._/-]+$/u.test(markerRoot)) {
|
|
1507
|
-
throw new Error("
|
|
2037
|
+
throw new Error("Sandbox Environment provider image marker root must be a safe absolute path");
|
|
1508
2038
|
}
|
|
1509
2039
|
const normalizedRoot = markerRoot.replace(/\/+$/u, "") || "/";
|
|
1510
2040
|
const marker = `${normalizedRoot === "/" ? "" : normalizedRoot}/rig-setup-content-${contentHash.slice("sha256:".length)}.done`;
|
|
@@ -1542,7 +2072,9 @@ async function verifyRigProviderImageColdBoot(input, dependencies = defaultRigPr
|
|
|
1542
2072
|
runContext.commandRunner
|
|
1543
2073
|
);
|
|
1544
2074
|
if (markerResult.exitCode !== 0) {
|
|
1545
|
-
throw new Error(
|
|
2075
|
+
throw new Error(
|
|
2076
|
+
"built provider image is missing its exact sandbox environment-content marker"
|
|
2077
|
+
);
|
|
1546
2078
|
}
|
|
1547
2079
|
for (const check of rigPlatformChecksForSettings(input.settings)) {
|
|
1548
2080
|
const result = await runCommand(
|
|
@@ -1627,7 +2159,7 @@ async function buildVerifiedRigProviderImage(input) {
|
|
|
1627
2159
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1628
2160
|
error: {
|
|
1629
2161
|
code: "provider_image_content_conflict",
|
|
1630
|
-
message: "this exact
|
|
2162
|
+
message: "this exact sandbox environment version already records a provider image for different effective content; mint and verify a new version",
|
|
1631
2163
|
retryable: false
|
|
1632
2164
|
}
|
|
1633
2165
|
};
|
|
@@ -1801,37 +2333,36 @@ async function buildVerifiedRigProviderImage(input) {
|
|
|
1801
2333
|
});
|
|
1802
2334
|
}
|
|
1803
2335
|
}
|
|
1804
|
-
function settingsForRigVerification(settings,
|
|
1805
|
-
void packRuntime;
|
|
2336
|
+
function settingsForRigVerification(settings, rigImage) {
|
|
1806
2337
|
void rigImage;
|
|
1807
2338
|
return settings;
|
|
1808
2339
|
}
|
|
1809
2340
|
async function loadChangeTarget(db, workspaceId, changeId) {
|
|
1810
2341
|
const change = await getRigChange(db, workspaceId, changeId);
|
|
1811
2342
|
if (!change) {
|
|
1812
|
-
throw new Error(`
|
|
2343
|
+
throw new Error(`Sandbox Environment change not found: ${changeId}`);
|
|
1813
2344
|
}
|
|
1814
2345
|
const rig = await getRig(db, workspaceId, change.rigId);
|
|
1815
2346
|
if (!rig) {
|
|
1816
|
-
throw new Error(`
|
|
2347
|
+
throw new Error(`Sandbox Environment not found for change: ${change.rigId}`);
|
|
1817
2348
|
}
|
|
1818
2349
|
if (!change.baseVersionId) {
|
|
1819
|
-
throw new Error(`
|
|
2350
|
+
throw new Error(`Sandbox Environment change ${change.id} has no base version`);
|
|
1820
2351
|
}
|
|
1821
2352
|
const baseVersion = await getRigVersionById(db, workspaceId, change.baseVersionId);
|
|
1822
2353
|
if (!baseVersion || baseVersion.rigId !== rig.id) {
|
|
1823
|
-
throw new Error(`Base
|
|
2354
|
+
throw new Error(`Base sandbox environment version not found: ${change.baseVersionId}`);
|
|
1824
2355
|
}
|
|
1825
2356
|
return { rig, baseVersion, change };
|
|
1826
2357
|
}
|
|
1827
2358
|
async function loadVersionTarget(db, workspaceId, versionId) {
|
|
1828
2359
|
const version = await getRigVersionById(db, workspaceId, versionId);
|
|
1829
2360
|
if (!version) {
|
|
1830
|
-
throw new Error(`
|
|
2361
|
+
throw new Error(`Sandbox Environment version not found: ${versionId}`);
|
|
1831
2362
|
}
|
|
1832
2363
|
const rig = await getRig(db, workspaceId, version.rigId);
|
|
1833
2364
|
if (!rig) {
|
|
1834
|
-
throw new Error(`
|
|
2365
|
+
throw new Error(`Sandbox Environment not found for version: ${version.rigId}`);
|
|
1835
2366
|
}
|
|
1836
2367
|
return { rig, version };
|
|
1837
2368
|
}
|
|
@@ -1874,12 +2405,7 @@ function createRigVerificationActivities(services) {
|
|
|
1874
2405
|
change,
|
|
1875
2406
|
candidateVersion
|
|
1876
2407
|
);
|
|
1877
|
-
const
|
|
1878
|
-
const runSettings = settingsForRigVerification(
|
|
1879
|
-
settings,
|
|
1880
|
-
packRuntime,
|
|
1881
|
-
candidateVersion.image
|
|
1882
|
-
);
|
|
2408
|
+
const runSettings = settingsForRigVerification(settings, candidateVersion.image);
|
|
1883
2409
|
const providerImageContentHash = rigProviderImageContentHash({
|
|
1884
2410
|
backend: runSettings.sandboxBackend,
|
|
1885
2411
|
sourceImage: rigProviderImageSourceImage(runSettings, runSettings.sandboxBackend),
|
|
@@ -1953,7 +2479,7 @@ function createRigVerificationActivities(services) {
|
|
|
1953
2479
|
);
|
|
1954
2480
|
if (markerResult.exitCode !== 0) {
|
|
1955
2481
|
throw new Error(
|
|
1956
|
-
`Failed to seal
|
|
2482
|
+
`Failed to seal sandbox environment provider image content marker: ${markerResult.output.slice(-2e3)}`
|
|
1957
2483
|
);
|
|
1958
2484
|
}
|
|
1959
2485
|
const platformCheckResults = await runRigChecks(
|
|
@@ -2064,8 +2590,7 @@ function createRigVerificationActivities(services) {
|
|
|
2064
2590
|
metadata: { versionId: version.id }
|
|
2065
2591
|
});
|
|
2066
2592
|
try {
|
|
2067
|
-
const
|
|
2068
|
-
const runSettings = settingsForRigVerification(settings, packRuntime, version.image);
|
|
2593
|
+
const runSettings = settingsForRigVerification(settings, version.image);
|
|
2069
2594
|
const providerImageContentHash = rigProviderImageContentHash({
|
|
2070
2595
|
backend: runSettings.sandboxBackend,
|
|
2071
2596
|
sourceImage: rigProviderImageSourceImage(runSettings, runSettings.sandboxBackend),
|
|
@@ -2107,7 +2632,7 @@ function createRigVerificationActivities(services) {
|
|
|
2107
2632
|
);
|
|
2108
2633
|
if (markerResult.exitCode !== 0) {
|
|
2109
2634
|
throw new Error(
|
|
2110
|
-
`Failed to seal
|
|
2635
|
+
`Failed to seal sandbox environment provider image content marker: ${markerResult.output.slice(-2e3)}`
|
|
2111
2636
|
);
|
|
2112
2637
|
}
|
|
2113
2638
|
const platformCheckResults = await runRigChecks(
|
|
@@ -2180,13 +2705,60 @@ function createRigVerificationActivities(services) {
|
|
|
2180
2705
|
};
|
|
2181
2706
|
}
|
|
2182
2707
|
|
|
2708
|
+
// src/retained-process-diagnostics.ts
|
|
2709
|
+
import {
|
|
2710
|
+
databaseFailureCode,
|
|
2711
|
+
nestedPostgresSqlState,
|
|
2712
|
+
safeDatabaseErrorFacts,
|
|
2713
|
+
SandboxWorkspaceMutationFencedError
|
|
2714
|
+
} from "@opengeni/db";
|
|
2715
|
+
function warnRetainedProcessProofFailure(observability, error, process) {
|
|
2716
|
+
let correlationId;
|
|
2717
|
+
try {
|
|
2718
|
+
const sqlState = nestedPostgresSqlState(error);
|
|
2719
|
+
const constraint = safeDatabaseErrorFacts(error).constraint;
|
|
2720
|
+
correlationId = observability.recordFailureDiagnostic({
|
|
2721
|
+
code: error instanceof SandboxWorkspaceMutationFencedError ? "retained_process_fenced" : sqlState ? databaseFailureCode(sqlState) : "retained_process_proof_failed",
|
|
2722
|
+
stage: "sandbox_retained_processes.proof",
|
|
2723
|
+
error,
|
|
2724
|
+
processId: process.id,
|
|
2725
|
+
sessionId: process.sessionId,
|
|
2726
|
+
...process.ownerAttemptId ? { attemptId: process.ownerAttemptId } : {},
|
|
2727
|
+
...process.ownerTurnId ? { turnId: process.ownerTurnId } : {},
|
|
2728
|
+
attempts: process.reconcileAttempts,
|
|
2729
|
+
sqlState,
|
|
2730
|
+
...constraint ? { constraint } : {}
|
|
2731
|
+
});
|
|
2732
|
+
} catch {
|
|
2733
|
+
}
|
|
2734
|
+
try {
|
|
2735
|
+
observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
|
|
2736
|
+
...correlationId ? { correlationId } : {}
|
|
2737
|
+
});
|
|
2738
|
+
} catch {
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
|
|
2742
|
+
// src/retained-process-retry.ts
|
|
2743
|
+
function retainedProcessDeadlineRetryMs(process, lease, settings, retryAfterMs, now = Date.now()) {
|
|
2744
|
+
if (process.providerBackend !== "modal" || process.routeTargetId !== null || !lease || lease.id !== process.leaseId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || !lease.providerDeadlineAt)
|
|
2745
|
+
return retryAfterMs;
|
|
2746
|
+
const untilRotation = lease.providerDeadlineAt.getTime() - settings.sandboxRotationLeadMs - now;
|
|
2747
|
+
if (!Number.isFinite(untilRotation)) return retryAfterMs;
|
|
2748
|
+
return Math.min(
|
|
2749
|
+
retryAfterMs,
|
|
2750
|
+
Math.max(1e3, settings.sandboxLeaseReaperPeriodMs, untilRotation)
|
|
2751
|
+
);
|
|
2752
|
+
}
|
|
2753
|
+
|
|
2183
2754
|
// src/activities/sandbox-lease.ts
|
|
2184
|
-
import { createHash, randomUUID as
|
|
2185
|
-
import {
|
|
2186
|
-
import { Context } from "@temporalio/activity";
|
|
2755
|
+
import { createHash, randomUUID as randomUUID3 } from "crypto";
|
|
2756
|
+
import { requestRetainedProcessDeadlineCancellation } from "@opengeni/db/retained-provider-commands";
|
|
2757
|
+
import { Context as Context2 } from "@temporalio/activity";
|
|
2187
2758
|
import { OpLostReason, OpState } from "@opengeni/agent-proto";
|
|
2188
2759
|
import {
|
|
2189
2760
|
accrueWarmSeconds,
|
|
2761
|
+
retainedProviderCommandPersistence,
|
|
2190
2762
|
adoptLegacyModalCheckpointArtifact,
|
|
2191
2763
|
confirmDrainCold,
|
|
2192
2764
|
appendSessionEventToSandboxGroup,
|
|
@@ -2202,10 +2774,17 @@ import {
|
|
|
2202
2774
|
countSandboxLeasesByLiveness,
|
|
2203
2775
|
deferRetainedProcessReconciliation,
|
|
2204
2776
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
2205
|
-
getBillingBalance as getBillingBalance3,
|
|
2206
2777
|
listCreditBalancesByAccount,
|
|
2778
|
+
countActiveUsers,
|
|
2779
|
+
readCreditGrantTotals,
|
|
2780
|
+
readManagedAuthNewSignupsSwitch,
|
|
2781
|
+
readVerifiedSignupTrialSwitch,
|
|
2207
2782
|
listLegacyModalCheckpointSlots,
|
|
2208
2783
|
listLiveModalSandboxLeaseAttributions,
|
|
2784
|
+
listPendingModalProviderCreates,
|
|
2785
|
+
claimModalProviderCreateRecovery,
|
|
2786
|
+
recordRecoveredModalProviderCreate,
|
|
2787
|
+
markWarmBillingStopCutoff,
|
|
2209
2788
|
listMeterableWarmLeases,
|
|
2210
2789
|
listSandboxViewerForceDrainWorkspaceIds,
|
|
2211
2790
|
markSandboxCheckpointArtifactDeletePending as markSandboxCheckpointArtifactDeletePending2,
|
|
@@ -2214,23 +2793,29 @@ import {
|
|
|
2214
2793
|
registerSandboxCheckpointArtifact as registerSandboxCheckpointArtifact2,
|
|
2215
2794
|
recordRetainedProcessReconciliationProof,
|
|
2216
2795
|
replaceWorkspaceArchiveCaptureAfterProof,
|
|
2217
|
-
readLease,
|
|
2796
|
+
readLease as readLease2,
|
|
2797
|
+
readWorkspaceArchiveCapturePreflight,
|
|
2218
2798
|
reapExpiredSessionListSnapshots,
|
|
2219
2799
|
reapStaleLeaseHoldersGlobal,
|
|
2220
2800
|
requestDueSandboxRotationsGlobal,
|
|
2221
2801
|
readSandboxRotationBacklog,
|
|
2802
|
+
readRecentSandboxRecoveryObservations,
|
|
2222
2803
|
workspaceArchiveCaptureDeadlineElapsed,
|
|
2223
2804
|
retainedProcessReconciliationProof,
|
|
2224
2805
|
retainedProcessSettlementIdentity,
|
|
2806
|
+
recoverManagedSessionBackgroundCommand,
|
|
2225
2807
|
settleClaimedConnectedMachineBackgroundCommand,
|
|
2226
2808
|
settleSandboxCheckpointArtifactGc,
|
|
2227
2809
|
rlsContextForWorkspace,
|
|
2228
2810
|
settleRetainedProcess
|
|
2229
2811
|
} from "@opengeni/db";
|
|
2230
2812
|
import {
|
|
2813
|
+
claimConnectedCommandOutputReleases,
|
|
2231
2814
|
claimConnectedMachineSessionBackgroundCommands,
|
|
2232
2815
|
deferConnectedMachineBackgroundCommandReconciliation,
|
|
2233
|
-
recordConnectedMachineBackgroundCommandProof
|
|
2816
|
+
recordConnectedMachineBackgroundCommandProof,
|
|
2817
|
+
recordConnectedCommandOutputConsumption,
|
|
2818
|
+
settleConnectedCommandOutputReleaseClaim
|
|
2234
2819
|
} from "@opengeni/db/session-background-commands";
|
|
2235
2820
|
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
2236
2821
|
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
@@ -2240,9 +2825,9 @@ import {
|
|
|
2240
2825
|
disposeWorkspaceArchive,
|
|
2241
2826
|
cancelSelfhostedOp,
|
|
2242
2827
|
assertConsistentSandboxProviderIdentity,
|
|
2243
|
-
createSandboxClientForBackend,
|
|
2828
|
+
createSandboxClientForBackend as createSandboxClientForBackend2,
|
|
2244
2829
|
deleteModalCheckpointSnapshot as deleteModalCheckpointSnapshot2,
|
|
2245
|
-
deserializeSandboxSessionStateEnvelope,
|
|
2830
|
+
deserializeSandboxSessionStateEnvelope as deserializeSandboxSessionStateEnvelope2,
|
|
2246
2831
|
establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
|
|
2247
2832
|
inspectModalSandboxLifecycle,
|
|
2248
2833
|
isExecSessionLostBanner,
|
|
@@ -2258,9 +2843,10 @@ import {
|
|
|
2258
2843
|
providerWorkspaceCapturePolicy,
|
|
2259
2844
|
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
2260
2845
|
resolveModalCheckpointProviderBinding,
|
|
2846
|
+
findModalProviderCreateReceipt,
|
|
2261
2847
|
resolveModalCheckpointProviderBindingForSession,
|
|
2262
2848
|
querySelfhostedOp,
|
|
2263
|
-
resumeExactSandboxSession,
|
|
2849
|
+
resumeExactSandboxSession as resumeExactSandboxSession2,
|
|
2264
2850
|
sandboxProviderContinuityForState,
|
|
2265
2851
|
sandboxBackendForSdkBackendId,
|
|
2266
2852
|
sandboxProviderInstanceIdFromEnvelope,
|
|
@@ -2277,6 +2863,10 @@ import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni
|
|
|
2277
2863
|
var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
2278
2864
|
var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
|
|
2279
2865
|
var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
|
|
2866
|
+
var RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS = 8;
|
|
2867
|
+
var RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS = 1e4;
|
|
2868
|
+
var RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS = 6e4;
|
|
2869
|
+
var RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES = 256 * 1024;
|
|
2280
2870
|
var RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
|
|
2281
2871
|
var RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 6e4;
|
|
2282
2872
|
var CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
|
|
@@ -2315,18 +2905,18 @@ async function awaitProviderCaptureWithLatePublication(input) {
|
|
|
2315
2905
|
function startSandboxReaperHeartbeat(details) {
|
|
2316
2906
|
let context;
|
|
2317
2907
|
try {
|
|
2318
|
-
context =
|
|
2908
|
+
context = Context2.current();
|
|
2319
2909
|
} catch {
|
|
2320
2910
|
return () => void 0;
|
|
2321
2911
|
}
|
|
2322
|
-
const
|
|
2912
|
+
const heartbeat2 = () => {
|
|
2323
2913
|
try {
|
|
2324
2914
|
context.heartbeat({ ...details, heartbeatAt: (/* @__PURE__ */ new Date()).toISOString() });
|
|
2325
2915
|
} catch {
|
|
2326
2916
|
}
|
|
2327
2917
|
};
|
|
2328
|
-
|
|
2329
|
-
const timer = setInterval(
|
|
2918
|
+
heartbeat2();
|
|
2919
|
+
const timer = setInterval(heartbeat2, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);
|
|
2330
2920
|
if ("unref" in timer && typeof timer.unref === "function") timer.unref();
|
|
2331
2921
|
return () => clearInterval(timer);
|
|
2332
2922
|
}
|
|
@@ -2349,25 +2939,30 @@ function sandboxDrainCaptureId(operationId, attempt) {
|
|
|
2349
2939
|
}
|
|
2350
2940
|
function sandboxDrainActivityAttempt() {
|
|
2351
2941
|
try {
|
|
2352
|
-
const attempt =
|
|
2942
|
+
const attempt = Context2.current().info.attempt;
|
|
2353
2943
|
return Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1;
|
|
2354
2944
|
} catch {
|
|
2355
2945
|
return 1;
|
|
2356
2946
|
}
|
|
2357
2947
|
}
|
|
2358
2948
|
function createSandboxLeaseActivities(services, options = {}) {
|
|
2359
|
-
const terminateBox = options.terminateBox
|
|
2949
|
+
const terminateBox = (options.terminateBox ? async (...args) => {
|
|
2950
|
+
await args[10]?.();
|
|
2951
|
+
return options.terminateBox(...args);
|
|
2952
|
+
} : null) ?? (async (settings, lease, observability, persistArchive, providerCaptureRequestId, captureDisposition, capturePolicy, diskBackedArchives, releaseFailedCapture, workspaceId, beforeProviderStop) => await terminateProviderBox(
|
|
2360
2953
|
settings,
|
|
2361
2954
|
lease,
|
|
2362
2955
|
observability,
|
|
2363
2956
|
persistArchive,
|
|
2364
|
-
|
|
2957
|
+
createSandboxClientForBackend2,
|
|
2365
2958
|
terminateModalSandboxById,
|
|
2366
2959
|
providerCaptureRequestId,
|
|
2367
2960
|
captureDisposition,
|
|
2368
2961
|
capturePolicy,
|
|
2369
2962
|
diskBackedArchives,
|
|
2370
|
-
releaseFailedCapture
|
|
2963
|
+
releaseFailedCapture,
|
|
2964
|
+
workspaceId,
|
|
2965
|
+
beforeProviderStop
|
|
2371
2966
|
));
|
|
2372
2967
|
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
2373
2968
|
const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
|
|
@@ -2378,14 +2973,18 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2378
2973
|
try {
|
|
2379
2974
|
const { db, settings, observability } = await services();
|
|
2380
2975
|
const timing = sandboxDrainTiming(settings);
|
|
2381
|
-
const
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2976
|
+
const commandContainment = {
|
|
2977
|
+
idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
|
|
2978
|
+
onCommandContainment: (outcome) => recordSandboxCommandContainment(observability, outcome),
|
|
2979
|
+
onCommandContainmentError: (error) => {
|
|
2980
|
+
observability.warn("sandbox reaper: retained command containment inspection failed", {
|
|
2981
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2982
|
+
});
|
|
2983
|
+
}
|
|
2385
2984
|
};
|
|
2386
2985
|
if (!settings.sandboxOwnershipEnabled) {
|
|
2387
2986
|
const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
|
|
2388
|
-
|
|
2987
|
+
...commandContainment,
|
|
2389
2988
|
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
2390
2989
|
turnHolderTtlMs: settings.sandboxLeaseTtlMs,
|
|
2391
2990
|
interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
|
|
@@ -2412,7 +3011,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2412
3011
|
});
|
|
2413
3012
|
recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
|
|
2414
3013
|
const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
|
|
2415
|
-
|
|
3014
|
+
...commandContainment,
|
|
2416
3015
|
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
2417
3016
|
// Dead-worker turn holders: a live holder is touched every 10s from the
|
|
2418
3017
|
// moment it is registered (resumeBoxForTurn's holder-liveness loop covers
|
|
@@ -2453,7 +3052,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2453
3052
|
instanceId: input.target.instanceId
|
|
2454
3053
|
});
|
|
2455
3054
|
try {
|
|
2456
|
-
const { db, settings, observability, objectStorage } = await services();
|
|
3055
|
+
const { db, settings, observability, objectStorage, bus } = await services();
|
|
2457
3056
|
assertSandboxDrainInputTiming(input);
|
|
2458
3057
|
const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
|
|
2459
3058
|
const row = input.target;
|
|
@@ -2473,11 +3072,12 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2473
3072
|
terminateBox,
|
|
2474
3073
|
probeDrainableProvider,
|
|
2475
3074
|
captureAttempt,
|
|
2476
|
-
objectStorage
|
|
3075
|
+
objectStorage,
|
|
3076
|
+
bus
|
|
2477
3077
|
);
|
|
2478
3078
|
return { status: drainedCold ? "terminated" : "skipped" };
|
|
2479
3079
|
} catch (error) {
|
|
2480
|
-
const lease = await
|
|
3080
|
+
const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId).catch(() => null);
|
|
2481
3081
|
const details = {
|
|
2482
3082
|
sandboxLeaseKey: sandboxLeaseTelemetryKey(row.workspaceId, row.sandboxGroupId),
|
|
2483
3083
|
workspaceId: row.workspaceId,
|
|
@@ -2540,6 +3140,19 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2540
3140
|
const rotationsRequested = input.rotationsRequested;
|
|
2541
3141
|
let modalOrphansTerminated = 0;
|
|
2542
3142
|
await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
|
|
3143
|
+
await reconcileConnectedCommandOutputReleases(db, settings, observability, service.bus);
|
|
3144
|
+
try {
|
|
3145
|
+
await reconcileModalProviderCreates(
|
|
3146
|
+
db,
|
|
3147
|
+
settings,
|
|
3148
|
+
observability,
|
|
3149
|
+
options.findModalProviderCreateReceipt
|
|
3150
|
+
);
|
|
3151
|
+
} catch (error) {
|
|
3152
|
+
observability.warn("sandbox reaper: Modal create recovery inventory failed", {
|
|
3153
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3154
|
+
});
|
|
3155
|
+
}
|
|
2543
3156
|
if (!settings.sandboxOwnershipEnabled) {
|
|
2544
3157
|
await refreshQueueLeaseAndCreditGauges(
|
|
2545
3158
|
db,
|
|
@@ -2557,7 +3170,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2557
3170
|
const meterResult = await accrueWarmTick(db, settings, observability);
|
|
2558
3171
|
metered = meterResult.accrued;
|
|
2559
3172
|
const forceDrainWorkspaceIds = /* @__PURE__ */ new Set();
|
|
2560
|
-
if (settings.
|
|
3173
|
+
if (settings.sandboxWarmBillingMode === "credits" || settings.sandboxMaxWarmSecondsPerWorkspace > 0) {
|
|
2561
3174
|
for (const workspaceId of meterResult.workspaceIds) {
|
|
2562
3175
|
forceDrainWorkspaceIds.add(workspaceId);
|
|
2563
3176
|
}
|
|
@@ -2657,7 +3270,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2657
3270
|
timeoutClass: plan.timeoutClass,
|
|
2658
3271
|
snapshotTimeoutMs: plan.snapshotTimeoutMs,
|
|
2659
3272
|
captureTimeoutMs: plan.captureTimeoutMs,
|
|
2660
|
-
operationId:
|
|
3273
|
+
operationId: randomUUID3()
|
|
2661
3274
|
})
|
|
2662
3275
|
)
|
|
2663
3276
|
);
|
|
@@ -2849,6 +3462,7 @@ async function reconcileConnectedMachineBackgroundCommands(db, settings, observa
|
|
|
2849
3462
|
);
|
|
2850
3463
|
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
2851
3464
|
let proof = claim.proof;
|
|
3465
|
+
let outputReplay = null;
|
|
2852
3466
|
const connectionKey = JSON.stringify([
|
|
2853
3467
|
claim.controlWorkspaceId,
|
|
2854
3468
|
claim.enrollmentId,
|
|
@@ -2904,18 +3518,14 @@ async function reconcileConnectedMachineBackgroundCommands(db, settings, observa
|
|
|
2904
3518
|
journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {
|
|
2905
3519
|
} }
|
|
2906
3520
|
});
|
|
2907
|
-
|
|
3521
|
+
outputReplay = await replayConnectedCommandOutput(
|
|
3522
|
+
client,
|
|
2908
3523
|
claim.opId,
|
|
2909
|
-
proof
|
|
3524
|
+
Boolean(proof),
|
|
2910
3525
|
async (frames) => {
|
|
2911
3526
|
await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
|
|
2912
3527
|
}
|
|
2913
3528
|
);
|
|
2914
|
-
if (proof && replay.status !== "completed" && !replay.terminal) {
|
|
2915
|
-
throw new Error(
|
|
2916
|
-
"Connected command terminal output replay has not reached its exit frontier"
|
|
2917
|
-
);
|
|
2918
|
-
}
|
|
2919
3529
|
}
|
|
2920
3530
|
if (!proof) {
|
|
2921
3531
|
await deferConnectedCommandClaim(
|
|
@@ -2966,6 +3576,15 @@ async function reconcileConnectedMachineBackgroundCommands(db, settings, observa
|
|
|
2966
3576
|
if (!settlement.settled) {
|
|
2967
3577
|
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
2968
3578
|
}
|
|
3579
|
+
const receipt = outputReplay?.outputReceipt ?? (outputReplay?.status === "running" ? outputReplay.terminal?.outputReceipt : void 0);
|
|
3580
|
+
const terminal = outputReplay?.status === "completed" ? outputReplay : outputReplay?.terminal;
|
|
3581
|
+
if (proof.outcome === "exited" && receipt && terminal) {
|
|
3582
|
+
await recordConnectedCommandOutputConsumption(db, {
|
|
3583
|
+
...claim,
|
|
3584
|
+
receipt,
|
|
3585
|
+
exitCode: terminal.outcome.response.exitCode
|
|
3586
|
+
});
|
|
3587
|
+
}
|
|
2969
3588
|
if (settlement.events.length > 0) {
|
|
2970
3589
|
try {
|
|
2971
3590
|
await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
|
|
@@ -2989,6 +3608,133 @@ async function reconcileConnectedMachineBackgroundCommands(db, settings, observa
|
|
|
2989
3608
|
if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
|
|
2990
3609
|
} while (Date.now() < deadline);
|
|
2991
3610
|
}
|
|
3611
|
+
async function replayConnectedCommandOutput(client, opId, terminalKnown, capture) {
|
|
3612
|
+
let capturedThrough = 0n;
|
|
3613
|
+
while (true) {
|
|
3614
|
+
const before = capturedThrough;
|
|
3615
|
+
const replay = await client.readExisting(opId, terminalKnown ? 5e3 : 250, capture);
|
|
3616
|
+
if (!terminalKnown || replay.status === "completed" || replay.terminal) return replay;
|
|
3617
|
+
capturedThrough = BigInt(replay.replaySequence);
|
|
3618
|
+
if (capturedThrough === before) {
|
|
3619
|
+
throw new Error("Connected command terminal output replay has not reached its exit frontier");
|
|
3620
|
+
}
|
|
3621
|
+
}
|
|
3622
|
+
}
|
|
3623
|
+
async function reconcileConnectedCommandOutputReleases(db, settings, observability, bus, controlRpcOverride) {
|
|
3624
|
+
const dueBefore = /* @__PURE__ */ new Date();
|
|
3625
|
+
const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
|
|
3626
|
+
const controlRpc = controlRpcOverride ?? new NatsControlRpc(async () => bus.getRequestConnection());
|
|
3627
|
+
do {
|
|
3628
|
+
let claims;
|
|
3629
|
+
try {
|
|
3630
|
+
claims = await claimConnectedCommandOutputReleases(db, {
|
|
3631
|
+
claimId: crypto.randomUUID(),
|
|
3632
|
+
dueBefore,
|
|
3633
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
3634
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
|
|
3635
|
+
});
|
|
3636
|
+
} catch (error) {
|
|
3637
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
3638
|
+
observability.warn("sandbox reaper: connected-command output claim failed", {
|
|
3639
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3640
|
+
});
|
|
3641
|
+
return;
|
|
3642
|
+
}
|
|
3643
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
3644
|
+
try {
|
|
3645
|
+
if (!bus.getOpStreamConnection)
|
|
3646
|
+
throw new Error("Connected command output transport unavailable");
|
|
3647
|
+
const rpcSubject = subjectFor(
|
|
3648
|
+
claim.controlWorkspaceId,
|
|
3649
|
+
claim.enrollmentId,
|
|
3650
|
+
claim.connectionInstanceId
|
|
3651
|
+
);
|
|
3652
|
+
const controlTimeoutMs = Math.min(
|
|
3653
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
3654
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
3655
|
+
);
|
|
3656
|
+
const client = new OpStreamExecClient({
|
|
3657
|
+
workspaceId: claim.controlWorkspaceId,
|
|
3658
|
+
agentId: claim.enrollmentId,
|
|
3659
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
3660
|
+
epoch: 0,
|
|
3661
|
+
controlRpc,
|
|
3662
|
+
rpcSubject,
|
|
3663
|
+
transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
|
|
3664
|
+
controlTimeoutMs,
|
|
3665
|
+
retryClock: defaultSelfhostedRetryClock,
|
|
3666
|
+
journal: {
|
|
3667
|
+
attachGeneration: () => String(Date.now()),
|
|
3668
|
+
persistSettled: () => {
|
|
3669
|
+
throw new Error("Background output release requires its database receipt");
|
|
3670
|
+
}
|
|
3671
|
+
}
|
|
3672
|
+
});
|
|
3673
|
+
let receipt = claim.receipt;
|
|
3674
|
+
if (!receipt) {
|
|
3675
|
+
const status = await querySelfhostedOp({
|
|
3676
|
+
controlRpc,
|
|
3677
|
+
rpcSubject,
|
|
3678
|
+
opId: claim.opId,
|
|
3679
|
+
controlTimeoutMs
|
|
3680
|
+
});
|
|
3681
|
+
if (status.state === OpState.OP_STATE_LOST) {
|
|
3682
|
+
if (!await settleConnectedCommandOutputReleaseClaim(db, {
|
|
3683
|
+
claim,
|
|
3684
|
+
outcome: "unavailable",
|
|
3685
|
+
retryAfterMs: 0
|
|
3686
|
+
}))
|
|
3687
|
+
throw new Error("Connected command output loss no longer owns its exact claim");
|
|
3688
|
+
recordConnectedCommandReconciliation(observability, "output_unavailable");
|
|
3689
|
+
return;
|
|
3690
|
+
}
|
|
3691
|
+
if (status.state !== OpState.OP_STATE_COMPLETE)
|
|
3692
|
+
throw new Error("Terminal command output returned a live operation");
|
|
3693
|
+
const replay = await replayConnectedCommandOutput(
|
|
3694
|
+
client,
|
|
3695
|
+
claim.opId,
|
|
3696
|
+
true,
|
|
3697
|
+
async (frames) => captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames)
|
|
3698
|
+
);
|
|
3699
|
+
const terminal = replay.status === "completed" ? replay : replay.terminal;
|
|
3700
|
+
receipt = replay.outputReceipt ?? (replay.status === "running" ? replay.terminal?.outputReceipt : void 0) ?? null;
|
|
3701
|
+
if (!terminal || !receipt)
|
|
3702
|
+
throw new Error("Connected command output replay has no complete custody receipt");
|
|
3703
|
+
await recordConnectedCommandOutputConsumption(db, {
|
|
3704
|
+
...claim,
|
|
3705
|
+
receipt,
|
|
3706
|
+
exitCode: terminal.outcome.response.exitCode
|
|
3707
|
+
});
|
|
3708
|
+
}
|
|
3709
|
+
const outcome = await client.releaseCapturedOutput(claim.opId, receipt);
|
|
3710
|
+
if (!await settleConnectedCommandOutputReleaseClaim(db, {
|
|
3711
|
+
claim,
|
|
3712
|
+
outcome,
|
|
3713
|
+
retryAfterMs: outcome === "published" ? settings.sandboxLeaseReaperPeriodMs : 0
|
|
3714
|
+
}))
|
|
3715
|
+
throw new Error("Connected command output release no longer owns its exact claim");
|
|
3716
|
+
recordConnectedCommandReconciliation(
|
|
3717
|
+
observability,
|
|
3718
|
+
outcome === "published" ? "output_ack_published" : "output_released"
|
|
3719
|
+
);
|
|
3720
|
+
} catch (error) {
|
|
3721
|
+
await settleConnectedCommandOutputReleaseClaim(db, {
|
|
3722
|
+
claim,
|
|
3723
|
+
outcome: "retry",
|
|
3724
|
+
retryAfterMs: Math.min(
|
|
3725
|
+
5 * 6e4,
|
|
3726
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** Math.min(4, claim.reconcileAttempts - 1)
|
|
3727
|
+
)
|
|
3728
|
+
}).catch(() => false);
|
|
3729
|
+
observability.warn("sandbox reaper: connected-command output release deferred", {
|
|
3730
|
+
commandId: claim.commandId,
|
|
3731
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3732
|
+
});
|
|
3733
|
+
}
|
|
3734
|
+
});
|
|
3735
|
+
if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
|
|
3736
|
+
} while (Date.now() < deadline);
|
|
3737
|
+
}
|
|
2992
3738
|
async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
|
|
2993
3739
|
const input = {
|
|
2994
3740
|
controlRpc,
|
|
@@ -3075,10 +3821,65 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3075
3821
|
});
|
|
3076
3822
|
return;
|
|
3077
3823
|
}
|
|
3824
|
+
const drainBudget = {
|
|
3825
|
+
until: Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS
|
|
3826
|
+
};
|
|
3078
3827
|
for (const claim of claims) {
|
|
3079
3828
|
let process = claim.process;
|
|
3080
3829
|
const expected = retainedProcessSettlementIdentity(process);
|
|
3081
3830
|
let proof = retainedProcessReconciliationProof(process);
|
|
3831
|
+
const processScope = {
|
|
3832
|
+
accountId: process.accountId,
|
|
3833
|
+
workspaceId: process.workspaceId,
|
|
3834
|
+
sessionId: process.sessionId,
|
|
3835
|
+
processId: process.id
|
|
3836
|
+
};
|
|
3837
|
+
if (process.ownerActorKind === "turn" && process.providerBackend === "modal" && !proof && !claim.ownerState.startsWith("background_")) {
|
|
3838
|
+
try {
|
|
3839
|
+
const command = await recoverManagedSessionBackgroundCommand(db, {
|
|
3840
|
+
...processScope,
|
|
3841
|
+
expected,
|
|
3842
|
+
reconciliationClaimId: claim.claimId
|
|
3843
|
+
});
|
|
3844
|
+
if (command) {
|
|
3845
|
+
claim.ownerState = `background_${command.state}`;
|
|
3846
|
+
recordRetainedProcessReconciliation(observability, "background_recovered");
|
|
3847
|
+
}
|
|
3848
|
+
} catch (error) {
|
|
3849
|
+
observability.warn("sandbox reaper: retained-command background recovery failed", {
|
|
3850
|
+
processId: process.id,
|
|
3851
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3852
|
+
});
|
|
3853
|
+
}
|
|
3854
|
+
}
|
|
3855
|
+
const storedCommandPersistence = retainedProviderCommandPersistence(
|
|
3856
|
+
db,
|
|
3857
|
+
processScope,
|
|
3858
|
+
bus ? (events) => bus.publish(process.workspaceId, process.sessionId, events) : void 0
|
|
3859
|
+
);
|
|
3860
|
+
const commandPersistence = {
|
|
3861
|
+
...storedCommandPersistence,
|
|
3862
|
+
requestCancellation: async (reason) => {
|
|
3863
|
+
if (reason === "provider_deadline") {
|
|
3864
|
+
await requestRetainedProcessDeadlineCancellation(db, processScope);
|
|
3865
|
+
if (!await storedCommandPersistence.cancellationRequested())
|
|
3866
|
+
throw new Error(
|
|
3867
|
+
"Supervised deadline cancellation no longer owns its original rotating lease"
|
|
3868
|
+
);
|
|
3869
|
+
} else await storedCommandPersistence.requestCancellation(reason);
|
|
3870
|
+
}
|
|
3871
|
+
};
|
|
3872
|
+
if (proof && process.providerBackend === "modal") {
|
|
3873
|
+
try {
|
|
3874
|
+
const command = await commandPersistence.load();
|
|
3875
|
+
if (command?.kind === "modal-router-v1" && command.supervision && (!await commandPersistence.loadSupervisionReceipt() || ![command.streams.stdout, command.streams.stderr].every(
|
|
3876
|
+
(stream) => stream.eof && stream.exitCode === 0
|
|
3877
|
+
)))
|
|
3878
|
+
proof = null;
|
|
3879
|
+
} catch {
|
|
3880
|
+
proof = null;
|
|
3881
|
+
}
|
|
3882
|
+
}
|
|
3082
3883
|
if (!proof) {
|
|
3083
3884
|
let observation = null;
|
|
3084
3885
|
if (process.providerBackend === "modal" && process.routeTargetId === null && !process.providerBindingKey) {
|
|
@@ -3121,8 +3922,18 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3121
3922
|
};
|
|
3122
3923
|
}
|
|
3123
3924
|
}
|
|
3925
|
+
if (observation?.status === "deferred" && process.providerBackend === "modal" && !process.providerBindingKey && !process.deadlineCancellationRequestedAt) {
|
|
3926
|
+
try {
|
|
3927
|
+
await requestRetainedProcessDeadlineCancellation(db, processScope);
|
|
3928
|
+
} catch (error) {
|
|
3929
|
+
observability.warn("sandbox reaper: legacy deadline cancellation intent failed", {
|
|
3930
|
+
processId: process.id,
|
|
3931
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3932
|
+
});
|
|
3933
|
+
}
|
|
3934
|
+
}
|
|
3124
3935
|
if (observation === null) {
|
|
3125
|
-
const lease = await
|
|
3936
|
+
const lease = await readLease2(db, process.workspaceId, process.sandboxGroupId).catch(
|
|
3126
3937
|
() => null
|
|
3127
3938
|
);
|
|
3128
3939
|
const currentLeaseIdentityMatches = Boolean(lease) && lease.id === process.leaseId && lease.leaseEpoch === process.leaseEpoch && lease.backend === process.providerBackend && lease.instanceId === process.providerInstanceId && process.routeTargetId === null;
|
|
@@ -3162,11 +3973,25 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3162
3973
|
}
|
|
3163
3974
|
} else {
|
|
3164
3975
|
try {
|
|
3165
|
-
|
|
3976
|
+
const command = await commandPersistence.load();
|
|
3977
|
+
const supervised = command?.kind === "modal-router-v1" && Boolean(command.supervision);
|
|
3978
|
+
const supervisionMetric = (outcome) => observability.incrementCounter({
|
|
3979
|
+
name: "opengeni_command_supervision_total",
|
|
3980
|
+
help: "Native retained command supervision reconciliation outcomes.",
|
|
3981
|
+
labels: { outcome }
|
|
3982
|
+
});
|
|
3983
|
+
if (supervised && (lease.rotationReason === "provider_deadline" || claim.ownerState === "background_stopping")) {
|
|
3984
|
+
await commandPersistence.requestCancellation(
|
|
3985
|
+
lease.rotationReason === "provider_deadline" ? "provider_deadline" : "explicit_stop"
|
|
3986
|
+
);
|
|
3987
|
+
supervisionMetric("cancellation_intent");
|
|
3988
|
+
}
|
|
3989
|
+
const legacyDeadlineStop = lease.rotationReason === "provider_deadline" && !supervised && !process.deadlineCancellationRequestedAt;
|
|
3990
|
+
const probeOutcome = await probe(
|
|
3166
3991
|
settings,
|
|
3167
3992
|
lease,
|
|
3168
3993
|
process,
|
|
3169
|
-
claim.ownerState === "background_stopping" ? "cancel" : "observe",
|
|
3994
|
+
claim.ownerState === "background_stopping" && !process.deadlineCancellationRequestedAt || legacyDeadlineStop ? "cancel" : "observe",
|
|
3170
3995
|
async (result, chunkId, stream, streamFidelity) => {
|
|
3171
3996
|
if (typeof result !== "string" || !stream && isExecSessionLostBanner(result, process.providerSessionId))
|
|
3172
3997
|
return;
|
|
@@ -3183,13 +4008,25 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3183
4008
|
if (events.length && bus)
|
|
3184
4009
|
await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
|
|
3185
4010
|
},
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
})
|
|
4011
|
+
commandPersistence,
|
|
4012
|
+
drainBudget
|
|
4013
|
+
).then(
|
|
4014
|
+
(value) => ({ ok: true, value }),
|
|
4015
|
+
(error) => ({ ok: false, error })
|
|
3192
4016
|
);
|
|
4017
|
+
if (legacyDeadlineStop && !await requestRetainedProcessDeadlineCancellation(db, processScope))
|
|
4018
|
+
throw new Error("Deadline cancellation no longer owns its rotating lease");
|
|
4019
|
+
if (!probeOutcome.ok) throw probeOutcome.error;
|
|
4020
|
+
observation = probeOutcome.value;
|
|
4021
|
+
if (supervised) {
|
|
4022
|
+
supervisionMetric(
|
|
4023
|
+
await commandPersistence.loadSupervisionReceipt() ? "proof_retained" : "proof_missing"
|
|
4024
|
+
);
|
|
4025
|
+
if (observation.status === "deferred")
|
|
4026
|
+
supervisionMetric(
|
|
4027
|
+
observation.reason === "provider_running" ? "checkpoint_blocked" : "provider_failure"
|
|
4028
|
+
);
|
|
4029
|
+
}
|
|
3193
4030
|
} catch (error) {
|
|
3194
4031
|
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
3195
4032
|
processId: process.id,
|
|
@@ -3230,7 +4067,8 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3230
4067
|
process,
|
|
3231
4068
|
expected,
|
|
3232
4069
|
claim.claimId,
|
|
3233
|
-
observation.reason
|
|
4070
|
+
observation.reason,
|
|
4071
|
+
observation.outputBacklog === true
|
|
3234
4072
|
);
|
|
3235
4073
|
continue;
|
|
3236
4074
|
}
|
|
@@ -3248,10 +4086,7 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3248
4086
|
recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);
|
|
3249
4087
|
} catch (error) {
|
|
3250
4088
|
recordRetainedProcessReconciliation(observability, "proof_checkpoint_failed");
|
|
3251
|
-
observability
|
|
3252
|
-
processId: process.id,
|
|
3253
|
-
error: error instanceof Error ? error.message : String(error)
|
|
3254
|
-
});
|
|
4089
|
+
warnRetainedProcessProofFailure(observability, error, process);
|
|
3255
4090
|
continue;
|
|
3256
4091
|
}
|
|
3257
4092
|
}
|
|
@@ -3304,9 +4139,10 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3304
4139
|
}
|
|
3305
4140
|
}
|
|
3306
4141
|
}
|
|
3307
|
-
async function deferRetainedProcessClaim(db, settings, observability, process, expected, claimId, outcome) {
|
|
3308
|
-
const deferral = retainedProcessReconciliationDeferral(settings, process, outcome);
|
|
4142
|
+
async function deferRetainedProcessClaim(db, settings, observability, process, expected, claimId, outcome, outputBacklog = false) {
|
|
4143
|
+
const deferral = retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog);
|
|
3309
4144
|
try {
|
|
4145
|
+
const lease = process.providerBackend === "modal" && process.routeTargetId === null ? await readLease2(db, process.workspaceId, process.sandboxGroupId) : null;
|
|
3310
4146
|
await deferRetainedProcessReconciliation(db, {
|
|
3311
4147
|
accountId: process.accountId,
|
|
3312
4148
|
workspaceId: process.workspaceId,
|
|
@@ -3315,7 +4151,7 @@ async function deferRetainedProcessClaim(db, settings, observability, process, e
|
|
|
3315
4151
|
expected,
|
|
3316
4152
|
claimId,
|
|
3317
4153
|
outcome: deferral.durableOutcome,
|
|
3318
|
-
retryAfterMs: deferral.retryAfterMs
|
|
4154
|
+
retryAfterMs: retainedProcessDeadlineRetryMs(process, lease, settings, deferral.retryAfterMs)
|
|
3319
4155
|
});
|
|
3320
4156
|
recordRetainedProcessReconciliation(observability, deferral.metricOutcome);
|
|
3321
4157
|
} catch (error) {
|
|
@@ -3326,7 +4162,7 @@ async function deferRetainedProcessClaim(db, settings, observability, process, e
|
|
|
3326
4162
|
});
|
|
3327
4163
|
}
|
|
3328
4164
|
}
|
|
3329
|
-
function retainedProcessReconciliationDeferral(settings, process, outcome) {
|
|
4165
|
+
function retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog = false) {
|
|
3330
4166
|
const bindingQuarantine = process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS && (outcome === "provider_binding_missing" || outcome === "provider_binding_mismatch" || outcome === "process_observation_unavailable");
|
|
3331
4167
|
if (bindingQuarantine) {
|
|
3332
4168
|
return {
|
|
@@ -3336,11 +4172,9 @@ function retainedProcessReconciliationDeferral(settings, process, outcome) {
|
|
|
3336
4172
|
quarantined: true
|
|
3337
4173
|
};
|
|
3338
4174
|
}
|
|
3339
|
-
const
|
|
3340
|
-
const
|
|
3341
|
-
|
|
3342
|
-
Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
|
|
3343
|
-
);
|
|
4175
|
+
const baseRetryMs = Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4);
|
|
4176
|
+
const exponent = outcome === "provider_running" && outputBacklog ? 0 : Math.min(4, Math.max(0, process.reconcileAttempts - 1));
|
|
4177
|
+
const retryAfterMs = Math.min(5 * 6e4, baseRetryMs * 2 ** exponent);
|
|
3344
4178
|
return {
|
|
3345
4179
|
durableOutcome: outcome,
|
|
3346
4180
|
metricOutcome: outcome,
|
|
@@ -3349,6 +4183,45 @@ function retainedProcessReconciliationDeferral(settings, process, outcome) {
|
|
|
3349
4183
|
};
|
|
3350
4184
|
}
|
|
3351
4185
|
var RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
|
|
4186
|
+
async function drainRetainedCommandBacklog(session, providerSessionId, first, capturePage, budget) {
|
|
4187
|
+
let backlog = retainedBacklogRemains(session, first);
|
|
4188
|
+
await capturePage(first);
|
|
4189
|
+
let observation = classifyRetainedProcessPollResult(first, providerSessionId, session);
|
|
4190
|
+
if (typeof session.writeStdin !== "function") return observation;
|
|
4191
|
+
const until = Math.min(
|
|
4192
|
+
Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,
|
|
4193
|
+
budget?.until ?? Number.POSITIVE_INFINITY
|
|
4194
|
+
);
|
|
4195
|
+
for (let reads = 0; backlog && observation.status === "deferred" && observation.reason === "provider_running" && reads < RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS && Date.now() < until; reads++) {
|
|
4196
|
+
let next;
|
|
4197
|
+
try {
|
|
4198
|
+
next = await withRetainedProcessProbeTimeout(
|
|
4199
|
+
session.writeStdin({
|
|
4200
|
+
sessionId: providerSessionId,
|
|
4201
|
+
chars: "",
|
|
4202
|
+
yieldTimeMs: 1e3,
|
|
4203
|
+
maxOutputTokens: 2e3
|
|
4204
|
+
})
|
|
4205
|
+
);
|
|
4206
|
+
} catch {
|
|
4207
|
+
break;
|
|
4208
|
+
}
|
|
4209
|
+
backlog = retainedBacklogRemains(session, next);
|
|
4210
|
+
await capturePage(next);
|
|
4211
|
+
observation = classifyRetainedProcessPollResult(next, providerSessionId, session);
|
|
4212
|
+
}
|
|
4213
|
+
return backlog && observation.status === "deferred" && observation.reason === "provider_running" ? { ...observation, outputBacklog: true } : observation;
|
|
4214
|
+
}
|
|
4215
|
+
function retainedBacklogRemains(session, result) {
|
|
4216
|
+
const page = session.getProviderCommandOutput?.(result);
|
|
4217
|
+
if (!page?.expected || page.command.kind !== "modal-router-v1") return false;
|
|
4218
|
+
if (page.command.supervision || page.exitCode !== null) return false;
|
|
4219
|
+
if (page.providerExited) return true;
|
|
4220
|
+
const { command, expected } = page;
|
|
4221
|
+
return ["stdout", "stderr"].some(
|
|
4222
|
+
(stream) => command.streams[stream].byteOffset - expected.streams[stream].byteOffset >= RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES
|
|
4223
|
+
);
|
|
4224
|
+
}
|
|
3352
4225
|
async function withRetainedProcessProbeTimeout(promise) {
|
|
3353
4226
|
let timer;
|
|
3354
4227
|
try {
|
|
@@ -3376,8 +4249,8 @@ async function captureRetainedProbeOutput(processId, result, capture) {
|
|
|
3376
4249
|
await capture(pending.result, pending.chunkId);
|
|
3377
4250
|
pendingRetainedProbeOutput.delete(processId);
|
|
3378
4251
|
}
|
|
3379
|
-
async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput, providerPersistence) {
|
|
3380
|
-
if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
|
|
4252
|
+
async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput, providerPersistence, drainBudget) {
|
|
4253
|
+
if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || !lease.resumeBackendId || sandboxBackendForSdkBackendId(lease.resumeBackendId) !== process.providerBackend || process.routeTargetId !== null) {
|
|
3381
4254
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
3382
4255
|
}
|
|
3383
4256
|
if (!lease.resumeState) {
|
|
@@ -3387,10 +4260,14 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3387
4260
|
if (pending) {
|
|
3388
4261
|
if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
|
|
3389
4262
|
await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
|
|
3390
|
-
|
|
4263
|
+
const pendingObservation = classifyRetainedProcessPollResult(
|
|
4264
|
+
pending.result,
|
|
4265
|
+
process.providerSessionId
|
|
4266
|
+
);
|
|
4267
|
+
if (mode !== "cancel" || pendingObservation.status === "proved") return pendingObservation;
|
|
3391
4268
|
}
|
|
3392
4269
|
const envelopeBackend = lease.resumeState.backendId;
|
|
3393
|
-
if (envelopeBackend !== void 0 && envelopeBackend !== process.providerBackend) {
|
|
4270
|
+
if (envelopeBackend !== void 0 && (typeof envelopeBackend !== "string" || sandboxBackendForSdkBackendId(envelopeBackend) !== process.providerBackend)) {
|
|
3394
4271
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
3395
4272
|
}
|
|
3396
4273
|
if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
|
|
@@ -3414,17 +4291,17 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3414
4291
|
}
|
|
3415
4292
|
let client;
|
|
3416
4293
|
try {
|
|
3417
|
-
client =
|
|
4294
|
+
client = createSandboxClientForBackend2(process.providerBackend, settings);
|
|
3418
4295
|
} catch {
|
|
3419
4296
|
return { status: "deferred", reason: "provider_error" };
|
|
3420
4297
|
}
|
|
3421
|
-
if (!client || client.backendId !== process.providerBackend || !client.resume) {
|
|
4298
|
+
if (!client || sandboxBackendForSdkBackendId(client.backendId) !== process.providerBackend || !client.resume) {
|
|
3422
4299
|
return { status: "deferred", reason: "backend_unsupported" };
|
|
3423
4300
|
}
|
|
3424
4301
|
const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
|
|
3425
4302
|
let resumedState;
|
|
3426
4303
|
try {
|
|
3427
|
-
resumedState = await
|
|
4304
|
+
resumedState = await deserializeSandboxSessionStateEnvelope2(
|
|
3428
4305
|
client,
|
|
3429
4306
|
envelopeSessionState,
|
|
3430
4307
|
process.providerInstanceId
|
|
@@ -3438,7 +4315,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3438
4315
|
let session;
|
|
3439
4316
|
try {
|
|
3440
4317
|
const resumed = await withRetainedProcessProbeTimeout(
|
|
3441
|
-
|
|
4318
|
+
resumeExactSandboxSession2(
|
|
3442
4319
|
client,
|
|
3443
4320
|
process.providerBackend,
|
|
3444
4321
|
resumedState,
|
|
@@ -3468,6 +4345,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3468
4345
|
if (typeof session.writeStdin !== "function") {
|
|
3469
4346
|
return { status: "deferred", reason: "backend_unsupported" };
|
|
3470
4347
|
}
|
|
4348
|
+
let legacyPty = false;
|
|
3471
4349
|
if (process.providerBackend === "modal") {
|
|
3472
4350
|
let liveBinding;
|
|
3473
4351
|
try {
|
|
@@ -3494,6 +4372,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3494
4372
|
if (!command || !providerPersistence || !session.bindProviderCommand)
|
|
3495
4373
|
return { status: "deferred", reason: "process_observation_unavailable" };
|
|
3496
4374
|
session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
|
|
4375
|
+
legacyPty = command.pty === true;
|
|
3497
4376
|
}
|
|
3498
4377
|
const capturePage = async (value) => {
|
|
3499
4378
|
if (!captureOutput) {
|
|
@@ -3502,6 +4381,11 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3502
4381
|
return;
|
|
3503
4382
|
}
|
|
3504
4383
|
const page = session.getProviderCommandOutput?.(value);
|
|
4384
|
+
if (page?.command.kind === "modal-router-v1" && typeof value === "string") {
|
|
4385
|
+
if (!session.captureCommandOutput || !await session.captureCommandOutput(value))
|
|
4386
|
+
throw new Error("Byte-offset output requires atomic capture before settlement");
|
|
4387
|
+
return;
|
|
4388
|
+
}
|
|
3505
4389
|
if (page) {
|
|
3506
4390
|
for (const chunk of page.chunks)
|
|
3507
4391
|
if (chunk.text)
|
|
@@ -3513,10 +4397,16 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3513
4397
|
};
|
|
3514
4398
|
let result;
|
|
3515
4399
|
try {
|
|
4400
|
+
const supervisedCancelled = mode === "cancel" || lease.rotationReason === "provider_deadline" ? await session.cancelSupervisedCommand?.(
|
|
4401
|
+
process.providerSessionId,
|
|
4402
|
+
lease.rotationReason === "provider_deadline" ? "provider_deadline" : "explicit_stop"
|
|
4403
|
+
) ?? false : false;
|
|
3516
4404
|
result = await withRetainedProcessProbeTimeout(
|
|
3517
4405
|
session.writeStdin({
|
|
3518
4406
|
sessionId: process.providerSessionId,
|
|
3519
|
-
|
|
4407
|
+
// Ctrl-C is a signal only through a PTY. For a non-PTY command it is
|
|
4408
|
+
// ordinary stdin data and can corrupt the command's input.
|
|
4409
|
+
chars: mode === "cancel" && !supervisedCancelled && (process.providerBackend !== "modal" || legacyPty) ? "" : "",
|
|
3520
4410
|
yieldTimeMs: 1e3,
|
|
3521
4411
|
maxOutputTokens: 2e3
|
|
3522
4412
|
})
|
|
@@ -3540,9 +4430,14 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3540
4430
|
}
|
|
3541
4431
|
return { status: "deferred", reason: "provider_error" };
|
|
3542
4432
|
}
|
|
3543
|
-
await
|
|
3544
|
-
|
|
3545
|
-
|
|
4433
|
+
const observation = await drainRetainedCommandBacklog(
|
|
4434
|
+
session,
|
|
4435
|
+
process.providerSessionId,
|
|
4436
|
+
result,
|
|
4437
|
+
capturePage,
|
|
4438
|
+
drainBudget
|
|
4439
|
+
);
|
|
4440
|
+
if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null && mode !== "cancel" && !process.cancellationRequestedAt && legacyPty && !await providerPersistence?.load().then((command) => command?.kind === "modal-router-v1" && command.supervision)) {
|
|
3546
4441
|
try {
|
|
3547
4442
|
const interrupted = await withRetainedProcessProbeTimeout(
|
|
3548
4443
|
session.writeStdin({
|
|
@@ -3619,19 +4514,22 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3619
4514
|
}
|
|
3620
4515
|
await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
|
|
3621
4516
|
try {
|
|
3622
|
-
const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
4517
|
+
const rate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
3623
4518
|
const result = await accrueWarmSeconds(db, {
|
|
3624
4519
|
accountId: lease.accountId,
|
|
3625
4520
|
workspaceId: lease.workspaceId,
|
|
3626
4521
|
sandboxGroupId: lease.sandboxGroupId,
|
|
3627
4522
|
expectedEpoch: lease.leaseEpoch,
|
|
3628
4523
|
warmRateMicrosPerSecond: rate,
|
|
4524
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
3629
4525
|
subjectId: lease.sandboxGroupId
|
|
3630
4526
|
});
|
|
3631
4527
|
if (result.accrued) {
|
|
3632
4528
|
accrued += 1;
|
|
3633
4529
|
}
|
|
3634
|
-
|
|
4530
|
+
if (result.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
4531
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
4532
|
+
}
|
|
3635
4533
|
} catch (error) {
|
|
3636
4534
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
3637
4535
|
workspaceId: lease.workspaceId,
|
|
@@ -3667,6 +4565,17 @@ async function refreshQueueLeaseAndCreditGauges(db, settings, observability, ins
|
|
|
3667
4565
|
);
|
|
3668
4566
|
}
|
|
3669
4567
|
),
|
|
4568
|
+
refreshSandboxInventoryGauge(
|
|
4569
|
+
observability,
|
|
4570
|
+
"recovery_observations",
|
|
4571
|
+
"recovery-observations",
|
|
4572
|
+
async () => {
|
|
4573
|
+
recordSandboxRecoveryObservationGauges(
|
|
4574
|
+
observability,
|
|
4575
|
+
await readRecentSandboxRecoveryObservations(db)
|
|
4576
|
+
);
|
|
4577
|
+
}
|
|
4578
|
+
),
|
|
3670
4579
|
refreshSandboxInventoryGauge(
|
|
3671
4580
|
observability,
|
|
3672
4581
|
"rotation_backlog",
|
|
@@ -3715,6 +4624,51 @@ async function refreshQueueLeaseAndCreditGauges(db, settings, observability, ins
|
|
|
3715
4624
|
error: error instanceof Error ? error.message : String(error)
|
|
3716
4625
|
});
|
|
3717
4626
|
}
|
|
4627
|
+
})(),
|
|
4628
|
+
(async () => {
|
|
4629
|
+
try {
|
|
4630
|
+
recordActiveUserGauges(observability, await countActiveUsers(db));
|
|
4631
|
+
} catch (error) {
|
|
4632
|
+
observability.warn("sandbox reaper: active-user gauge refresh failed", {
|
|
4633
|
+
error: error instanceof Error ? error.message : String(error)
|
|
4634
|
+
});
|
|
4635
|
+
}
|
|
4636
|
+
})(),
|
|
4637
|
+
(async () => {
|
|
4638
|
+
try {
|
|
4639
|
+
recordCreditGrantGauges(observability, await readCreditGrantTotals(db));
|
|
4640
|
+
} catch (error) {
|
|
4641
|
+
observability.warn("sandbox reaper: credit-grant gauge refresh failed", {
|
|
4642
|
+
error: error instanceof Error ? error.message : String(error)
|
|
4643
|
+
});
|
|
4644
|
+
}
|
|
4645
|
+
})(),
|
|
4646
|
+
(async () => {
|
|
4647
|
+
recordVerifiedSignupTrialDeploymentFlagGauge(
|
|
4648
|
+
observability,
|
|
4649
|
+
settings.verifiedSignupTrialCreditsEnabled
|
|
4650
|
+
);
|
|
4651
|
+
try {
|
|
4652
|
+
const trialSwitch = await readVerifiedSignupTrialSwitch(db);
|
|
4653
|
+
recordVerifiedSignupTrialSwitchGauge(observability, trialSwitch?.grantsEnabled === true);
|
|
4654
|
+
} catch (error) {
|
|
4655
|
+
observability.warn("sandbox reaper: trial-credit switch gauge refresh failed", {
|
|
4656
|
+
error: error instanceof Error ? error.message : String(error)
|
|
4657
|
+
});
|
|
4658
|
+
}
|
|
4659
|
+
})(),
|
|
4660
|
+
(async () => {
|
|
4661
|
+
try {
|
|
4662
|
+
const signupsSwitch = await readManagedAuthNewSignupsSwitch(db);
|
|
4663
|
+
recordManagedAuthNewSignupsSwitchGauge(
|
|
4664
|
+
observability,
|
|
4665
|
+
signupsSwitch?.signupsEnabled !== false
|
|
4666
|
+
);
|
|
4667
|
+
} catch (error) {
|
|
4668
|
+
observability.warn("sandbox reaper: new-signups switch gauge refresh failed", {
|
|
4669
|
+
error: error instanceof Error ? error.message : String(error)
|
|
4670
|
+
});
|
|
4671
|
+
}
|
|
3718
4672
|
})()
|
|
3719
4673
|
]);
|
|
3720
4674
|
}
|
|
@@ -3730,7 +4684,7 @@ async function refreshSandboxInventoryGauge(observability, domain, diagnosticNam
|
|
|
3730
4684
|
}
|
|
3731
4685
|
}
|
|
3732
4686
|
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
3733
|
-
const enforceBalance =
|
|
4687
|
+
const enforceBalance = false;
|
|
3734
4688
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
3735
4689
|
let forceDrained = 0;
|
|
3736
4690
|
await forEachWithConcurrency(
|
|
@@ -3738,11 +4692,8 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
3738
4692
|
SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
|
|
3739
4693
|
async (workspaceId) => {
|
|
3740
4694
|
try {
|
|
3741
|
-
const { accountId } = await rlsContextForWorkspace(db, workspaceId);
|
|
3742
|
-
const balance = enforceBalance ? await getBillingBalance3(db, accountId) : { balanceMicros: 1 };
|
|
3743
4695
|
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
3744
4696
|
workspaceId,
|
|
3745
|
-
balanceMicros: balance.balanceMicros,
|
|
3746
4697
|
enforceBalance,
|
|
3747
4698
|
maxWarmSecondsPerWorkspace: cap,
|
|
3748
4699
|
idleGraceMs: settings.sandboxIdleGraceMs
|
|
@@ -3765,7 +4716,37 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
3765
4716
|
);
|
|
3766
4717
|
return forceDrained;
|
|
3767
4718
|
}
|
|
4719
|
+
async function reconcileModalProviderCreates(db, settings, observability, discover = findModalProviderCreateReceipt) {
|
|
4720
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret)
|
|
4721
|
+
return;
|
|
4722
|
+
const candidates = await listPendingModalProviderCreates(db);
|
|
4723
|
+
await forEachWithConcurrency(candidates, 2, async (candidate) => {
|
|
4724
|
+
try {
|
|
4725
|
+
const { accountId } = await rlsContextForWorkspace(db, candidate.workspaceId);
|
|
4726
|
+
const identity = { ...candidate, accountId };
|
|
4727
|
+
const attempt = await claimModalProviderCreateRecovery(db, identity);
|
|
4728
|
+
if (!attempt) return;
|
|
4729
|
+
const instanceId = await discover(settings, attempt);
|
|
4730
|
+
if (!instanceId) return;
|
|
4731
|
+
if (await recordRecoveredModalProviderCreate(db, { ...identity, attempt, instanceId })) {
|
|
4732
|
+
observability.info("sandbox reaper: recovered Modal create receipt", {
|
|
4733
|
+
...candidate,
|
|
4734
|
+
operationId: attempt.operationId,
|
|
4735
|
+
instanceId
|
|
4736
|
+
});
|
|
4737
|
+
}
|
|
4738
|
+
} catch (error) {
|
|
4739
|
+
observability.warn("sandbox reaper: Modal create remains fenced pending discovery", {
|
|
4740
|
+
...candidate,
|
|
4741
|
+
error: error instanceof Error ? error.message : String(error)
|
|
4742
|
+
});
|
|
4743
|
+
}
|
|
4744
|
+
});
|
|
4745
|
+
}
|
|
3768
4746
|
function modalOrphanTerminationStillEligible(latest, candidate) {
|
|
4747
|
+
if (latest.some((lease) => lease.liveness === "warming" && lease.instanceId === null)) {
|
|
4748
|
+
return false;
|
|
4749
|
+
}
|
|
3769
4750
|
if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
|
|
3770
4751
|
return false;
|
|
3771
4752
|
}
|
|
@@ -3875,9 +4856,9 @@ function sandboxDrainReason(lease) {
|
|
|
3875
4856
|
if (lease.rotationRequestedAt === null) return "idle";
|
|
3876
4857
|
return lease.rotationReason ?? "rotation_requested";
|
|
3877
4858
|
}
|
|
3878
|
-
async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage) {
|
|
4859
|
+
async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage, bus = null) {
|
|
3879
4860
|
const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
|
|
3880
|
-
const lease = await
|
|
4861
|
+
const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
|
|
3881
4862
|
if (!lease) {
|
|
3882
4863
|
return false;
|
|
3883
4864
|
}
|
|
@@ -4118,6 +5099,27 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
4118
5099
|
}
|
|
4119
5100
|
return result;
|
|
4120
5101
|
};
|
|
5102
|
+
if (!providerMissingBeforeCapture && backend === "modal" && lease.instanceId && !await readWorkspaceArchiveCapturePreflight(db, {
|
|
5103
|
+
accountId,
|
|
5104
|
+
workspaceId: row.workspaceId,
|
|
5105
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
5106
|
+
expectedEpoch: row.leaseEpoch,
|
|
5107
|
+
expectedInstanceId: lease.instanceId,
|
|
5108
|
+
liveness: "draining"
|
|
5109
|
+
})) {
|
|
5110
|
+
return false;
|
|
5111
|
+
}
|
|
5112
|
+
let stopCutoffMarked = false;
|
|
5113
|
+
const beforeProviderStop = async () => {
|
|
5114
|
+
const cutoff = await markWarmBillingStopCutoff(db, {
|
|
5115
|
+
accountId,
|
|
5116
|
+
workspaceId: row.workspaceId,
|
|
5117
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
5118
|
+
expectedEpoch: row.leaseEpoch
|
|
5119
|
+
});
|
|
5120
|
+
if (!cutoff) throw new Error("sandbox stop cutoff was fenced before provider termination");
|
|
5121
|
+
stopCutoffMarked = true;
|
|
5122
|
+
};
|
|
4121
5123
|
const termination = providerMissingBeforeCapture ? { terminated: true, providerMissingBeforeCapture: true } : await terminateBox(
|
|
4122
5124
|
settings,
|
|
4123
5125
|
lease,
|
|
@@ -4137,22 +5139,62 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
4137
5139
|
expectedEpoch: row.leaseEpoch,
|
|
4138
5140
|
expectedInstanceId: lease.instanceId
|
|
4139
5141
|
});
|
|
4140
|
-
}
|
|
5142
|
+
},
|
|
5143
|
+
row.workspaceId,
|
|
5144
|
+
beforeProviderStop
|
|
4141
5145
|
);
|
|
4142
5146
|
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
4143
5147
|
if (!terminated) {
|
|
4144
5148
|
return false;
|
|
4145
5149
|
}
|
|
5150
|
+
const providerMissingDuringStop = providerMissingBeforeCapture || typeof termination !== "boolean" && termination.providerMissingBeforeCapture;
|
|
5151
|
+
if (!stopCutoffMarked && !providerMissingDuringStop) {
|
|
5152
|
+
if (managedProvider && lease.instanceId) {
|
|
5153
|
+
throw new Error("managed sandbox stopped without a warm usage cutoff");
|
|
5154
|
+
}
|
|
5155
|
+
await beforeProviderStop();
|
|
5156
|
+
}
|
|
5157
|
+
const finalRate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, backend);
|
|
5158
|
+
if (!providerMissingDuringStop) {
|
|
5159
|
+
const finalTick = await accrueWarmSeconds(db, {
|
|
5160
|
+
accountId,
|
|
5161
|
+
workspaceId: row.workspaceId,
|
|
5162
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
5163
|
+
expectedEpoch: row.leaseEpoch,
|
|
5164
|
+
warmRateMicrosPerSecond: finalRate,
|
|
5165
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
5166
|
+
finalDrain: true
|
|
5167
|
+
});
|
|
5168
|
+
if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
5169
|
+
recordCreditMicros(observability, "usage", finalTick.costMicros);
|
|
5170
|
+
}
|
|
5171
|
+
}
|
|
4146
5172
|
const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
|
|
4147
|
-
const { wentCold } = await confirmDrainCold(db, {
|
|
5173
|
+
const { wentCold, backgroundCommandEvents } = await confirmDrainCold(db, {
|
|
4148
5174
|
accountId,
|
|
4149
5175
|
workspaceId: row.workspaceId,
|
|
4150
5176
|
sandboxGroupId: row.sandboxGroupId,
|
|
4151
5177
|
expectedEpoch: row.leaseEpoch,
|
|
4152
5178
|
...captureClaim ? { expectedCaptureId: captureClaim.id } : {},
|
|
4153
|
-
providerMissingBeforeCapture: providerMissing
|
|
5179
|
+
providerMissingBeforeCapture: providerMissing,
|
|
5180
|
+
idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs
|
|
5181
|
+
});
|
|
5182
|
+
await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
|
|
5183
|
+
observability.warn("sandbox reaper: contained command event fanout failed", {
|
|
5184
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
5185
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5186
|
+
});
|
|
4154
5187
|
});
|
|
4155
5188
|
if (wentCold) {
|
|
5189
|
+
if (providerMissing) {
|
|
5190
|
+
recordSandboxProviderMissingBeforeCapture(observability, backend);
|
|
5191
|
+
}
|
|
5192
|
+
if (lease.unobservableCommandDrainIds?.length) {
|
|
5193
|
+
recordSandboxCommandContainment(
|
|
5194
|
+
observability,
|
|
5195
|
+
providerMissing ? "provider_missing" : "contained"
|
|
5196
|
+
);
|
|
5197
|
+
}
|
|
4156
5198
|
try {
|
|
4157
5199
|
await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
|
|
4158
5200
|
type: "sandbox.box.terminated",
|
|
@@ -4193,18 +5235,20 @@ function providerDrainLogIdentity(lease, backend) {
|
|
|
4193
5235
|
instanceId: lease.instanceId
|
|
4194
5236
|
};
|
|
4195
5237
|
}
|
|
4196
|
-
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend =
|
|
5238
|
+
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend2, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture, workspaceId, beforeProviderStop) {
|
|
4197
5239
|
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
4198
5240
|
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
4199
5241
|
const logIdentity = providerDrainLogIdentity(lease, backend);
|
|
5242
|
+
const warnCaptureFailure = (message, error) => warnDrainSnapshotFailure(observability, message, error, {
|
|
5243
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
5244
|
+
leaseEpoch: lease.leaseEpoch,
|
|
5245
|
+
...workspaceId ? { workspaceId } : {}
|
|
5246
|
+
});
|
|
4200
5247
|
const releaseAfterCaptureFailure = async () => {
|
|
4201
5248
|
try {
|
|
4202
5249
|
await releaseFailedCapture?.();
|
|
4203
5250
|
} catch (error) {
|
|
4204
|
-
|
|
4205
|
-
...logIdentity,
|
|
4206
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4207
|
-
});
|
|
5251
|
+
warnCaptureFailure("sandbox reaper: failed capture claim release failed", error);
|
|
4208
5252
|
}
|
|
4209
5253
|
};
|
|
4210
5254
|
if (!backend || backend === "none") {
|
|
@@ -4230,6 +5274,11 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4230
5274
|
}
|
|
4231
5275
|
const persistedInstanceId = assertConsistentSandboxProviderIdentity(backend, lease.resumeState);
|
|
4232
5276
|
if (!lease.instanceId) {
|
|
5277
|
+
if (lease.providerCreateAttempt?.instanceId === null) {
|
|
5278
|
+
throw new Error(
|
|
5279
|
+
"provider_create_outcome_unknown: cannot infer termination from missing instance ID"
|
|
5280
|
+
);
|
|
5281
|
+
}
|
|
4233
5282
|
if (persistedInstanceId) {
|
|
4234
5283
|
throw new Error(
|
|
4235
5284
|
`sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`
|
|
@@ -4255,7 +5304,13 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4255
5304
|
return { terminated: false, providerMissingBeforeCapture: false };
|
|
4256
5305
|
}
|
|
4257
5306
|
try {
|
|
4258
|
-
await
|
|
5307
|
+
await beforeProviderStop?.();
|
|
5308
|
+
await terminateModalById(
|
|
5309
|
+
settings,
|
|
5310
|
+
lease.instanceId,
|
|
5311
|
+
void 0,
|
|
5312
|
+
lease.providerCreateAttempt?.providerBindingKey
|
|
5313
|
+
);
|
|
4259
5314
|
} catch (error) {
|
|
4260
5315
|
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
4261
5316
|
throw error;
|
|
@@ -4282,7 +5337,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4282
5337
|
throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);
|
|
4283
5338
|
}
|
|
4284
5339
|
const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
|
|
4285
|
-
const resumedState = await
|
|
5340
|
+
const resumedState = await deserializeSandboxSessionStateEnvelope2(
|
|
4286
5341
|
client,
|
|
4287
5342
|
envelopeSessionState,
|
|
4288
5343
|
lease.instanceId
|
|
@@ -4291,7 +5346,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4291
5346
|
throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
|
|
4292
5347
|
}
|
|
4293
5348
|
const continuity = sandboxProviderContinuityForState(backend, resumedState, lease.instanceId);
|
|
4294
|
-
const resumed = await
|
|
5349
|
+
const resumed = await resumeExactSandboxSession2(
|
|
4295
5350
|
client,
|
|
4296
5351
|
backend,
|
|
4297
5352
|
resumedState,
|
|
@@ -4329,7 +5384,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4329
5384
|
session,
|
|
4330
5385
|
Date.now(),
|
|
4331
5386
|
{
|
|
4332
|
-
requestId: providerCaptureRequestId ??
|
|
5387
|
+
requestId: providerCaptureRequestId ?? randomUUID3(),
|
|
4333
5388
|
strategy: liveCapturePolicy.strategy
|
|
4334
5389
|
},
|
|
4335
5390
|
diskBackedArchives
|
|
@@ -4359,26 +5414,17 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4359
5414
|
}
|
|
4360
5415
|
);
|
|
4361
5416
|
} catch (error) {
|
|
4362
|
-
|
|
4363
|
-
...logIdentity,
|
|
4364
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4365
|
-
});
|
|
5417
|
+
warnCaptureFailure("sandbox reaper: late workspace capture publication failed", error);
|
|
4366
5418
|
} finally {
|
|
4367
5419
|
await disposeWorkspaceArchive(archive);
|
|
4368
5420
|
}
|
|
4369
5421
|
},
|
|
4370
5422
|
observeLatePublicationFailure: (error) => {
|
|
4371
|
-
|
|
4372
|
-
...logIdentity,
|
|
4373
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4374
|
-
});
|
|
5423
|
+
warnCaptureFailure("sandbox reaper: late capture publication cleanup failed", error);
|
|
4375
5424
|
},
|
|
4376
5425
|
observeLateFailure: (error) => {
|
|
4377
5426
|
void releaseAfterCaptureFailure();
|
|
4378
|
-
|
|
4379
|
-
...logIdentity,
|
|
4380
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4381
|
-
});
|
|
5427
|
+
warnCaptureFailure("sandbox reaper: timed-out provider capture later failed", error);
|
|
4382
5428
|
}
|
|
4383
5429
|
});
|
|
4384
5430
|
}
|
|
@@ -4389,12 +5435,9 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4389
5435
|
});
|
|
4390
5436
|
return { terminated: true, providerMissingBeforeCapture: true };
|
|
4391
5437
|
}
|
|
4392
|
-
|
|
5438
|
+
warnCaptureFailure(
|
|
4393
5439
|
"sandbox reaper: persistWorkspace failed \u2014 leaving box draining (files NOT lost)",
|
|
4394
|
-
|
|
4395
|
-
...logIdentity,
|
|
4396
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4397
|
-
}
|
|
5440
|
+
error
|
|
4398
5441
|
);
|
|
4399
5442
|
if (!(error instanceof SandboxProviderCaptureTimeoutError)) {
|
|
4400
5443
|
await releaseAfterCaptureFailure();
|
|
@@ -4436,6 +5479,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4436
5479
|
throw new Error(`sandbox backend ${backend} returned no terminable session state`);
|
|
4437
5480
|
}
|
|
4438
5481
|
prepareProviderForTeardownAfterCapture(backend, session);
|
|
5482
|
+
await beforeProviderStop?.();
|
|
4439
5483
|
await terminateManagedSandboxSession2(client, sessionState, session);
|
|
4440
5484
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
4441
5485
|
} catch (error) {
|
|
@@ -4450,6 +5494,11 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4450
5494
|
}
|
|
4451
5495
|
|
|
4452
5496
|
// src/activities/scheduled-tasks.ts
|
|
5497
|
+
import { getScheduledTaskClaudeProviderAccountAuthoritySnapshot } from "@opengeni/db";
|
|
5498
|
+
import {
|
|
5499
|
+
ATLASSIAN_NATIVE_RETIRED_REASON,
|
|
5500
|
+
isRetiredNativeAtlassianTask
|
|
5501
|
+
} from "@opengeni/contracts/atlassian-native-retirement";
|
|
4453
5502
|
import {
|
|
4454
5503
|
AUTOMATIC_SESSION_TITLE_FALLBACK,
|
|
4455
5504
|
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
|
|
@@ -4462,25 +5511,40 @@ import {
|
|
|
4462
5511
|
SessionAgentAccess,
|
|
4463
5512
|
SessionMemoryScope,
|
|
4464
5513
|
normalizeAutomaticSessionTitle,
|
|
5514
|
+
metadataWithTurnExecutionPolicyV1,
|
|
5515
|
+
mergeResourceRefs,
|
|
4465
5516
|
scheduledOccurrencePayloadUtf8Bytes,
|
|
4466
5517
|
stableJson,
|
|
4467
5518
|
TurnExecutionPolicyV1
|
|
4468
5519
|
} from "@opengeni/contracts";
|
|
4469
5520
|
import {
|
|
4470
|
-
defaultSessionMcpServerIds,
|
|
4471
5521
|
openGeniSlackBotMetadata,
|
|
4472
5522
|
requireOpenGeniSlackBotConnection,
|
|
5523
|
+
withScheduledSlackBotPostingTools,
|
|
4473
5524
|
resolveWorkspaceCatalogSettings as resolveWorkspaceCatalogSettings2,
|
|
5525
|
+
resolveScheduledTaskDefaultModel,
|
|
4474
5526
|
resolveSessionToolPolicy,
|
|
4475
5527
|
workspaceCustomModelReference,
|
|
4476
5528
|
lockActiveCustomModelForAdmission,
|
|
4477
5529
|
scheduledSlackBotConnectionId,
|
|
4478
|
-
|
|
5530
|
+
freezeConnectionAccounts,
|
|
5531
|
+
ConnectionAccountSelectionError,
|
|
5532
|
+
scheduledConnectionSurfaceEligibility,
|
|
5533
|
+
scheduledConnectionTools,
|
|
5534
|
+
settingsWithEnabledCapabilityMcpServers,
|
|
5535
|
+
settingsWithSessionMcpServerMetadata,
|
|
5536
|
+
swapActiveSandbox,
|
|
5537
|
+
applySessionAgentConfigWriteThrough as applySessionAgentConfigWriteThrough2,
|
|
5538
|
+
resolveSessionAgentConfigForCreate as resolveSessionAgentConfigForCreate2,
|
|
5539
|
+
recordSessionCreated,
|
|
5540
|
+
recordUserMessageAccepted
|
|
4479
5541
|
} from "@opengeni/core";
|
|
4480
5542
|
import {
|
|
4481
5543
|
addSessionSystemUpdateWithSourceMutation,
|
|
4482
5544
|
bindScheduledTaskRunSessionInTransaction,
|
|
4483
5545
|
createScheduledTaskRun,
|
|
5546
|
+
recordScheduledTaskAdmissionFailure,
|
|
5547
|
+
recordScheduledTaskAdmissionRefusal,
|
|
4484
5548
|
createSessionWithIdempotencyKeyResult,
|
|
4485
5549
|
enqueueSessionWorkflowWakeIfRunnable,
|
|
4486
5550
|
failScheduledGeneratedSessionRoute,
|
|
@@ -4489,7 +5553,6 @@ import {
|
|
|
4489
5553
|
getScheduledTaskRunAcceptedExecution,
|
|
4490
5554
|
getScheduledTaskRunByProducerKey,
|
|
4491
5555
|
getScheduledTargetSessionExecution,
|
|
4492
|
-
getScheduledTaskPersonalConnectionDelegations,
|
|
4493
5556
|
getScheduledTaskRunPersonalResourceAuthority,
|
|
4494
5557
|
getScheduledTaskPersonalResourceAuthoritySubject,
|
|
4495
5558
|
getScheduledTaskRevisionAuthority,
|
|
@@ -4501,31 +5564,31 @@ import {
|
|
|
4501
5564
|
getNestedAgentDepthDeploymentPolicy,
|
|
4502
5565
|
getSessionByCreateIdempotencyKey,
|
|
4503
5566
|
getSessionCreationExecutionPolicy,
|
|
5567
|
+
metadataWithAgentConfigCreateIdentity,
|
|
4504
5568
|
getVariableSet,
|
|
4505
5569
|
isCodexBilledModel,
|
|
4506
5570
|
initializeSessionStartAtomically,
|
|
4507
5571
|
materializeScheduledTaskReusableSessionFromRun,
|
|
4508
|
-
listEnabledMcpCapabilityServerIds,
|
|
4509
5572
|
listInstalledApiIntegrationServerIdsForDelegations,
|
|
4510
5573
|
markScheduledTaskRunFailedIfQueued,
|
|
4511
5574
|
markScheduledTaskRunSkippedIfQueued,
|
|
4512
5575
|
recordUsageEvent as recordUsageEvent2,
|
|
4513
5576
|
requireScheduledTaskIncidentAuthorityInTransaction,
|
|
4514
5577
|
requireSession,
|
|
4515
|
-
requireWorkspace,
|
|
5578
|
+
requireWorkspace as requireWorkspace2,
|
|
4516
5579
|
settleScheduledTaskRunInTransaction,
|
|
4517
5580
|
SessionSpawnDeniedDbError,
|
|
4518
5581
|
updateSessionTitleWithEvent,
|
|
4519
5582
|
upsertScheduledSessionGoalForRun,
|
|
4520
5583
|
withSessionActivityRlsContext
|
|
4521
5584
|
} from "@opengeni/db";
|
|
4522
|
-
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
5585
|
+
import { publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
|
|
4523
5586
|
import {
|
|
4524
5587
|
allowedFirstPartyMcpToolsForSession,
|
|
4525
5588
|
resolveFirstPartyMcpToolPolicy,
|
|
4526
5589
|
resolveTurnExecutionPolicyV1 as resolveTurnExecutionPolicyV12
|
|
4527
5590
|
} from "@opengeni/config";
|
|
4528
|
-
import { Context as
|
|
5591
|
+
import { Context as Context3 } from "@temporalio/activity";
|
|
4529
5592
|
import { createHash as createHash3 } from "crypto";
|
|
4530
5593
|
|
|
4531
5594
|
// src/scheduled-alert-occurrence.ts
|
|
@@ -4641,13 +5704,21 @@ function canonicalLabels(value) {
|
|
|
4641
5704
|
}
|
|
4642
5705
|
|
|
4643
5706
|
// src/activities/scheduled-tasks.ts
|
|
5707
|
+
function scheduledTaskRunExecutionPolicy(policy, input, creatorRestriction) {
|
|
5708
|
+
return creatorRestriction === "developer_setup" || input.triggerType !== "scheduled" && input.credentialRestriction === "developer_setup" ? { ...policy, credentialRestriction: "developer_setup" } : policy;
|
|
5709
|
+
}
|
|
5710
|
+
function scheduledSessionExecutionPolicyMetadata(policy, creatorRestriction) {
|
|
5711
|
+
if (policy === void 0 || policy === null) return {};
|
|
5712
|
+
const acceptedPolicy = TurnExecutionPolicyV1.parse(policy);
|
|
5713
|
+
return creatorRestriction === "developer_setup" && acceptedPolicy.credentialRestriction === "developer_setup" ? metadataWithTurnExecutionPolicyV1({}, acceptedPolicy) : {};
|
|
5714
|
+
}
|
|
4644
5715
|
function scheduledTaskRunProducerKey(input, suppliedActivityIdentity) {
|
|
4645
5716
|
if (input.producerKey) return input.producerKey;
|
|
4646
5717
|
if (input.triggerType !== "scheduled") return input.agentRunUsageIdempotencyKey;
|
|
4647
5718
|
let info = suppliedActivityIdentity;
|
|
4648
5719
|
if (!info) {
|
|
4649
5720
|
try {
|
|
4650
|
-
info =
|
|
5721
|
+
info = Context3.current().info;
|
|
4651
5722
|
} catch {
|
|
4652
5723
|
throw new Error(
|
|
4653
5724
|
"scheduled task delivery without a producer key requires a Temporal activity execution"
|
|
@@ -4666,6 +5737,13 @@ function scheduledTaskGeneratedSessionCreateIdempotencyKey(producerKey) {
|
|
|
4666
5737
|
const digest = createHash3("sha256").update("scheduled-task-run\0", "utf8").update(producerKey, "utf8").digest("hex");
|
|
4667
5738
|
return `scheduled-task-run:${digest}`;
|
|
4668
5739
|
}
|
|
5740
|
+
function recordScheduledSessionCreated(observability) {
|
|
5741
|
+
recordSessionCreated(observability, {
|
|
5742
|
+
surface: "scheduled",
|
|
5743
|
+
createdByKind: "service",
|
|
5744
|
+
parentSessionId: null
|
|
5745
|
+
});
|
|
5746
|
+
}
|
|
4669
5747
|
function scheduledTaskSessionTitle(taskName) {
|
|
4670
5748
|
return normalizeAutomaticSessionTitle(taskName) ?? "Scheduled run";
|
|
4671
5749
|
}
|
|
@@ -4708,6 +5786,9 @@ function createScheduledTaskActivities(services) {
|
|
|
4708
5786
|
producerKey: stableProducerKey
|
|
4709
5787
|
});
|
|
4710
5788
|
if (priorRun?.actionKind === "agent_turn") {
|
|
5789
|
+
if (priorRun.admissionDiagnostic || priorRun.admissionRefusal) {
|
|
5790
|
+
return scheduledAdmissionRefusalResult(priorRun);
|
|
5791
|
+
}
|
|
4711
5792
|
const acceptedExecution = await getScheduledTaskRunAcceptedExecution(db, {
|
|
4712
5793
|
workspaceId: input.workspaceId,
|
|
4713
5794
|
runId: priorRun.id
|
|
@@ -4718,6 +5799,15 @@ function createScheduledTaskActivities(services) {
|
|
|
4718
5799
|
}
|
|
4719
5800
|
throw new Error("scheduled agent run is missing accepted execution truth");
|
|
4720
5801
|
}
|
|
5802
|
+
if (priorRun.status === "queued" && isRetiredNativeAtlassianTask(acceptedExecution.task)) {
|
|
5803
|
+
await markScheduledTaskRunFailedIfQueued(
|
|
5804
|
+
db,
|
|
5805
|
+
priorRun.workspaceId,
|
|
5806
|
+
priorRun.id,
|
|
5807
|
+
ATLASSIAN_NATIVE_RETIRED_REASON
|
|
5808
|
+
);
|
|
5809
|
+
return scheduledRunTerminalResult(ATLASSIAN_NATIVE_RETIRED_REASON);
|
|
5810
|
+
}
|
|
4721
5811
|
await recordScheduledTaskFiredUsage(db, acceptedExecution, priorRun);
|
|
4722
5812
|
if (priorRun.status === "dispatched") {
|
|
4723
5813
|
return await replayScheduledTaskDispatch({
|
|
@@ -4737,6 +5827,7 @@ function createScheduledTaskActivities(services) {
|
|
|
4737
5827
|
try {
|
|
4738
5828
|
return await recoverBoundScheduledTaskDispatch({
|
|
4739
5829
|
db,
|
|
5830
|
+
observability: baseService.observability,
|
|
4740
5831
|
bus,
|
|
4741
5832
|
settings: settings2,
|
|
4742
5833
|
wakeSessionWorkflow,
|
|
@@ -4778,6 +5869,8 @@ function createScheduledTaskActivities(services) {
|
|
|
4778
5869
|
if (!task) {
|
|
4779
5870
|
return { action: "deleted" };
|
|
4780
5871
|
}
|
|
5872
|
+
if (isRetiredNativeAtlassianTask(task))
|
|
5873
|
+
return { action: "blocked", reason: ATLASSIAN_NATIVE_RETIRED_REASON };
|
|
4781
5874
|
if (task.action.kind !== "agent_turn") {
|
|
4782
5875
|
return { action: "blocked", reason: "legacy_source_schedule_requires_migration" };
|
|
4783
5876
|
}
|
|
@@ -4805,20 +5898,6 @@ function createScheduledTaskActivities(services) {
|
|
|
4805
5898
|
if (incidentDeclaration.action === "blocked") {
|
|
4806
5899
|
return incidentDeclaration;
|
|
4807
5900
|
}
|
|
4808
|
-
const taskPersonalConnectionDelegations = await getScheduledTaskPersonalConnectionDelegations(
|
|
4809
|
-
db,
|
|
4810
|
-
task.workspaceId,
|
|
4811
|
-
task.id
|
|
4812
|
-
);
|
|
4813
|
-
const taskConnectionAuthoritySubjects = [
|
|
4814
|
-
...new Set(
|
|
4815
|
-
taskPersonalConnectionDelegations.map((delegation) => delegation.ownerSubjectId)
|
|
4816
|
-
)
|
|
4817
|
-
];
|
|
4818
|
-
if (taskConnectionAuthoritySubjects.length > 1) {
|
|
4819
|
-
throw new Error("scheduled connection authority has multiple causal humans");
|
|
4820
|
-
}
|
|
4821
|
-
const taskConnectionAuthoritySubjectId = taskConnectionAuthoritySubjects[0] ?? null;
|
|
4822
5901
|
const taskPersonalResourceAuthoritySubjectId = await getScheduledTaskPersonalResourceAuthoritySubject(db, {
|
|
4823
5902
|
accountId: task.accountId,
|
|
4824
5903
|
workspaceId: task.workspaceId,
|
|
@@ -4831,10 +5910,35 @@ function createScheduledTaskActivities(services) {
|
|
|
4831
5910
|
taskId: task.id,
|
|
4832
5911
|
taskAuthorityRevision: task.authorityRevision
|
|
4833
5912
|
});
|
|
4834
|
-
|
|
4835
|
-
|
|
5913
|
+
const taskAuthoritySubjectId = task.ownerSubjectId;
|
|
5914
|
+
const refuseAdmission = async (reason, retryable, detail) => {
|
|
5915
|
+
console.warn(
|
|
5916
|
+
`[scheduled-task] refused occurrence of ${task.id} (revision ${task.authorityRevision}): ${reason}: ${detail}`
|
|
5917
|
+
);
|
|
5918
|
+
const receipt = await recordScheduledTaskAdmissionRefusal(db, {
|
|
5919
|
+
workspaceId: task.workspaceId,
|
|
5920
|
+
taskId: task.id,
|
|
5921
|
+
taskAuthorityRevision: task.authorityRevision,
|
|
5922
|
+
taskExecutionDigest: task.executionDigest,
|
|
5923
|
+
triggerType: input.triggerType,
|
|
5924
|
+
producerKey: stableProducerKey,
|
|
5925
|
+
reason,
|
|
5926
|
+
retryable
|
|
5927
|
+
});
|
|
5928
|
+
if (!receipt.admissionRefusal) {
|
|
5929
|
+
return createScheduledTaskActivities(services).dispatchScheduledTaskRun(input);
|
|
5930
|
+
}
|
|
5931
|
+
return scheduledAdmissionRefusalResult(receipt);
|
|
5932
|
+
};
|
|
5933
|
+
if (task.action.kind === "agent_turn" && ((taskRevisionAuthority?.subjectId ?? null) !== taskAuthoritySubjectId || [taskPersonalResourceAuthoritySubjectId].some(
|
|
5934
|
+
(subject) => subject !== null && subject !== taskAuthoritySubjectId
|
|
5935
|
+
))) {
|
|
5936
|
+
return await refuseAdmission(
|
|
5937
|
+
"scheduled_authority_unavailable",
|
|
5938
|
+
false,
|
|
5939
|
+
"scheduled authority differs from its immutable execution owner"
|
|
5940
|
+
);
|
|
4836
5941
|
}
|
|
4837
|
-
const taskAuthoritySubjectId = taskRevisionAuthority?.subjectId ?? taskPersonalResourceAuthoritySubjectId ?? taskConnectionAuthoritySubjectId;
|
|
4838
5942
|
const acceptedTargetSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.runMode === "reusable_session" ? task.reusableSessionId : null;
|
|
4839
5943
|
const targetSessionExecutionBase = acceptedTargetSessionId ? await getScheduledTargetSessionExecution(
|
|
4840
5944
|
db,
|
|
@@ -4843,18 +5947,113 @@ function createScheduledTaskActivities(services) {
|
|
|
4843
5947
|
taskAuthoritySubjectId
|
|
4844
5948
|
) : null;
|
|
4845
5949
|
const settings = await settingsForTask(task, targetSessionExecutionBase?.model);
|
|
4846
|
-
const
|
|
4847
|
-
const
|
|
5950
|
+
const resolvedDefault = task.agentConfig.model || targetSessionExecutionBase ? null : await resolveScheduledTaskDefaultModel(db, settings, task);
|
|
5951
|
+
const model = task.agentConfig.model ?? resolvedDefault?.model ?? settings.openaiModel;
|
|
5952
|
+
const reasoningEffort = task.agentConfig.reasoningEffort ?? resolvedDefault?.reasoningEffort ?? settings.openaiReasoningEffort;
|
|
4848
5953
|
let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
4849
5954
|
let sandboxOs = "linux";
|
|
4850
|
-
|
|
5955
|
+
let taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
4851
5956
|
const creatorPolicy = await getScheduledTaskCreatorPolicy(db, task.workspaceId, task.id);
|
|
4852
|
-
|
|
5957
|
+
let firstPartyMcpTools = withScheduledSlackBotPostingTools(
|
|
5958
|
+
creatorPolicy?.firstPartyMcpTools ? allowedFirstPartyMcpToolsForSession(settings, creatorPolicy.firstPartyMcpTools) : resolveFirstPartyMcpToolPolicy(settings).default,
|
|
5959
|
+
task.agentConfig,
|
|
5960
|
+
resolveFirstPartyMcpToolPolicy(settings).allowed
|
|
5961
|
+
);
|
|
5962
|
+
let resolvedAgentConfig = null;
|
|
5963
|
+
let resolvedAgentInstructions;
|
|
5964
|
+
const generatedTarget = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
|
|
5965
|
+
if (generatedTarget) {
|
|
5966
|
+
try {
|
|
5967
|
+
const agentResolution = resolveSessionAgentConfigForCreate2({
|
|
5968
|
+
settings,
|
|
5969
|
+
creator: "scheduled",
|
|
5970
|
+
request: task.agentConfig.agent,
|
|
5971
|
+
instructions: void 0,
|
|
5972
|
+
workspaceSettings: (await requireWorkspace2(db, task.workspaceId)).settings,
|
|
5973
|
+
parent: null,
|
|
5974
|
+
goal: task.agentConfig.goal !== void 0
|
|
5975
|
+
});
|
|
5976
|
+
resolvedAgentConfig = agentResolution.config;
|
|
5977
|
+
resolvedAgentInstructions = agentResolution.instructions;
|
|
5978
|
+
const written = applySessionAgentConfigWriteThrough2({
|
|
5979
|
+
config: resolvedAgentConfig,
|
|
5980
|
+
firstPartyMcpTools,
|
|
5981
|
+
tools: taskTools,
|
|
5982
|
+
toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
|
|
5983
|
+
productServerIds: task.agentConfig.tools.map((tool) => tool.id)
|
|
5984
|
+
});
|
|
5985
|
+
firstPartyMcpTools = written.firstPartyMcpTools;
|
|
5986
|
+
taskTools = written.tools;
|
|
5987
|
+
} catch (error) {
|
|
5988
|
+
const status = error instanceof Error ? error.status : void 0;
|
|
5989
|
+
if (!(error instanceof Error) || status !== 422) throw error;
|
|
5990
|
+
return await refuseAdmission(
|
|
5991
|
+
"scheduled_authority_unavailable",
|
|
5992
|
+
false,
|
|
5993
|
+
`scheduled agent configuration is not admissible: ${error.message}`
|
|
5994
|
+
);
|
|
5995
|
+
}
|
|
5996
|
+
}
|
|
4853
5997
|
const firstPartyMcpPermissions = creatorPolicy?.firstPartyMcpPermissions ? [...creatorPolicy.firstPartyMcpPermissions] : [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS];
|
|
5998
|
+
const connectionSettings = await settingsWithEnabledCapabilityMcpServers(
|
|
5999
|
+
db,
|
|
6000
|
+
task.workspaceId,
|
|
6001
|
+
settings,
|
|
6002
|
+
taskAuthoritySubjectId ? { subjectId: taskAuthoritySubjectId } : {}
|
|
6003
|
+
);
|
|
6004
|
+
const connectionTarget = acceptedTargetSessionId && targetSessionExecutionBase ? await requireSession(db, task.workspaceId, acceptedTargetSessionId) : null;
|
|
6005
|
+
const connectionTools = await scheduledConnectionTools(
|
|
6006
|
+
db,
|
|
6007
|
+
task.workspaceId,
|
|
6008
|
+
connectionSettings,
|
|
6009
|
+
connectionTarget,
|
|
6010
|
+
taskTools,
|
|
6011
|
+
taskAuthoritySubjectId ?? void 0
|
|
6012
|
+
);
|
|
6013
|
+
const taskConnections = await freezeConnectionAccounts({
|
|
6014
|
+
db,
|
|
6015
|
+
accountId: task.accountId,
|
|
6016
|
+
workspaceId: task.workspaceId,
|
|
6017
|
+
settings: connectionTarget ? settingsWithSessionMcpServerMetadata(connectionSettings, connectionTarget.mcpServers) : connectionSettings,
|
|
6018
|
+
tools: connectionTools,
|
|
6019
|
+
resources: mergeResourceRefs(connectionTarget?.resources ?? [], task.agentConfig.resources),
|
|
6020
|
+
source: taskAuthoritySubjectId ? { kind: "subject", subjectId: taskAuthoritySubjectId, accountId: task.accountId } : { kind: "none" },
|
|
6021
|
+
authoritySelections: task.agentConfig.connectionAccounts ?? [],
|
|
6022
|
+
authoritySelectionsFrozen: task.agentConfig.connectionAccountsFrozen === true,
|
|
6023
|
+
...scheduledConnectionSurfaceEligibility(
|
|
6024
|
+
settings,
|
|
6025
|
+
connectionTarget ?? {
|
|
6026
|
+
firstPartyMcpTools,
|
|
6027
|
+
firstPartyMcpPermissions
|
|
6028
|
+
}
|
|
6029
|
+
)
|
|
6030
|
+
}).catch((error) => {
|
|
6031
|
+
if (error instanceof ConnectionAccountSelectionError) return error;
|
|
6032
|
+
throw error;
|
|
6033
|
+
});
|
|
6034
|
+
if (taskConnections instanceof ConnectionAccountSelectionError) {
|
|
6035
|
+
const receipt = await recordScheduledTaskAdmissionFailure(db, {
|
|
6036
|
+
workspaceId: task.workspaceId,
|
|
6037
|
+
taskId: task.id,
|
|
6038
|
+
taskAuthorityRevision: task.authorityRevision,
|
|
6039
|
+
taskExecutionDigest: task.executionDigest,
|
|
6040
|
+
triggerType: input.triggerType,
|
|
6041
|
+
producerKey: stableProducerKey,
|
|
6042
|
+
diagnostic: taskConnections.diagnostic
|
|
6043
|
+
});
|
|
6044
|
+
if (!receipt.admissionDiagnostic) {
|
|
6045
|
+
return createScheduledTaskActivities(services).dispatchScheduledTaskRun(input);
|
|
6046
|
+
}
|
|
6047
|
+
return scheduledAdmissionRefusalResult(receipt);
|
|
6048
|
+
}
|
|
6049
|
+
const {
|
|
6050
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
6051
|
+
mcpAccountBindings: taskMcpAccountBindings
|
|
6052
|
+
} = taskConnections;
|
|
6053
|
+
const taskConnectionAuthoritySubjectId = taskPersonalConnectionDelegations.length > 0 ? taskAuthoritySubjectId : null;
|
|
4854
6054
|
const creatorSessionPolicy = scheduledCreatorSessionPolicyInput(
|
|
4855
6055
|
creatorPolicy?.sessionPolicy ?? null
|
|
4856
6056
|
);
|
|
4857
|
-
const generatedTarget = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
|
|
4858
6057
|
if (generatedTarget && task.agentConfig.machineTarget) {
|
|
4859
6058
|
const access = {
|
|
4860
6059
|
accountId: task.accountId,
|
|
@@ -4867,19 +6066,38 @@ function createScheduledTaskActivities(services) {
|
|
|
4867
6066
|
task.agentConfig.machineTarget.targetSandboxId
|
|
4868
6067
|
);
|
|
4869
6068
|
if (!machine || machine.kind !== "selfhosted" || !machine.enrollmentId || machine.scope === "user") {
|
|
4870
|
-
|
|
6069
|
+
return await refuseAdmission(
|
|
6070
|
+
"machine_target_unavailable",
|
|
6071
|
+
false,
|
|
6072
|
+
"scheduled Connected Machine target is unavailable"
|
|
6073
|
+
);
|
|
4871
6074
|
}
|
|
4872
6075
|
const enrollment = await getEnrollment(db, access, machine.enrollmentId);
|
|
4873
|
-
if (!enrollment
|
|
4874
|
-
|
|
6076
|
+
if (!enrollment) {
|
|
6077
|
+
return await refuseAdmission(
|
|
6078
|
+
"machine_target_unavailable",
|
|
6079
|
+
false,
|
|
6080
|
+
"scheduled Connected Machine enrollment is gone"
|
|
6081
|
+
);
|
|
6082
|
+
}
|
|
6083
|
+
if (enrollment.status !== "active") {
|
|
6084
|
+
return await refuseAdmission(
|
|
6085
|
+
"machine_enrollment_inactive",
|
|
6086
|
+
true,
|
|
6087
|
+
`scheduled Connected Machine enrollment is ${enrollment.status}`
|
|
6088
|
+
);
|
|
4875
6089
|
}
|
|
4876
6090
|
sandboxBackend = "selfhosted";
|
|
4877
6091
|
sandboxOs = enrollment.os;
|
|
4878
6092
|
} else if (generatedTarget && sandboxBackend === "selfhosted") {
|
|
4879
|
-
|
|
6093
|
+
return await refuseAdmission(
|
|
6094
|
+
"machine_target_unavailable",
|
|
6095
|
+
false,
|
|
6096
|
+
"self-hosted scheduled task has no Connected Machine target"
|
|
6097
|
+
);
|
|
4880
6098
|
}
|
|
4881
6099
|
const generatedSessionDepthPolicy = generatedTarget ? await (async () => {
|
|
4882
|
-
const workspace = await
|
|
6100
|
+
const workspace = await requireWorkspace2(db, task.workspaceId);
|
|
4883
6101
|
const deployment = await getNestedAgentDepthDeploymentPolicy(db);
|
|
4884
6102
|
const taskOverride = task.agentConfig.maxNestedAgentDepth ?? null;
|
|
4885
6103
|
const workspaceOverride = typeof workspace.settings.maxNestedAgentDepth === "number" ? workspace.settings.maxNestedAgentDepth : null;
|
|
@@ -4898,20 +6116,15 @@ function createScheduledTaskActivities(services) {
|
|
|
4898
6116
|
};
|
|
4899
6117
|
})() : null;
|
|
4900
6118
|
const targetSessionExecution = targetSessionExecutionBase ? await (async () => {
|
|
4901
|
-
const
|
|
4902
|
-
|
|
4903
|
-
|
|
4904
|
-
|
|
4905
|
-
task.workspaceId,
|
|
4906
|
-
taskPersonalConnectionDelegations
|
|
4907
|
-
)
|
|
4908
|
-
]);
|
|
4909
|
-
const workspaceDefaultMcpServerIds = new Set(
|
|
4910
|
-
settings.mcpServers.map((server) => server.id)
|
|
6119
|
+
const apiIntegrationServerIds = await listInstalledApiIntegrationServerIdsForDelegations(
|
|
6120
|
+
db,
|
|
6121
|
+
task.workspaceId,
|
|
6122
|
+
taskPersonalConnectionDelegations
|
|
4911
6123
|
);
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
|
|
6124
|
+
const availableMcpServerIds = /* @__PURE__ */ new Set([
|
|
6125
|
+
...connectionSettings.mcpServers.map((server) => server.id),
|
|
6126
|
+
...apiIntegrationServerIds
|
|
6127
|
+
]);
|
|
4915
6128
|
for (const id of targetSessionExecutionBase.mcpServerIds) {
|
|
4916
6129
|
availableMcpServerIds.add(id);
|
|
4917
6130
|
}
|
|
@@ -4919,9 +6132,10 @@ function createScheduledTaskActivities(services) {
|
|
|
4919
6132
|
toolPolicy: targetSessionExecutionBase.toolPolicy,
|
|
4920
6133
|
sessionTools: targetSessionExecutionBase.tools,
|
|
4921
6134
|
availableMcpServerIds,
|
|
4922
|
-
defaultMcpServerIds:
|
|
4923
|
-
|
|
4924
|
-
|
|
6135
|
+
defaultMcpServerIds: [
|
|
6136
|
+
...connectionTools.map((tool) => tool.id),
|
|
6137
|
+
...apiIntegrationServerIds
|
|
6138
|
+
]
|
|
4925
6139
|
});
|
|
4926
6140
|
return {
|
|
4927
6141
|
...targetSessionExecutionBase,
|
|
@@ -4946,7 +6160,11 @@ function createScheduledTaskActivities(services) {
|
|
|
4946
6160
|
task.variableSetId
|
|
4947
6161
|
) : null;
|
|
4948
6162
|
if (generatedTarget && task.variableSetId && !acceptedVariableSet) {
|
|
4949
|
-
|
|
6163
|
+
return await refuseAdmission(
|
|
6164
|
+
"variable_set_unavailable",
|
|
6165
|
+
false,
|
|
6166
|
+
`variable set not found: ${task.variableSetId}`
|
|
6167
|
+
);
|
|
4950
6168
|
}
|
|
4951
6169
|
const acceptedRig = generatedTarget && task.rigId ? await getRig2(
|
|
4952
6170
|
db,
|
|
@@ -4958,11 +6176,16 @@ function createScheduledTaskActivities(services) {
|
|
|
4958
6176
|
task.rigId
|
|
4959
6177
|
) : null;
|
|
4960
6178
|
if (generatedTarget && task.rigId && (!acceptedRig || !acceptedRig.activeVersion)) {
|
|
4961
|
-
|
|
6179
|
+
return await refuseAdmission(
|
|
6180
|
+
"rig_version_unavailable",
|
|
6181
|
+
false,
|
|
6182
|
+
`sandbox environment has no active version to bind: ${task.rigId}`
|
|
6183
|
+
);
|
|
4962
6184
|
}
|
|
4963
|
-
const
|
|
4964
|
-
acceptedRig.activeVersion.defaultVariableSetIds.map(async (variableSetId) => {
|
|
4965
|
-
|
|
6185
|
+
const acceptedRigDefaultVariableSetRows = acceptedRig?.activeVersion ? await Promise.all(
|
|
6186
|
+
acceptedRig.activeVersion.defaultVariableSetIds.map(async (variableSetId) => ({
|
|
6187
|
+
variableSetId,
|
|
6188
|
+
variableSet: await getVariableSet(
|
|
4966
6189
|
db,
|
|
4967
6190
|
{
|
|
4968
6191
|
accountId: task.accountId,
|
|
@@ -4970,13 +6193,22 @@ function createScheduledTaskActivities(services) {
|
|
|
4970
6193
|
subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
|
|
4971
6194
|
},
|
|
4972
6195
|
variableSetId
|
|
4973
|
-
)
|
|
4974
|
-
|
|
4975
|
-
throw new Error(`rig default Variable Set not found: ${variableSetId}`);
|
|
4976
|
-
}
|
|
4977
|
-
return { id: variableSet.id, generation: variableSet.generation };
|
|
4978
|
-
})
|
|
6196
|
+
)
|
|
6197
|
+
}))
|
|
4979
6198
|
) : [];
|
|
6199
|
+
const missingRigDefaultVariableSet = acceptedRigDefaultVariableSetRows.find(
|
|
6200
|
+
(row) => !row.variableSet
|
|
6201
|
+
);
|
|
6202
|
+
if (missingRigDefaultVariableSet) {
|
|
6203
|
+
return await refuseAdmission(
|
|
6204
|
+
"variable_set_unavailable",
|
|
6205
|
+
false,
|
|
6206
|
+
`sandbox environment default Variable Set not found: ${missingRigDefaultVariableSet.variableSetId}`
|
|
6207
|
+
);
|
|
6208
|
+
}
|
|
6209
|
+
const acceptedRigDefaultVariableSets = acceptedRigDefaultVariableSetRows.map(
|
|
6210
|
+
({ variableSet }) => ({ id: variableSet.id, generation: variableSet.generation })
|
|
6211
|
+
);
|
|
4980
6212
|
let incidentPreflightRequired = incidentDeclaration.action === "required";
|
|
4981
6213
|
if (incidentPreflightRequired) {
|
|
4982
6214
|
const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
@@ -5031,6 +6263,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5031
6263
|
incidentPreflightRequired = preflight.action === "ready";
|
|
5032
6264
|
}
|
|
5033
6265
|
const taskXaiProviderAccountAuthoritySnapshot = await getScheduledTaskXaiProviderAccountAuthoritySnapshot(db, task.workspaceId, task.id);
|
|
6266
|
+
const taskClaudeProviderAccountAuthoritySnapshot = await getScheduledTaskClaudeProviderAccountAuthoritySnapshot(db, task.workspaceId, task.id);
|
|
5034
6267
|
const slackBotConnection = task.agentConfig.slackBotConnectionId ? await requireOpenGeniSlackBotConnection(
|
|
5035
6268
|
db,
|
|
5036
6269
|
task.workspaceId,
|
|
@@ -5038,23 +6271,58 @@ function createScheduledTaskActivities(services) {
|
|
|
5038
6271
|
) : null;
|
|
5039
6272
|
const slackBotMetadata = slackBotConnection ? openGeniSlackBotMetadata(slackBotConnection.metadata) : null;
|
|
5040
6273
|
if (slackBotConnection && !slackBotMetadata) {
|
|
5041
|
-
throw new Error("
|
|
6274
|
+
throw new Error("Opengeni Slack bot connection metadata is invalid");
|
|
5042
6275
|
}
|
|
5043
6276
|
const xaiAuthoritySubjectId = taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? task.createdBy.subjectId : null;
|
|
5044
6277
|
if (taskXaiProviderAccountAuthoritySnapshot.scope === "user" && !xaiAuthoritySubjectId) {
|
|
5045
|
-
|
|
6278
|
+
return await refuseAdmission(
|
|
6279
|
+
"scheduled_authority_unavailable",
|
|
6280
|
+
false,
|
|
6281
|
+
"scheduled user-scoped xAI authority has no causal human"
|
|
6282
|
+
);
|
|
5046
6283
|
}
|
|
5047
6284
|
if (xaiAuthoritySubjectId && taskAuthoritySubjectId && xaiAuthoritySubjectId !== taskAuthoritySubjectId) {
|
|
5048
|
-
|
|
6285
|
+
return await refuseAdmission(
|
|
6286
|
+
"scheduled_authority_unavailable",
|
|
6287
|
+
false,
|
|
6288
|
+
"scheduled authority classes have different causal humans"
|
|
6289
|
+
);
|
|
5049
6290
|
}
|
|
5050
|
-
const
|
|
6291
|
+
const claudeAuthoritySubjectId = taskClaudeProviderAccountAuthoritySnapshot.scope === "user" && task.ownerSubjectId ? task.ownerSubjectId : null;
|
|
6292
|
+
if (taskClaudeProviderAccountAuthoritySnapshot.scope === "user" && !claudeAuthoritySubjectId) {
|
|
6293
|
+
return await refuseAdmission(
|
|
6294
|
+
"scheduled_authority_unavailable",
|
|
6295
|
+
false,
|
|
6296
|
+
"scheduled user-scoped Claude authority has no causal human"
|
|
6297
|
+
);
|
|
6298
|
+
}
|
|
6299
|
+
if (claudeAuthoritySubjectId && taskAuthoritySubjectId && claudeAuthoritySubjectId !== taskAuthoritySubjectId) {
|
|
6300
|
+
return await refuseAdmission(
|
|
6301
|
+
"scheduled_authority_unavailable",
|
|
6302
|
+
false,
|
|
6303
|
+
"scheduled authority classes have different causal humans"
|
|
6304
|
+
);
|
|
6305
|
+
}
|
|
6306
|
+
const causalHumanSubjectId = taskAuthoritySubjectId ?? xaiAuthoritySubjectId ?? claudeAuthoritySubjectId;
|
|
6307
|
+
if (xaiAuthoritySubjectId && claudeAuthoritySubjectId && xaiAuthoritySubjectId !== claudeAuthoritySubjectId)
|
|
6308
|
+
return await refuseAdmission(
|
|
6309
|
+
"scheduled_authority_unavailable",
|
|
6310
|
+
false,
|
|
6311
|
+
"scheduled subscription pools have different causal humans"
|
|
6312
|
+
);
|
|
5051
6313
|
if (task.agentConfig.knowledgeSource && causalHumanSubjectId !== task.agentConfig.knowledgeSource.initiatingSubjectId) {
|
|
5052
|
-
|
|
6314
|
+
return await refuseAdmission(
|
|
6315
|
+
"scheduled_authority_unavailable",
|
|
6316
|
+
false,
|
|
5053
6317
|
"Source schedule requires its connection owner's current human authorization; update the source schedule to authorize it"
|
|
5054
6318
|
);
|
|
5055
6319
|
}
|
|
5056
6320
|
if ((acceptedVariableSet?.scope === "user" || acceptedRig?.scope === "user") && !causalHumanSubjectId) {
|
|
5057
|
-
|
|
6321
|
+
return await refuseAdmission(
|
|
6322
|
+
"scheduled_authority_unavailable",
|
|
6323
|
+
false,
|
|
6324
|
+
"scheduled personal-resource execution has no causal human"
|
|
6325
|
+
);
|
|
5058
6326
|
}
|
|
5059
6327
|
const admissionDenial = await agentRunAdmissionDenial(
|
|
5060
6328
|
{ ...baseService, settings },
|
|
@@ -5062,11 +6330,12 @@ function createScheduledTaskActivities(services) {
|
|
|
5062
6330
|
accountId: task.accountId,
|
|
5063
6331
|
workspaceId: task.workspaceId,
|
|
5064
6332
|
model: targetSessionExecution?.model ?? model,
|
|
5065
|
-
requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1
|
|
6333
|
+
requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1,
|
|
6334
|
+
initiatingHumanSubjectId: causalHumanSubjectId
|
|
5066
6335
|
}
|
|
5067
6336
|
);
|
|
5068
6337
|
if (admissionDenial) {
|
|
5069
|
-
return
|
|
6338
|
+
return await refuseAdmission(admissionDenial, true, "agent run admission denied");
|
|
5070
6339
|
}
|
|
5071
6340
|
const acceptedModel = targetSessionExecution?.model ?? model;
|
|
5072
6341
|
const acceptedReasoningEffort = targetSessionExecution?.reasoningEffort ?? reasoningEffort;
|
|
@@ -5085,15 +6354,19 @@ function createScheduledTaskActivities(services) {
|
|
|
5085
6354
|
);
|
|
5086
6355
|
}
|
|
5087
6356
|
} : void 0;
|
|
5088
|
-
const turnExecutionPolicy =
|
|
5089
|
-
|
|
5090
|
-
|
|
5091
|
-
|
|
5092
|
-
|
|
5093
|
-
|
|
5094
|
-
|
|
5095
|
-
|
|
5096
|
-
|
|
6357
|
+
const turnExecutionPolicy = scheduledTaskRunExecutionPolicy(
|
|
6358
|
+
resolveTurnExecutionPolicyV12(settings, {
|
|
6359
|
+
modelId: acceptedModel,
|
|
6360
|
+
requestedModelId: generatedTarget && task.agentConfig.model ? task.agentConfig.model : null,
|
|
6361
|
+
modelSource: generatedTarget ? task.agentConfig.model ? "explicit" : "deployment" : "session",
|
|
6362
|
+
reasoningEffort: acceptedReasoningEffort,
|
|
6363
|
+
reasoningSource: generatedTarget ? task.agentConfig.reasoningEffort ? "explicit" : "deployment" : "session",
|
|
6364
|
+
latencyMode: acceptedLatencyMode,
|
|
6365
|
+
latencyModeSource: generatedTarget ? "deployment" : "session"
|
|
6366
|
+
}),
|
|
6367
|
+
input,
|
|
6368
|
+
creatorPolicy?.credentialRestriction
|
|
6369
|
+
);
|
|
5097
6370
|
const deferredEvents = [];
|
|
5098
6371
|
const executeDispatch = async (dispatchDb, deferPublications) => {
|
|
5099
6372
|
const admittedRunId = crypto.randomUUID();
|
|
@@ -5115,6 +6388,10 @@ function createScheduledTaskActivities(services) {
|
|
|
5115
6388
|
resolvedTools: taskTools,
|
|
5116
6389
|
resolvedFirstPartyMcpTools: firstPartyMcpTools,
|
|
5117
6390
|
resolvedFirstPartyMcpPermissions: firstPartyMcpPermissions,
|
|
6391
|
+
...resolvedAgentConfig ? {
|
|
6392
|
+
resolvedAgentConfig,
|
|
6393
|
+
...resolvedAgentInstructions !== void 0 ? { resolvedAgentInstructions } : {}
|
|
6394
|
+
} : {},
|
|
5118
6395
|
resolvedVariableSet: acceptedVariableSet ? { id: acceptedVariableSet.id, generation: acceptedVariableSet.generation } : null,
|
|
5119
6396
|
resolvedRig: acceptedRig?.activeVersion ? {
|
|
5120
6397
|
id: acceptedRig.id,
|
|
@@ -5130,11 +6407,14 @@ function createScheduledTaskActivities(services) {
|
|
|
5130
6407
|
targetSessionExecution,
|
|
5131
6408
|
generatedSessionBinding,
|
|
5132
6409
|
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
6410
|
+
mcpAccountBindings: taskMcpAccountBindings,
|
|
5133
6411
|
personalResourceAuthoritySubjectId: taskPersonalResourceAuthoritySubjectId,
|
|
5134
6412
|
causalHumanSubjectId,
|
|
5135
6413
|
causalHumanAuthority: taskRevisionAuthority,
|
|
5136
6414
|
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
6415
|
+
claudeProviderAccountAuthoritySnapshot: taskClaudeProviderAccountAuthoritySnapshot,
|
|
5137
6416
|
xaiAuthoritySubjectId,
|
|
6417
|
+
claudeAuthoritySubjectId,
|
|
5138
6418
|
connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId,
|
|
5139
6419
|
triggerInitiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
|
|
5140
6420
|
agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey ?? null,
|
|
@@ -5243,13 +6523,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5243
6523
|
taskId: task.id
|
|
5244
6524
|
}
|
|
5245
6525
|
);
|
|
5246
|
-
if (stableJson({
|
|
5247
|
-
task: exactAuthority.task,
|
|
5248
|
-
personalConnectionDelegations: exactAuthority.personalConnectionDelegations
|
|
5249
|
-
}) !== stableJson({
|
|
5250
|
-
task,
|
|
5251
|
-
personalConnectionDelegations: taskPersonalConnectionDelegations
|
|
5252
|
-
})) {
|
|
6526
|
+
if (stableJson(exactAuthority) !== stableJson(task)) {
|
|
5253
6527
|
throw new IncidentTelemetryPreflightBlockedError(
|
|
5254
6528
|
"incident_preflight_metadata_missing"
|
|
5255
6529
|
);
|
|
@@ -5270,14 +6544,16 @@ function createScheduledTaskActivities(services) {
|
|
|
5270
6544
|
if (task.rigId) {
|
|
5271
6545
|
const rig = acceptedRig;
|
|
5272
6546
|
if (!rig || !rig.activeVersion) {
|
|
5273
|
-
throw new Error(`
|
|
6547
|
+
throw new Error(`sandbox environment has no active version to bind: ${task.rigId}`);
|
|
5274
6548
|
}
|
|
5275
6549
|
frozenRigId = rig.id;
|
|
5276
6550
|
frozenRigVersionId = runPersonalResourceAuthority?.resources.find(
|
|
5277
6551
|
(resource) => resource.resourceKind === "rig" && resource.resourceId === rig.id
|
|
5278
6552
|
)?.resourceVersionId ?? rig.activeVersion.id;
|
|
5279
6553
|
if (frozenRigVersionId !== rig.activeVersion.id) {
|
|
5280
|
-
throw new Error(
|
|
6554
|
+
throw new Error(
|
|
6555
|
+
"scheduled run sandbox environment version changed during admission"
|
|
6556
|
+
);
|
|
5281
6557
|
}
|
|
5282
6558
|
}
|
|
5283
6559
|
let session;
|
|
@@ -5294,6 +6570,8 @@ function createScheduledTaskActivities(services) {
|
|
|
5294
6570
|
tools: taskTools,
|
|
5295
6571
|
firstPartyMcpTools,
|
|
5296
6572
|
firstPartyMcpPermissions,
|
|
6573
|
+
...resolvedAgentConfig ? { agentConfig: resolvedAgentConfig } : {},
|
|
6574
|
+
...resolvedAgentInstructions !== void 0 ? { instructions: resolvedAgentInstructions } : {},
|
|
5297
6575
|
...creatorSessionPolicy,
|
|
5298
6576
|
...task.agentConfig.knowledgeSource?.destination.kind === "personal" ? {
|
|
5299
6577
|
visibility: "user_private",
|
|
@@ -5301,6 +6579,10 @@ function createScheduledTaskActivities(services) {
|
|
|
5301
6579
|
} : {},
|
|
5302
6580
|
metadata: {
|
|
5303
6581
|
...taskMetadata,
|
|
6582
|
+
...scheduledSessionExecutionPolicyMetadata(
|
|
6583
|
+
turnExecutionPolicy,
|
|
6584
|
+
creatorPolicy?.credentialRestriction
|
|
6585
|
+
),
|
|
5304
6586
|
model,
|
|
5305
6587
|
reasoningEffort,
|
|
5306
6588
|
scheduledTaskId: task.id,
|
|
@@ -5330,6 +6612,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5330
6612
|
rigVersionId: frozenRigVersionId,
|
|
5331
6613
|
personalConnectionDelegations: [],
|
|
5332
6614
|
initialXaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
6615
|
+
initialClaudeProviderAccountAuthoritySnapshot: taskClaudeProviderAccountAuthoritySnapshot,
|
|
5333
6616
|
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
5334
6617
|
frozenNestedAgentDepthPolicy: {
|
|
5335
6618
|
effectiveMaxNestedAgentDepth: generatedSessionBinding.effectiveMaxNestedAgentDepth,
|
|
@@ -5390,6 +6673,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5390
6673
|
}
|
|
5391
6674
|
session = created.session;
|
|
5392
6675
|
sessionCreated = created.created;
|
|
6676
|
+
if (sessionCreated) recordScheduledSessionCreated(baseService.observability);
|
|
5393
6677
|
if (!sessionCreated) {
|
|
5394
6678
|
await bindScheduledTaskRunSessionInTransaction(dispatchDb, {
|
|
5395
6679
|
accountId: task.accountId,
|
|
@@ -5420,7 +6704,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5420
6704
|
}
|
|
5421
6705
|
if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
|
|
5422
6706
|
throw new Error(
|
|
5423
|
-
"scheduled alert occurrence
|
|
6707
|
+
"scheduled alert occurrence Opengeni Slack bot binding does not match its canonical session"
|
|
5424
6708
|
);
|
|
5425
6709
|
}
|
|
5426
6710
|
}
|
|
@@ -5476,7 +6760,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5476
6760
|
events: started.events
|
|
5477
6761
|
});
|
|
5478
6762
|
} else {
|
|
5479
|
-
await
|
|
6763
|
+
await publishDurableSessionEvents2(
|
|
5480
6764
|
bus,
|
|
5481
6765
|
task.workspaceId,
|
|
5482
6766
|
session.id,
|
|
@@ -5519,7 +6803,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5519
6803
|
if (deferPublications) {
|
|
5520
6804
|
deferredEvents.push({ sessionId: session.id, events: started.events });
|
|
5521
6805
|
} else {
|
|
5522
|
-
await
|
|
6806
|
+
await publishDurableSessionEvents2(
|
|
5523
6807
|
bus,
|
|
5524
6808
|
task.workspaceId,
|
|
5525
6809
|
session.id,
|
|
@@ -5548,7 +6832,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5548
6832
|
if (deferPublications) {
|
|
5549
6833
|
deferredEvents.push({ sessionId: session.id, events: titleEvents });
|
|
5550
6834
|
} else {
|
|
5551
|
-
await
|
|
6835
|
+
await publishDurableSessionEvents2(bus, task.workspaceId, session.id, titleEvents);
|
|
5552
6836
|
}
|
|
5553
6837
|
}
|
|
5554
6838
|
const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
|
|
@@ -5574,10 +6858,13 @@ function createScheduledTaskActivities(services) {
|
|
|
5574
6858
|
scheduledTaskRunId: run.id,
|
|
5575
6859
|
...causalHumanSubjectId ? { causalHumanSubjectId } : {},
|
|
5576
6860
|
...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
|
|
5577
|
-
...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
|
|
6861
|
+
...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {},
|
|
6862
|
+
...claudeAuthoritySubjectId ? { claudeAuthoritySubjectId } : {}
|
|
5578
6863
|
},
|
|
5579
6864
|
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
6865
|
+
mcpAccountBindings: taskMcpAccountBindings,
|
|
5580
6866
|
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
6867
|
+
claudeProviderAccountAuthoritySnapshot: taskClaudeProviderAccountAuthoritySnapshot,
|
|
5581
6868
|
scheduledTaskRunId: run.id
|
|
5582
6869
|
},
|
|
5583
6870
|
async (tx, wakeEventId, _updateId, rejectionReason) => {
|
|
@@ -5624,6 +6911,9 @@ function createScheduledTaskActivities(services) {
|
|
|
5624
6911
|
run: { ...run, status: "skipped", error }
|
|
5625
6912
|
};
|
|
5626
6913
|
}
|
|
6914
|
+
if (scheduledUpdate.added) {
|
|
6915
|
+
recordUserMessageAccepted(baseService.observability, { surface: "scheduled" });
|
|
6916
|
+
}
|
|
5627
6917
|
if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
|
|
5628
6918
|
if (deferPublications) {
|
|
5629
6919
|
deferredEvents.push({
|
|
@@ -5631,7 +6921,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5631
6921
|
events: scheduledUpdate.events
|
|
5632
6922
|
});
|
|
5633
6923
|
} else {
|
|
5634
|
-
await
|
|
6924
|
+
await publishDurableSessionEvents2(
|
|
5635
6925
|
bus,
|
|
5636
6926
|
task.workspaceId,
|
|
5637
6927
|
session.id,
|
|
@@ -5683,10 +6973,13 @@ function createScheduledTaskActivities(services) {
|
|
|
5683
6973
|
scheduledTaskRunId: run.id,
|
|
5684
6974
|
...causalHumanSubjectId ? { causalHumanSubjectId } : {},
|
|
5685
6975
|
...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
|
|
5686
|
-
...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
|
|
6976
|
+
...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {},
|
|
6977
|
+
...claudeAuthoritySubjectId ? { claudeAuthoritySubjectId } : {}
|
|
5687
6978
|
},
|
|
5688
6979
|
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
6980
|
+
mcpAccountBindings: taskMcpAccountBindings,
|
|
5689
6981
|
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
6982
|
+
claudeProviderAccountAuthoritySnapshot: taskClaudeProviderAccountAuthoritySnapshot,
|
|
5690
6983
|
scheduledTaskRunId: run.id
|
|
5691
6984
|
},
|
|
5692
6985
|
async (tx, wakeEventId, _updateId, rejectionReason) => {
|
|
@@ -5747,6 +7040,9 @@ function createScheduledTaskActivities(services) {
|
|
|
5747
7040
|
run: { ...run, status: "skipped", error }
|
|
5748
7041
|
};
|
|
5749
7042
|
}
|
|
7043
|
+
if (bundled.added) {
|
|
7044
|
+
recordUserMessageAccepted(baseService.observability, { surface: "scheduled" });
|
|
7045
|
+
}
|
|
5750
7046
|
if (bundled.added && bundled.events.length > 0) {
|
|
5751
7047
|
if (deferPublications) {
|
|
5752
7048
|
deferredEvents.push({
|
|
@@ -5754,7 +7050,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5754
7050
|
events: bundled.events
|
|
5755
7051
|
});
|
|
5756
7052
|
} else {
|
|
5757
|
-
await
|
|
7053
|
+
await publishDurableSessionEvents2(
|
|
5758
7054
|
bus,
|
|
5759
7055
|
task.workspaceId,
|
|
5760
7056
|
session.id,
|
|
@@ -5836,7 +7132,7 @@ function createScheduledTaskActivities(services) {
|
|
|
5836
7132
|
});
|
|
5837
7133
|
}
|
|
5838
7134
|
for (const deferred of deferredEvents) {
|
|
5839
|
-
await
|
|
7135
|
+
await publishDurableSessionEvents2(
|
|
5840
7136
|
bus,
|
|
5841
7137
|
task.workspaceId,
|
|
5842
7138
|
deferred.sessionId,
|
|
@@ -5866,16 +7162,11 @@ var IncidentTelemetryPreflightBlockedError = class extends Error {
|
|
|
5866
7162
|
}
|
|
5867
7163
|
};
|
|
5868
7164
|
function assertReusableSessionBindingMatches(session, task) {
|
|
5869
|
-
if (
|
|
5870
|
-
throw new ScheduledRunTerminalAuthorityError(
|
|
5871
|
-
"scheduled_reusable_binding_changed",
|
|
5872
|
-
"scheduled task variableSet attachment does not match its reusable session"
|
|
5873
|
-
);
|
|
5874
|
-
}
|
|
7165
|
+
if (task.runMode === "existing_session") return;
|
|
5875
7166
|
if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
|
|
5876
7167
|
throw new ScheduledRunTerminalAuthorityError(
|
|
5877
7168
|
"scheduled_reusable_binding_changed",
|
|
5878
|
-
"scheduled task
|
|
7169
|
+
"scheduled task Opengeni Slack bot binding does not match its reusable session"
|
|
5879
7170
|
);
|
|
5880
7171
|
}
|
|
5881
7172
|
}
|
|
@@ -5921,7 +7212,25 @@ function scheduledRunRecoveryTerminalError(error) {
|
|
|
5921
7212
|
}
|
|
5922
7213
|
return null;
|
|
5923
7214
|
}
|
|
7215
|
+
function scheduledAdmissionRefusalResult(run) {
|
|
7216
|
+
if (run.admissionRefusal) {
|
|
7217
|
+
return {
|
|
7218
|
+
action: "blocked",
|
|
7219
|
+
reason: run.admissionRefusal.reason,
|
|
7220
|
+
runId: run.id,
|
|
7221
|
+
refusal: run.admissionRefusal
|
|
7222
|
+
};
|
|
7223
|
+
}
|
|
7224
|
+
return {
|
|
7225
|
+
action: "blocked",
|
|
7226
|
+
reason: "connection_account_unavailable",
|
|
7227
|
+
runId: run.id,
|
|
7228
|
+
...run.admissionDiagnostic ? { diagnostic: run.admissionDiagnostic } : {}
|
|
7229
|
+
};
|
|
7230
|
+
}
|
|
5924
7231
|
function scheduledRunTerminalResult(error) {
|
|
7232
|
+
if (error === ATLASSIAN_NATIVE_RETIRED_REASON)
|
|
7233
|
+
return { action: "blocked", reason: ATLASSIAN_NATIVE_RETIRED_REASON };
|
|
5925
7234
|
if (error === "scheduled_authority_exhausted") {
|
|
5926
7235
|
return { action: "blocked", reason: "scheduled_authority_exhausted" };
|
|
5927
7236
|
}
|
|
@@ -5941,46 +7250,38 @@ function scheduledSessionRejectionError(rejectionReason) {
|
|
|
5941
7250
|
}
|
|
5942
7251
|
async function prepareIncidentTelemetrySource(input) {
|
|
5943
7252
|
const session = await requireSession(input.tx, input.workspaceId, input.sessionId);
|
|
5944
|
-
const { task, personalConnectionDelegations } = input.acceptedExecution
|
|
5945
|
-
task: input.acceptedExecution.task,
|
|
5946
|
-
personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations
|
|
5947
|
-
} : await requireScheduledTaskIncidentAuthorityInTransaction(input.tx, {
|
|
5948
|
-
workspaceId: input.workspaceId,
|
|
5949
|
-
taskId: input.taskId
|
|
5950
|
-
});
|
|
7253
|
+
const { task, personalConnectionDelegations } = input.acceptedExecution;
|
|
5951
7254
|
const responder = await resolveIncidentTelemetryResponderMetadata({
|
|
5952
7255
|
db: input.tx,
|
|
5953
7256
|
settings: input.settings,
|
|
5954
7257
|
task,
|
|
5955
7258
|
session,
|
|
5956
7259
|
personalConnectionDelegations,
|
|
5957
|
-
personalResourceAuthoritySubjectId: input.acceptedExecution
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
5964
|
-
|
|
5965
|
-
|
|
5966
|
-
|
|
5967
|
-
|
|
5968
|
-
|
|
5969
|
-
|
|
5970
|
-
|
|
5971
|
-
|
|
5972
|
-
|
|
5973
|
-
|
|
5974
|
-
|
|
5975
|
-
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
|
|
5979
|
-
|
|
5980
|
-
|
|
5981
|
-
|
|
5982
|
-
}
|
|
5983
|
-
} : {}
|
|
7260
|
+
personalResourceAuthoritySubjectId: input.acceptedExecution.personalResourceAuthoritySubjectId ?? null,
|
|
7261
|
+
executionPolicy: input.acceptedExecution.targetSessionExecution ? {
|
|
7262
|
+
tools: input.acceptedExecution.targetSessionExecution.tools,
|
|
7263
|
+
firstPartyMcpTools: input.acceptedExecution.targetSessionExecution.firstPartyMcpTools,
|
|
7264
|
+
firstPartyMcpPermissions: input.acceptedExecution.targetSessionExecution.firstPartyMcpPermissions,
|
|
7265
|
+
variableSetIds: input.acceptedExecution.targetSessionExecution.variableSets.map(
|
|
7266
|
+
(variableSet) => variableSet.id
|
|
7267
|
+
),
|
|
7268
|
+
variableSetId: input.acceptedExecution.targetSessionExecution.variableSetId,
|
|
7269
|
+
rigId: input.acceptedExecution.targetSessionExecution.rigId,
|
|
7270
|
+
rigVersionId: input.acceptedExecution.targetSessionExecution.rigVersionId,
|
|
7271
|
+
toolPolicy: input.acceptedExecution.targetSessionExecution.toolPolicy,
|
|
7272
|
+
mcpServerIds: input.acceptedExecution.targetSessionExecution.mcpServerIds,
|
|
7273
|
+
toolPolicyVersion: input.acceptedExecution.targetSessionExecution.toolPolicyVersion
|
|
7274
|
+
} : {
|
|
7275
|
+
tools: input.acceptedExecution.resolvedTools,
|
|
7276
|
+
firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
|
|
7277
|
+
firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
|
|
7278
|
+
variableSetId: input.acceptedExecution.resolvedVariableSet?.id ?? null,
|
|
7279
|
+
rigId: input.acceptedExecution.resolvedRig?.id ?? null,
|
|
7280
|
+
rigVersionId: input.acceptedExecution.resolvedRig?.versionId ?? null,
|
|
7281
|
+
toolPolicy: session.toolPolicy,
|
|
7282
|
+
mcpServerIds: [],
|
|
7283
|
+
toolPolicyVersion: session.toolPolicyVersion
|
|
7284
|
+
}
|
|
5984
7285
|
});
|
|
5985
7286
|
const preflight = evaluateIncidentTelemetryPreflight({
|
|
5986
7287
|
agentConfig: task.agentConfig,
|
|
@@ -6060,7 +7361,7 @@ async function seedScheduledGeneratedSessionRoute(input) {
|
|
|
6060
7361
|
if (input.deferPublications) {
|
|
6061
7362
|
input.deferredEvents.push({ sessionId: input.session.id, events: failed.events });
|
|
6062
7363
|
} else {
|
|
6063
|
-
await
|
|
7364
|
+
await publishDurableSessionEvents2(
|
|
6064
7365
|
input.bus,
|
|
6065
7366
|
input.task.workspaceId,
|
|
6066
7367
|
input.session.id,
|
|
@@ -6080,6 +7381,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6080
7381
|
throw new Error("scheduled run is not an exact queued recovery");
|
|
6081
7382
|
}
|
|
6082
7383
|
const generatedSession = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
|
|
7384
|
+
const recoveredCreatorPolicy = generatedSession ? await getScheduledTaskCreatorPolicy(input.db, task.workspaceId, task.id) : null;
|
|
6083
7385
|
const frozenSlack = input.acceptedExecution.resolvedSlackBotConnection;
|
|
6084
7386
|
if (frozenSlack) {
|
|
6085
7387
|
const currentSlack = await requireOpenGeniSlackBotConnection(
|
|
@@ -6139,11 +7441,6 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6139
7441
|
}
|
|
6140
7442
|
const taskMetadata = { ...task.agentConfig.metadata };
|
|
6141
7443
|
delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
6142
|
-
const recoveredCreatorPolicy = await getScheduledTaskCreatorPolicy(
|
|
6143
|
-
input.db,
|
|
6144
|
-
task.workspaceId,
|
|
6145
|
-
task.id
|
|
6146
|
-
);
|
|
6147
7444
|
const created = await createSessionWithIdempotencyKeyResult(input.db, {
|
|
6148
7445
|
accountId: task.accountId,
|
|
6149
7446
|
workspaceId: task.workspaceId,
|
|
@@ -6153,6 +7450,8 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6153
7450
|
tools: input.acceptedExecution.resolvedTools,
|
|
6154
7451
|
firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
|
|
6155
7452
|
firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
|
|
7453
|
+
...input.acceptedExecution.resolvedAgentConfig ? { agentConfig: input.acceptedExecution.resolvedAgentConfig } : {},
|
|
7454
|
+
...input.acceptedExecution.resolvedAgentInstructions !== void 0 ? { instructions: input.acceptedExecution.resolvedAgentInstructions } : {},
|
|
6156
7455
|
...scheduledCreatorSessionPolicyInput(recoveredCreatorPolicy?.sessionPolicy ?? null),
|
|
6157
7456
|
...task.agentConfig.knowledgeSource?.destination.kind === "personal" ? {
|
|
6158
7457
|
visibility: "user_private",
|
|
@@ -6160,6 +7459,10 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6160
7459
|
} : {},
|
|
6161
7460
|
metadata: {
|
|
6162
7461
|
...taskMetadata,
|
|
7462
|
+
...scheduledSessionExecutionPolicyMetadata(
|
|
7463
|
+
input.acceptedExecution.turnExecutionPolicy,
|
|
7464
|
+
recoveredCreatorPolicy?.credentialRestriction
|
|
7465
|
+
),
|
|
6163
7466
|
model: input.acceptedExecution.resolvedModel,
|
|
6164
7467
|
reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
|
|
6165
7468
|
scheduledTaskId: task.id,
|
|
@@ -6187,6 +7490,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6187
7490
|
rigVersionId: rig?.versionId ?? null,
|
|
6188
7491
|
personalConnectionDelegations: [],
|
|
6189
7492
|
initialXaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
|
|
7493
|
+
initialClaudeProviderAccountAuthoritySnapshot: input.acceptedExecution.claudeProviderAccountAuthoritySnapshot,
|
|
6190
7494
|
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
6191
7495
|
frozenNestedAgentDepthPolicy: {
|
|
6192
7496
|
effectiveMaxNestedAgentDepth: input.acceptedExecution.generatedSessionBinding.effectiveMaxNestedAgentDepth,
|
|
@@ -6212,6 +7516,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6212
7516
|
});
|
|
6213
7517
|
if (created.denied) throw new SessionSpawnDeniedDbError(created.denial);
|
|
6214
7518
|
session = created.session;
|
|
7519
|
+
if (created.created) recordScheduledSessionCreated(input.observability);
|
|
6215
7520
|
if (!created.created) {
|
|
6216
7521
|
await bindScheduledTaskRunSessionInTransaction(input.db, {
|
|
6217
7522
|
accountId: task.accountId,
|
|
@@ -6275,21 +7580,28 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6275
7580
|
);
|
|
6276
7581
|
const expectedTaskMetadata = { ...task.agentConfig.metadata };
|
|
6277
7582
|
delete expectedTaskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
6278
|
-
const expectedMetadata =
|
|
6279
|
-
|
|
6280
|
-
|
|
6281
|
-
|
|
6282
|
-
|
|
6283
|
-
|
|
6284
|
-
|
|
6285
|
-
|
|
6286
|
-
|
|
6287
|
-
|
|
6288
|
-
|
|
7583
|
+
const expectedMetadata = metadataWithAgentConfigCreateIdentity(
|
|
7584
|
+
{
|
|
7585
|
+
...expectedTaskMetadata,
|
|
7586
|
+
...scheduledSessionExecutionPolicyMetadata(
|
|
7587
|
+
input.acceptedExecution.turnExecutionPolicy,
|
|
7588
|
+
recoveredCreatorPolicy?.credentialRestriction
|
|
7589
|
+
),
|
|
7590
|
+
model: input.acceptedExecution.resolvedModel,
|
|
7591
|
+
reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
|
|
7592
|
+
scheduledTaskId: task.id,
|
|
7593
|
+
scheduledTaskRunMode: task.runMode,
|
|
7594
|
+
...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
|
|
7595
|
+
scheduledTaskRunId: canonicalGeneratedRunId,
|
|
7596
|
+
...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
|
|
7597
|
+
},
|
|
7598
|
+
input.acceptedExecution.resolvedAgentConfig
|
|
7599
|
+
);
|
|
7600
|
+
if (session.createIdempotencyKey !== expectedCreateKey || session.initialMessage !== task.agentConfig.prompt || session.instructions !== (input.acceptedExecution.resolvedAgentInstructions ?? null) || session.policyRole !== null || stableJson(session.skills) !== "[]" || stableJson(session.toolPolicy) !== stableJson({ mode: "explicit", inheritedFromSessionId: null }) || session.toolPolicyVersion !== 1 || session.mcpServers.length !== 0 || session.parentSessionId !== null || session.rootSessionId !== session.id || session.nestedAgentDepth !== 0 || session.effectiveMaxNestedAgentDepth !== generatedBinding.effectiveMaxNestedAgentDepth || session.nestedAgentDepthPolicySource !== generatedBinding.nestedAgentDepthPolicySource || session.nestedAgentDepthPolicySessionId !== (generatedBinding.nestedAgentDepthPolicySource === "session" ? session.id : null) || session.sandboxGroupId !== session.id || session.channelId !== null || session.codexCompactionMode !== generatedBinding.codexCompactionMode || stableJson(session.metadata) !== stableJson(expectedMetadata) || session.createdBy.kind !== "service" || session.createdBy.subjectId !== "scheduler" || stableJson(session.createdByContext) !== stableJson({
|
|
6289
7601
|
label: "OpenGeni scheduler",
|
|
6290
7602
|
scheduledTaskId: task.id,
|
|
6291
7603
|
scheduledTaskRunId: canonicalGeneratedRunId
|
|
6292
|
-
}) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || creationPolicy?.model !== input.acceptedExecution.resolvedModel || creationPolicy.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || creationPolicy.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || session.activeSandboxId !== (task.agentConfig.machineTarget?.targetSandboxId ?? null) || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
|
|
7604
|
+
}) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || creationPolicy?.model !== input.acceptedExecution.resolvedModel || creationPolicy.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || creationPolicy.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || session.activeSandboxId !== (task.agentConfig.machineTarget?.targetSandboxId ?? null) || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || stableJson(session.agent) !== stableJson(input.acceptedExecution.resolvedAgentConfig ?? null) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
|
|
6293
7605
|
throw new ScheduledRunTerminalAuthorityError(
|
|
6294
7606
|
"scheduled_generated_session_changed",
|
|
6295
7607
|
"scheduled generated session differs from accepted execution"
|
|
@@ -6320,7 +7632,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6320
7632
|
});
|
|
6321
7633
|
workflowId = started.temporalWorkflowId;
|
|
6322
7634
|
if (started.events.length > 0) {
|
|
6323
|
-
await
|
|
7635
|
+
await publishDurableSessionEvents2(input.bus, task.workspaceId, session.id, started.events);
|
|
6324
7636
|
}
|
|
6325
7637
|
if (task.runMode === "reusable_session") {
|
|
6326
7638
|
await materializeScheduledTaskReusableSessionFromRun(input.db, {
|
|
@@ -6339,7 +7651,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6339
7651
|
taskName: task.name
|
|
6340
7652
|
});
|
|
6341
7653
|
if (titleEvents.length > 0) {
|
|
6342
|
-
await
|
|
7654
|
+
await publishDurableSessionEvents2(input.bus, task.workspaceId, session.id, titleEvents);
|
|
6343
7655
|
}
|
|
6344
7656
|
}
|
|
6345
7657
|
const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
|
|
@@ -6367,10 +7679,13 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6367
7679
|
...input.acceptedExecution.connectionAuthoritySubjectId ? {
|
|
6368
7680
|
connectionAuthoritySubjectId: input.acceptedExecution.connectionAuthoritySubjectId
|
|
6369
7681
|
} : {},
|
|
6370
|
-
...input.acceptedExecution.xaiAuthoritySubjectId ? { xaiAuthoritySubjectId: input.acceptedExecution.xaiAuthoritySubjectId } : {}
|
|
7682
|
+
...input.acceptedExecution.xaiAuthoritySubjectId ? { xaiAuthoritySubjectId: input.acceptedExecution.xaiAuthoritySubjectId } : {},
|
|
7683
|
+
...input.acceptedExecution.claudeAuthoritySubjectId ? { claudeAuthoritySubjectId: input.acceptedExecution.claudeAuthoritySubjectId } : {}
|
|
6371
7684
|
},
|
|
6372
7685
|
personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations,
|
|
7686
|
+
mcpAccountBindings: input.acceptedExecution.mcpAccountBindings ?? null,
|
|
6373
7687
|
xaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
|
|
7688
|
+
claudeProviderAccountAuthoritySnapshot: input.acceptedExecution.claudeProviderAccountAuthoritySnapshot,
|
|
6374
7689
|
scheduledTaskRunId: input.run.id
|
|
6375
7690
|
},
|
|
6376
7691
|
async (tx, wakeEventId, _updateId, rejectionReason) => {
|
|
@@ -6427,8 +7742,11 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
6427
7742
|
if (scheduledUpdate.reason === "session_cancelled" || scheduledUpdate.reason === "session_not_idle") {
|
|
6428
7743
|
return { action: "blocked", reason: "scheduled_run_terminal" };
|
|
6429
7744
|
}
|
|
7745
|
+
if (scheduledUpdate.added) {
|
|
7746
|
+
recordUserMessageAccepted(input.observability, { surface: "scheduled" });
|
|
7747
|
+
}
|
|
6430
7748
|
if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
|
|
6431
|
-
await
|
|
7749
|
+
await publishDurableSessionEvents2(
|
|
6432
7750
|
input.bus,
|
|
6433
7751
|
task.workspaceId,
|
|
6434
7752
|
session.id,
|
|
@@ -6680,6 +7998,9 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
|
|
|
6680
7998
|
accountId: maintenance.accountId,
|
|
6681
7999
|
workspaceId: maintenance.workspaceId,
|
|
6682
8000
|
initialMessage: maintenancePrompt(maintenance),
|
|
8001
|
+
// OpenGeni's own periodic check: no human or product surface entered it.
|
|
8002
|
+
surface: "system",
|
|
8003
|
+
metrics: service.observability,
|
|
6683
8004
|
resources: [],
|
|
6684
8005
|
tools: [{ kind: "mcp", id: "opengeni" }],
|
|
6685
8006
|
toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
|
|
@@ -6766,43 +8087,55 @@ import {
|
|
|
6766
8087
|
applySessionTurnSettlement,
|
|
6767
8088
|
enqueueSessionWorkflowWake,
|
|
6768
8089
|
failSessionWorkBeforeAttemptClaim,
|
|
8090
|
+
blockSessionWorkBeforeAttemptClaim,
|
|
6769
8091
|
requestSessionTurnRecovery,
|
|
6770
8092
|
recoverSessionDispatch,
|
|
8093
|
+
reconcileSettledSessionAttempt,
|
|
6771
8094
|
reconcileSessionAttemptQuiescence,
|
|
8095
|
+
reconcileCompletedSandboxSetup,
|
|
6772
8096
|
peekSessionWork as peekSessionWorkDb,
|
|
6773
8097
|
settleSessionInputWait as settleSessionInputWaitDb,
|
|
6774
8098
|
countQueuedTurns as countQueuedTurns2,
|
|
6775
8099
|
getSessionAttemptActivityRef,
|
|
6776
8100
|
getSessionEvent,
|
|
6777
8101
|
getSessionTurnForAttempt,
|
|
8102
|
+
getWorkspace,
|
|
6778
8103
|
expireSessionInteractionIntervention as expireSessionInteractionInterventionDb,
|
|
8104
|
+
expireScheduledRunHumanWait as expireScheduledRunHumanWaitDb,
|
|
6779
8105
|
expireSessionHumanInputRequest,
|
|
6780
8106
|
markSessionAttemptQuiesced,
|
|
6781
8107
|
requireSession as requireSession2,
|
|
6782
|
-
settleSessionIdleWithParentOutbox
|
|
8108
|
+
settleSessionIdleWithParentOutbox,
|
|
8109
|
+
SANDBOX_SETUP_RECOVERY_LIMIT
|
|
6783
8110
|
} from "@opengeni/db";
|
|
6784
|
-
import { publishDurableSessionEvents as
|
|
8111
|
+
import { publishDurableSessionEvents as publishDurableSessionEvents3 } from "@opengeni/events";
|
|
8112
|
+
import { CancelledFailure } from "@temporalio/activity";
|
|
6785
8113
|
var WORKER_DEATH_MAX_REDISPATCHES = 3;
|
|
6786
8114
|
function createSessionStateActivities(services, overrides = {}) {
|
|
6787
8115
|
const settleSessionAttemptInterruptionsFn = overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
|
|
6788
8116
|
const applySessionTurnSettlementFn = overrides.applySessionTurnSettlement ?? applySessionTurnSettlement;
|
|
6789
8117
|
const enqueueSessionWorkflowWakeFn = overrides.enqueueSessionWorkflowWake ?? enqueueSessionWorkflowWake;
|
|
6790
8118
|
const failSessionWorkBeforeAttemptClaimFn = overrides.failSessionWorkBeforeAttemptClaim ?? failSessionWorkBeforeAttemptClaim;
|
|
8119
|
+
const blockSessionWorkBeforeAttemptClaimFn = overrides.blockSessionWorkBeforeAttemptClaim ?? blockSessionWorkBeforeAttemptClaim;
|
|
6791
8120
|
const requestSessionTurnRecoveryFn = overrides.requestSessionTurnRecovery ?? requestSessionTurnRecovery;
|
|
6792
8121
|
const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
|
|
8122
|
+
const reconcileSettledSessionAttemptFn = overrides.reconcileSettledSessionAttempt ?? reconcileSettledSessionAttempt;
|
|
6793
8123
|
const reconcileSessionAttemptQuiescenceFn = overrides.reconcileSessionAttemptQuiescence ?? reconcileSessionAttemptQuiescence;
|
|
6794
8124
|
const peekSessionWorkFn = overrides.peekSessionWork ?? peekSessionWorkDb;
|
|
8125
|
+
const reconcileCompletedSandboxSetupFn = overrides.reconcileCompletedSandboxSetup ?? reconcileCompletedSandboxSetup;
|
|
6795
8126
|
const settleSessionInputWaitFn = overrides.settleSessionInputWait ?? settleSessionInputWaitDb;
|
|
6796
8127
|
const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns2;
|
|
6797
8128
|
const getSessionAttemptActivityRefFn = overrides.getSessionAttemptActivityRef ?? getSessionAttemptActivityRef;
|
|
6798
8129
|
const getSessionEventFn = overrides.getSessionEvent ?? getSessionEvent;
|
|
6799
8130
|
const getSessionTurnForAttemptFn = overrides.getSessionTurnForAttempt ?? getSessionTurnForAttempt;
|
|
8131
|
+
const getWorkspaceFn = overrides.getWorkspace ?? getWorkspace;
|
|
6800
8132
|
const expireSessionHumanInputRequestFn = overrides.expireSessionHumanInputRequest ?? expireSessionHumanInputRequest;
|
|
6801
8133
|
const expireSessionInteractionInterventionFn = overrides.expireSessionInteractionIntervention ?? expireSessionInteractionInterventionDb;
|
|
8134
|
+
const expireScheduledRunHumanWaitFn = overrides.expireScheduledRunHumanWait ?? expireScheduledRunHumanWaitDb;
|
|
6802
8135
|
const requireSessionFn = overrides.requireSession ?? requireSession2;
|
|
6803
8136
|
const settleSessionIdleWithParentOutboxFn = overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
|
|
6804
8137
|
const markSessionAttemptQuiescedFn = overrides.markSessionAttemptQuiesced ?? markSessionAttemptQuiesced;
|
|
6805
|
-
const publishDurableSessionEventsFn = overrides.publishDurableSessionEvents ??
|
|
8138
|
+
const publishDurableSessionEventsFn = overrides.publishDurableSessionEvents ?? publishDurableSessionEvents3;
|
|
6806
8139
|
const deliverFailedChildTurnToParentFn = overrides.deliverFailedChildTurnToParent ?? deliverFailedChildTurnToParent;
|
|
6807
8140
|
const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
|
|
6808
8141
|
const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
|
|
@@ -6830,6 +8163,24 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
6830
8163
|
});
|
|
6831
8164
|
if (attempt) return { action: "stale" };
|
|
6832
8165
|
const preClaimFailureDisposition = input.preClaimFailure?.disposition ?? input.preClaimFailureDisposition;
|
|
8166
|
+
if (preClaimFailureDisposition === "blocked" && input.admissionFence && input.preClaimFailure?.reason) {
|
|
8167
|
+
const blocked = await blockSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
|
|
8168
|
+
accountId: input.accountId,
|
|
8169
|
+
sessionId: input.sessionId,
|
|
8170
|
+
workflowId,
|
|
8171
|
+
attemptId: input.attemptId,
|
|
8172
|
+
fence: input.admissionFence,
|
|
8173
|
+
reason: input.preClaimFailure.reason,
|
|
8174
|
+
sqlState: input.preClaimFailure.sqlState ?? null
|
|
8175
|
+
});
|
|
8176
|
+
await publishDurableSessionEventsFn(
|
|
8177
|
+
bus,
|
|
8178
|
+
input.workspaceId,
|
|
8179
|
+
input.sessionId,
|
|
8180
|
+
blocked.events
|
|
8181
|
+
);
|
|
8182
|
+
return { action: blocked.action };
|
|
8183
|
+
}
|
|
6833
8184
|
if (preClaimFailureDisposition === "permanent" && input.trigger) {
|
|
6834
8185
|
const failed = await failSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
|
|
6835
8186
|
accountId: input.accountId,
|
|
@@ -6837,7 +8188,12 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
6837
8188
|
workflowId,
|
|
6838
8189
|
trigger: input.trigger,
|
|
6839
8190
|
error: input.error ?? "Agent turn admission failed before attempt claim.",
|
|
6840
|
-
...input.preClaimFailure
|
|
8191
|
+
...input.preClaimFailure?.disposition === "permanent" ? {
|
|
8192
|
+
admissionFailure: {
|
|
8193
|
+
disposition: "permanent",
|
|
8194
|
+
code: input.preClaimFailure.code
|
|
8195
|
+
}
|
|
8196
|
+
} : {}
|
|
6841
8197
|
});
|
|
6842
8198
|
if (failed.action === "terminal") return { action: "terminal" };
|
|
6843
8199
|
if (failed.action === "stale") return { action: "stale" };
|
|
@@ -6868,10 +8224,18 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
6868
8224
|
const postClaimIdentityMatches = Boolean(
|
|
6869
8225
|
postClaimRecovery && turn.id === postClaimRecovery.turnId && turn.triggerEventId === postClaimRecovery.triggerEventId && turn.executionGeneration === postClaimRecovery.executionGeneration
|
|
6870
8226
|
);
|
|
8227
|
+
if (postClaimRecovery?.sandboxSetupRecoveryExhausted && !postClaimIdentityMatches) {
|
|
8228
|
+
return { action: "stale" };
|
|
8229
|
+
}
|
|
6871
8230
|
const providerRecoveryCount = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryCount : void 0;
|
|
6872
8231
|
const providerFailureCode = postClaimIdentityMatches ? postClaimRecovery?.providerFailureCode : void 0;
|
|
6873
8232
|
const hasProviderRecoveryCount = providerRecoveryCount !== void 0;
|
|
6874
8233
|
const hasProviderFailureCode = providerFailureCode !== void 0;
|
|
8234
|
+
const setupOutcomeUnknown = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupOutcomeUnknown === true;
|
|
8235
|
+
const setupRecoveryExhausted = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupRecoveryExhausted === true;
|
|
8236
|
+
if (setupOutcomeUnknown && setupRecoveryExhausted || (setupOutcomeUnknown || setupRecoveryExhausted) && (hasProviderRecoveryCount || hasProviderFailureCode) || setupRecoveryExhausted && turn.metadata?.providerRecoveryCount !== SANDBOX_SETUP_RECOVERY_LIMIT) {
|
|
8237
|
+
return { action: "stale" };
|
|
8238
|
+
}
|
|
6875
8239
|
if (hasProviderRecoveryCount !== hasProviderFailureCode) {
|
|
6876
8240
|
return { action: "stale" };
|
|
6877
8241
|
}
|
|
@@ -6885,11 +8249,20 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
6885
8249
|
turnId: turn.id,
|
|
6886
8250
|
triggerEventId: turn.triggerEventId,
|
|
6887
8251
|
attemptId: input.attemptId,
|
|
6888
|
-
reason: providerFailureCode ?? "claimed_attempt_database_failure",
|
|
8252
|
+
reason: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? "claimed_attempt_database_failure",
|
|
8253
|
+
...setupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {},
|
|
8254
|
+
...setupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
|
|
6889
8255
|
...providerRecoveryCount !== void 0 ? { providerRecoveryCount } : {},
|
|
6890
8256
|
detail: {
|
|
6891
|
-
code: providerFailureCode ?? recoveredClaimCode,
|
|
6892
|
-
retryable:
|
|
8257
|
+
code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? recoveredClaimCode,
|
|
8258
|
+
retryable: !setupOutcomeUnknown && !setupRecoveryExhausted,
|
|
8259
|
+
...setupOutcomeUnknown ? { setupOutcome: "unknown", replay: "blocked" } : {},
|
|
8260
|
+
...setupRecoveryExhausted ? {
|
|
8261
|
+
setupOutcome: "not_started",
|
|
8262
|
+
replay: "blocked",
|
|
8263
|
+
recoveryExhausted: true,
|
|
8264
|
+
providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT
|
|
8265
|
+
} : {},
|
|
6893
8266
|
...providerRecoveryCount !== void 0 ? {
|
|
6894
8267
|
databaseFailureCode: recoveredClaimCode,
|
|
6895
8268
|
providerRecoveryCount
|
|
@@ -7019,6 +8392,54 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7019
8392
|
}
|
|
7020
8393
|
return { action: result.action };
|
|
7021
8394
|
}
|
|
8395
|
+
async function reconcileSettledSessionAttemptActivity(input) {
|
|
8396
|
+
const { db, bus, inspectSessionAttemptActivity, settings, observability, wakeSessionWorkflow } = await services();
|
|
8397
|
+
const ref = await getSessionAttemptActivityRefFn(db, {
|
|
8398
|
+
...input,
|
|
8399
|
+
temporalWorkflowId: input.workflowId
|
|
8400
|
+
});
|
|
8401
|
+
if (!ref || ref.workflowRunId !== input.workflowRunId || ref.activityId !== input.activityId)
|
|
8402
|
+
return { action: "stale" };
|
|
8403
|
+
if (!inspectSessionAttemptActivity) return { action: "pending" };
|
|
8404
|
+
try {
|
|
8405
|
+
if (await inspectSessionAttemptActivity(ref) !== "settled") return { action: "pending" };
|
|
8406
|
+
} catch (error) {
|
|
8407
|
+
if (error instanceof CancelledFailure) throw error;
|
|
8408
|
+
if (currentActivityContext()?.cancellationSignal.aborted)
|
|
8409
|
+
throw new CancelledFailure("Settled-owner reconciliation cancelled");
|
|
8410
|
+
return { action: "pending" };
|
|
8411
|
+
}
|
|
8412
|
+
if (currentActivityContext()?.cancellationSignal.aborted)
|
|
8413
|
+
throw new CancelledFailure("Settled-owner reconciliation cancelled");
|
|
8414
|
+
const result = await reconcileSettledSessionAttemptFn(db, {
|
|
8415
|
+
accountId: input.accountId,
|
|
8416
|
+
workspaceId: input.workspaceId,
|
|
8417
|
+
sessionId: input.sessionId,
|
|
8418
|
+
turnId: input.turnId,
|
|
8419
|
+
attemptId: input.attemptId,
|
|
8420
|
+
executionGeneration: input.executionGeneration,
|
|
8421
|
+
temporalWorkflowId: ref.workflowId,
|
|
8422
|
+
temporalWorkflowRunId: ref.workflowRunId,
|
|
8423
|
+
temporalActivityId: ref.activityId,
|
|
8424
|
+
activitySettled: true,
|
|
8425
|
+
maxRedispatches: WORKER_DEATH_MAX_REDISPATCHES
|
|
8426
|
+
});
|
|
8427
|
+
if (result.events.length > 0)
|
|
8428
|
+
await publishDurableSessionEventsFn(
|
|
8429
|
+
bus,
|
|
8430
|
+
input.workspaceId,
|
|
8431
|
+
input.sessionId,
|
|
8432
|
+
result.events
|
|
8433
|
+
).catch(() => void 0);
|
|
8434
|
+
if (result.action === "exceeded")
|
|
8435
|
+
await deliverFailedChildTurnToParentFn(
|
|
8436
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
8437
|
+
input.workspaceId,
|
|
8438
|
+
input.sessionId,
|
|
8439
|
+
result.turnId
|
|
8440
|
+
);
|
|
8441
|
+
return { action: result.action };
|
|
8442
|
+
}
|
|
7022
8443
|
async function recoverDispatch(input) {
|
|
7023
8444
|
const { settings, db, bus, observability, wakeSessionWorkflow } = await services();
|
|
7024
8445
|
const result = await recoverSessionDispatchFn(db, input.workspaceId, {
|
|
@@ -7091,8 +8512,68 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7091
8512
|
return { action: "recovering" };
|
|
7092
8513
|
}
|
|
7093
8514
|
async function peekSessionWork(input) {
|
|
7094
|
-
const { db, observability } = await services();
|
|
7095
|
-
const
|
|
8515
|
+
const { db, bus, observability, inspectSessionAttemptActivity } = await services();
|
|
8516
|
+
const observerAccountId = input.observerAccountId ?? (await getWorkspaceFn(db, input.workspaceId))?.accountId;
|
|
8517
|
+
if (!observerAccountId) return { kind: "unavailable" };
|
|
8518
|
+
let peek = await peekSessionWorkFn(
|
|
8519
|
+
db,
|
|
8520
|
+
input.workspaceId,
|
|
8521
|
+
input.sessionId,
|
|
8522
|
+
input.includeAdmissionFence,
|
|
8523
|
+
observerAccountId
|
|
8524
|
+
);
|
|
8525
|
+
if (peek.kind === "unavailable") return peek;
|
|
8526
|
+
if (peek.kind === "admission-blocked" && peek.reason === "sandbox_setup_outcome_unknown") {
|
|
8527
|
+
const ref = peek.ref;
|
|
8528
|
+
if (ref && "turnId" in ref && "attemptId" in ref) {
|
|
8529
|
+
const settled = await reconcileCompletedSandboxSetupFn(db, {
|
|
8530
|
+
accountId: observerAccountId,
|
|
8531
|
+
workspaceId: input.workspaceId,
|
|
8532
|
+
sessionId: input.sessionId,
|
|
8533
|
+
turnId: ref.turnId,
|
|
8534
|
+
attemptId: ref.attemptId
|
|
8535
|
+
});
|
|
8536
|
+
if (settled.events.length > 0)
|
|
8537
|
+
await publishDurableSessionEventsFn(
|
|
8538
|
+
bus,
|
|
8539
|
+
input.workspaceId,
|
|
8540
|
+
input.sessionId,
|
|
8541
|
+
settled.events
|
|
8542
|
+
);
|
|
8543
|
+
if (settled.reconciled)
|
|
8544
|
+
peek = await peekSessionWorkFn(
|
|
8545
|
+
db,
|
|
8546
|
+
input.workspaceId,
|
|
8547
|
+
input.sessionId,
|
|
8548
|
+
input.includeAdmissionFence,
|
|
8549
|
+
observerAccountId
|
|
8550
|
+
);
|
|
8551
|
+
}
|
|
8552
|
+
}
|
|
8553
|
+
if (peek.kind === "attempt-owned") {
|
|
8554
|
+
let ownerActivityState = "unknown";
|
|
8555
|
+
if (inspectSessionAttemptActivity) {
|
|
8556
|
+
try {
|
|
8557
|
+
ownerActivityState = await inspectSessionAttemptActivity(peek.activityRef);
|
|
8558
|
+
} catch (error) {
|
|
8559
|
+
if (error instanceof CancelledFailure) throw error;
|
|
8560
|
+
if (currentActivityContext()?.cancellationSignal.aborted)
|
|
8561
|
+
throw new CancelledFailure("Control observation cancelled");
|
|
8562
|
+
}
|
|
8563
|
+
}
|
|
8564
|
+
if (currentActivityContext()?.cancellationSignal.aborted)
|
|
8565
|
+
throw new CancelledFailure("Control observation cancelled");
|
|
8566
|
+
const current = await peekSessionWorkFn(
|
|
8567
|
+
db,
|
|
8568
|
+
input.workspaceId,
|
|
8569
|
+
input.sessionId,
|
|
8570
|
+
input.includeAdmissionFence,
|
|
8571
|
+
observerAccountId
|
|
8572
|
+
);
|
|
8573
|
+
if (current.kind !== "attempt-owned" || current.turnId !== peek.turnId || current.attemptId !== peek.attemptId || current.executionGeneration !== peek.executionGeneration || current.activityRef.workflowId !== peek.activityRef.workflowId || current.activityRef.workflowRunId !== peek.activityRef.workflowRunId || current.activityRef.activityId !== peek.activityRef.activityId)
|
|
8574
|
+
return current;
|
|
8575
|
+
return { ...current, ownerActivityState };
|
|
8576
|
+
}
|
|
7096
8577
|
await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
|
|
7097
8578
|
return peek;
|
|
7098
8579
|
}
|
|
@@ -7123,6 +8604,14 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7123
8604
|
}
|
|
7124
8605
|
return { action: result.action };
|
|
7125
8606
|
}
|
|
8607
|
+
async function expireScheduledRunHumanWait(input) {
|
|
8608
|
+
const { db, bus } = await services();
|
|
8609
|
+
const result = await expireScheduledRunHumanWaitFn(db, input);
|
|
8610
|
+
if (result.events.length > 0) {
|
|
8611
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
8612
|
+
}
|
|
8613
|
+
return { action: result.action };
|
|
8614
|
+
}
|
|
7126
8615
|
async function markSessionIdle(input) {
|
|
7127
8616
|
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
7128
8617
|
const settled = await settleSessionIdleWithParentOutboxFn(
|
|
@@ -7149,12 +8638,14 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7149
8638
|
settleSessionInterruptions,
|
|
7150
8639
|
persistSessionAttemptQuiescence,
|
|
7151
8640
|
reconcileSessionAttemptQuiescence: reconcileSessionAttemptQuiescenceActivity,
|
|
8641
|
+
reconcileSettledSessionAttempt: reconcileSettledSessionAttemptActivity,
|
|
7152
8642
|
recoverDispatch,
|
|
7153
8643
|
recoverEscapedMcpTimeout,
|
|
7154
8644
|
peekSessionWork,
|
|
7155
8645
|
settleSessionInputWait,
|
|
7156
8646
|
expireSessionHumanInput,
|
|
7157
8647
|
expireSessionInteractionIntervention,
|
|
8648
|
+
expireScheduledRunHumanWait,
|
|
7158
8649
|
markSessionIdle
|
|
7159
8650
|
};
|
|
7160
8651
|
}
|
|
@@ -7173,6 +8664,7 @@ import {
|
|
|
7173
8664
|
getWorkspaceControlEvent
|
|
7174
8665
|
} from "@opengeni/db";
|
|
7175
8666
|
import { publishDurableWorkspaceControlEvent } from "@opengeni/events";
|
|
8667
|
+
import { recordWorkflowWakeReconciliation } from "@opengeni/observability";
|
|
7176
8668
|
var BATCH_SIZE = 1e3;
|
|
7177
8669
|
var MAX_DELIVERIES_PER_ACTIVITY = 1e4;
|
|
7178
8670
|
var MAX_TITLE_FANOUT_PER_ACTIVITY = 1e3;
|
|
@@ -7258,23 +8750,7 @@ function createWorkflowWakeActivities(services) {
|
|
|
7258
8750
|
pendingAdmissionBlockers,
|
|
7259
8751
|
exhaustedBatchLimit
|
|
7260
8752
|
};
|
|
7261
|
-
|
|
7262
|
-
service.observability.info("session workflow wake delivery reconciled", {
|
|
7263
|
-
claimed,
|
|
7264
|
-
signaled,
|
|
7265
|
-
delivered,
|
|
7266
|
-
pendingAdmission,
|
|
7267
|
-
unconfirmed,
|
|
7268
|
-
failed,
|
|
7269
|
-
exhaustedBatchLimit,
|
|
7270
|
-
...Object.fromEntries(
|
|
7271
|
-
Object.entries(pendingAdmissionBlockers).map(([blocker, count]) => [
|
|
7272
|
-
`pendingAdmissionBlockers.${blocker}`,
|
|
7273
|
-
count
|
|
7274
|
-
])
|
|
7275
|
-
)
|
|
7276
|
-
});
|
|
7277
|
-
}
|
|
8753
|
+
recordWorkflowWakeReconciliation(service.observability, result);
|
|
7278
8754
|
return result;
|
|
7279
8755
|
}
|
|
7280
8756
|
};
|
|
@@ -7298,7 +8774,9 @@ function createControlActivitiesFromServices(services, resolveDocumentServices)
|
|
|
7298
8774
|
...createCodexCapacityActivities(services),
|
|
7299
8775
|
...createRigVerificationActivities(services),
|
|
7300
8776
|
...createBrowserStateArtifactMaintenanceActivities(services),
|
|
8777
|
+
...createBrowserDeadlineCheckpointActivities(services),
|
|
7301
8778
|
...createFileUploadReaperActivities(services),
|
|
8779
|
+
...createModelCallFactReconcilerActivities(services),
|
|
7302
8780
|
...createRetainedScreenshotMaintenanceActivities(services),
|
|
7303
8781
|
...createWorkflowWakeActivities(services),
|
|
7304
8782
|
...createSandboxLeaseActivities(services)
|
|
@@ -7319,4 +8797,4 @@ export {
|
|
|
7319
8797
|
createControlActivities,
|
|
7320
8798
|
createControlActivitiesFromServices
|
|
7321
8799
|
};
|
|
7322
|
-
//# sourceMappingURL=activities-control-
|
|
8800
|
+
//# sourceMappingURL=activities-control-BNPTRTOE.js.map
|