@opengeni/worker-bundle 0.13.10 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +3 -3
- package/dist/index.d.ts +2 -2
- package/dist/index.js +122 -75
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +7 -1
- package/dist/sandbox-routing.d.ts +3 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +30 -7
- package/src/activities/parent-wake.ts +8 -0
- package/src/activities/sandbox-lease.ts +68 -44
- package/src/activities/scheduled-tasks.ts +9 -0
- package/src/index.ts +2 -2
- package/src/observability-metrics.ts +38 -26
- package/src/sandbox-routing.ts +5 -0
|
@@ -10,10 +10,10 @@ import { type SecretForRedaction } from "./redaction";
|
|
|
10
10
|
import { type TurnCodexAccount } from "./run-input";
|
|
11
11
|
import type { ActivityServices, EscapedMcpTimeoutRecoveryDetail, RunAgentTurnInput, RunAgentTurnResult, SessionAttemptQuiescenceProof } from "./types";
|
|
12
12
|
import { type TurnSandboxLeaseHolderId } from "../sandbox-resume";
|
|
13
|
-
import { type LatencyMode, type SessionEvent, type SessionEventType, type SessionTurn, type ToolAuthNeededPayload, type TurnExecutionPolicyV1
|
|
13
|
+
import { type LatencyMode, type SessionEvent, type SessionEventType, type SessionTurn, type ToolAuthNeededPayload, type TurnExecutionPolicyV1 } from "@opengeni/contracts";
|
|
14
14
|
export declare const PROVIDER_BACKPRESSURE_DELAY_MS = 60000;
|
|
15
|
-
/**
|
|
16
|
-
export declare function credentialSubjectIdForTurnInitiator(
|
|
15
|
+
/** Broad personal lookup is allowed only for a direct human/API command. */
|
|
16
|
+
export declare function credentialSubjectIdForTurnInitiator(turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">): string | undefined;
|
|
17
17
|
/**
|
|
18
18
|
* Direct authenticated-human work is the persisted human source with subject
|
|
19
19
|
* authority and no inherited agent or legacy provenance. Use this complete
|
package/dist/index.d.ts
CHANGED
|
@@ -62,8 +62,8 @@ export declare function createWorkerWorkflowSignaler(settings: Settings, db: Dat
|
|
|
62
62
|
/**
|
|
63
63
|
* Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
|
|
64
64
|
* driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
|
|
65
|
-
* sandbox ownership off the activity performs
|
|
66
|
-
* it never
|
|
65
|
+
* sandbox ownership off the activity performs bounded DB outbox repair and
|
|
66
|
+
* read-only observability projections; it never mutates or terminates sandbox leases.
|
|
67
67
|
*
|
|
68
68
|
* The Schedule fires sandboxReaperWorkflow on the worker's global task queue
|
|
69
69
|
* every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
|
package/dist/index.js
CHANGED
|
@@ -71,6 +71,7 @@ import {
|
|
|
71
71
|
CODEX_CREDENTIAL_LEASE_TTL_MS,
|
|
72
72
|
getCodexRotationSettings,
|
|
73
73
|
getWorkspaceModelPolicy as getWorkspaceModelPolicy2,
|
|
74
|
+
getWorkspaceGrant,
|
|
74
75
|
listCodexAccountStatuses as listCodexAccountStatuses2,
|
|
75
76
|
fetchCodexUsageForAccount as fetchCodexUsageForAccount2,
|
|
76
77
|
getSessionCodexState,
|
|
@@ -115,6 +116,7 @@ import {
|
|
|
115
116
|
markSessionAttemptQuiesced
|
|
116
117
|
} from "@opengeni/db";
|
|
117
118
|
import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
|
|
119
|
+
import { sandboxOperationMetricObserver } from "@opengeni/observability";
|
|
118
120
|
import {
|
|
119
121
|
sandboxStateEntryFromRunState,
|
|
120
122
|
maxTurnsExceededRunState,
|
|
@@ -1026,6 +1028,8 @@ function workflowIdForSession(sessionId) {
|
|
|
1026
1028
|
import {
|
|
1027
1029
|
assertSessionAllowsProductModel,
|
|
1028
1030
|
defaultSessionMcpServerIds,
|
|
1031
|
+
directPersonalConnectionSubjectId,
|
|
1032
|
+
withFrozenPersonalConnectionDelegations,
|
|
1029
1033
|
resolveSessionToolPolicy
|
|
1030
1034
|
} from "@opengeni/core";
|
|
1031
1035
|
|
|
@@ -2802,6 +2806,7 @@ function mergeRigDefaultVariableSetEnvironment(rigDefaultValues, sessionValues)
|
|
|
2802
2806
|
import {
|
|
2803
2807
|
getSession,
|
|
2804
2808
|
getSessionGoal as getSessionGoal2,
|
|
2809
|
+
getSessionParentPersonalConnectionDelegations,
|
|
2805
2810
|
addSessionSystemUpdateWithSourceMutation,
|
|
2806
2811
|
claimPendingSessionSystemUpdateOutbox,
|
|
2807
2812
|
claimPendingSessionWorkflowWakes,
|
|
@@ -2820,6 +2825,11 @@ async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episode
|
|
|
2820
2825
|
const goal = await getSessionGoal2(svc.db, workspaceId, childSessionId);
|
|
2821
2826
|
const clientEventId = `child-completion:${childSessionId}:${episodeKey}`;
|
|
2822
2827
|
const payload = childCompletionPayload(child, goal);
|
|
2828
|
+
const personalConnectionDelegations = await getSessionParentPersonalConnectionDelegations(
|
|
2829
|
+
svc.db,
|
|
2830
|
+
workspaceId,
|
|
2831
|
+
childSessionId
|
|
2832
|
+
);
|
|
2823
2833
|
const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
|
|
2824
2834
|
accountId: child.accountId,
|
|
2825
2835
|
workspaceId,
|
|
@@ -2831,7 +2841,8 @@ async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episode
|
|
|
2831
2841
|
dedupeKey: clientEventId,
|
|
2832
2842
|
summary: childCompletionSummary(child, goal, "idle"),
|
|
2833
2843
|
payload,
|
|
2834
|
-
lineage: { childSessionId: child.id, parentSessionId: child.parentSessionId }
|
|
2844
|
+
lineage: { childSessionId: child.id, parentSessionId: child.parentSessionId },
|
|
2845
|
+
personalConnectionDelegations
|
|
2835
2846
|
});
|
|
2836
2847
|
if (outbox.status === "delivered") {
|
|
2837
2848
|
return;
|
|
@@ -2881,7 +2892,8 @@ async function deliverParentSystemUpdateOutbox(svc, outbox) {
|
|
|
2881
2892
|
dedupeKey: outbox.dedupeKey,
|
|
2882
2893
|
summary: outbox.summary,
|
|
2883
2894
|
payload: outbox.payload,
|
|
2884
|
-
lineage: outbox.lineage
|
|
2895
|
+
lineage: outbox.lineage,
|
|
2896
|
+
personalConnectionDelegations: outbox.personalConnectionDelegations
|
|
2885
2897
|
},
|
|
2886
2898
|
async (tx) => {
|
|
2887
2899
|
await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
|
|
@@ -4474,6 +4486,7 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
4474
4486
|
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
4475
4487
|
},
|
|
4476
4488
|
resolveActiveBackend: resolver,
|
|
4489
|
+
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
4477
4490
|
...beforeMutation ? { beforeMutation } : {},
|
|
4478
4491
|
...afterMutation ? { afterMutation } : {},
|
|
4479
4492
|
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
@@ -4581,6 +4594,7 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
4581
4594
|
}
|
|
4582
4595
|
return routedResolver(pointer);
|
|
4583
4596
|
},
|
|
4597
|
+
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
4584
4598
|
...beforeMutation ? { beforeMutation } : {},
|
|
4585
4599
|
...afterMutation ? { afterMutation } : {},
|
|
4586
4600
|
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
@@ -4690,6 +4704,7 @@ function makeTurnOpJournal(context, details) {
|
|
|
4690
4704
|
// src/observability-metrics.ts
|
|
4691
4705
|
import { createHash as createHash2 } from "crypto";
|
|
4692
4706
|
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
4707
|
+
import { SandboxBackend } from "@opengeni/contracts";
|
|
4693
4708
|
import {
|
|
4694
4709
|
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
|
|
4695
4710
|
modelUsageTokenCountOrNull
|
|
@@ -4697,7 +4712,6 @@ import {
|
|
|
4697
4712
|
var turnTrackers = /* @__PURE__ */ new WeakMap();
|
|
4698
4713
|
var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
|
|
4699
4714
|
var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
|
|
4700
|
-
var retainedProcessTerminalBacklogTotals = /* @__PURE__ */ new WeakMap();
|
|
4701
4715
|
var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
|
|
4702
4716
|
function observabilityEventLogger(observability) {
|
|
4703
4717
|
return {
|
|
@@ -4949,6 +4963,21 @@ function recordSandboxLeaseGauges(observability, counts) {
|
|
|
4949
4963
|
});
|
|
4950
4964
|
}
|
|
4951
4965
|
}
|
|
4966
|
+
function recordSandboxInventoryProjectionSuccess(observability, domain, timestampSeconds = Date.now() / 1e3) {
|
|
4967
|
+
observability.setGauge({
|
|
4968
|
+
name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
|
|
4969
|
+
help: "Unix timestamp of the last complete successful sandbox inventory projection by domain.",
|
|
4970
|
+
labels: { domain },
|
|
4971
|
+
value: timestampSeconds
|
|
4972
|
+
});
|
|
4973
|
+
}
|
|
4974
|
+
function recordSandboxInventoryProjectionFailure(observability, domain) {
|
|
4975
|
+
observability.incrementCounter({
|
|
4976
|
+
name: "opengeni_sandbox_inventory_refresh_failures_total",
|
|
4977
|
+
help: "Failed sandbox inventory projection refreshes by domain.",
|
|
4978
|
+
labels: { domain }
|
|
4979
|
+
});
|
|
4980
|
+
}
|
|
4952
4981
|
function recordCreditBalanceGauges(observability, balances) {
|
|
4953
4982
|
const previous = creditBalanceGaugeAccounts.get(observability) ?? /* @__PURE__ */ new Set();
|
|
4954
4983
|
const current = /* @__PURE__ */ new Set();
|
|
@@ -5062,9 +5091,7 @@ function recordRetainedProcessInventoryGauges(observability, counts) {
|
|
|
5062
5091
|
current.active += Math.max(0, count.activeCount);
|
|
5063
5092
|
current.terminal += Math.max(0, count.terminalOwnerCount);
|
|
5064
5093
|
}
|
|
5065
|
-
let terminalTotal = 0;
|
|
5066
5094
|
for (const [ownerState, count] of normalized) {
|
|
5067
|
-
terminalTotal += count.terminal;
|
|
5068
5095
|
observability.setGauge({
|
|
5069
5096
|
name: "opengeni_retained_processes_active",
|
|
5070
5097
|
help: "Current active retained provider processes by durable owner state.",
|
|
@@ -5078,28 +5105,8 @@ function recordRetainedProcessInventoryGauges(observability, counts) {
|
|
|
5078
5105
|
value: count.terminal
|
|
5079
5106
|
});
|
|
5080
5107
|
}
|
|
5081
|
-
|
|
5082
|
-
|
|
5083
|
-
observability.incrementCounter({
|
|
5084
|
-
name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
|
|
5085
|
-
help: "Positive growth in terminal-owner retained-process backlog between app samples.",
|
|
5086
|
-
amount: terminalTotal - previous
|
|
5087
|
-
});
|
|
5088
|
-
}
|
|
5089
|
-
retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
|
|
5090
|
-
}
|
|
5091
|
-
var EXPIRED_DRAINING_BACKENDS = [
|
|
5092
|
-
"none",
|
|
5093
|
-
"local",
|
|
5094
|
-
"docker",
|
|
5095
|
-
"modal",
|
|
5096
|
-
"e2b",
|
|
5097
|
-
"daytona",
|
|
5098
|
-
"blaxel",
|
|
5099
|
-
"runloop",
|
|
5100
|
-
"selfhosted",
|
|
5101
|
-
"unknown"
|
|
5102
|
-
];
|
|
5108
|
+
}
|
|
5109
|
+
var EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, "unknown"];
|
|
5103
5110
|
var EXPIRED_DRAINING_BACKEND_SET = new Set(EXPIRED_DRAINING_BACKENDS);
|
|
5104
5111
|
var EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
|
|
5105
5112
|
function recordExpiredDrainingSandboxLeaseGauges(observability, counts) {
|
|
@@ -6288,11 +6295,11 @@ import {
|
|
|
6288
6295
|
} from "@opengeni/contracts";
|
|
6289
6296
|
import { createHash as createHash4, randomUUID as randomUUID2 } from "crypto";
|
|
6290
6297
|
var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
|
|
6291
|
-
function credentialSubjectIdForTurnInitiator(
|
|
6292
|
-
return
|
|
6298
|
+
function credentialSubjectIdForTurnInitiator(turn) {
|
|
6299
|
+
return directPersonalConnectionSubjectId(turn);
|
|
6293
6300
|
}
|
|
6294
6301
|
function isDirectHumanTurnInitiation(turn) {
|
|
6295
|
-
if (turn.source !== "user" || turn.initiator.kind !== "subject") {
|
|
6302
|
+
if (turn.source !== "user" && turn.source !== "api" || turn.initiator.kind !== "subject") {
|
|
6296
6303
|
return false;
|
|
6297
6304
|
}
|
|
6298
6305
|
return !["via", "viaTruncated", "provenanceError", "backfill"].some(
|
|
@@ -7291,6 +7298,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7291
7298
|
const machineOpObserver = makeMachineOpObserver(
|
|
7292
7299
|
runtimeMetricsHooksForObservability(observability)
|
|
7293
7300
|
);
|
|
7301
|
+
const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
|
|
7294
7302
|
let isCodexTurn = false;
|
|
7295
7303
|
let isExternallyBilledTurn = false;
|
|
7296
7304
|
let executionGeneration = 0;
|
|
@@ -9459,6 +9467,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9459
9467
|
settings,
|
|
9460
9468
|
bus,
|
|
9461
9469
|
onOp: machineOpObserver.observer,
|
|
9470
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
9462
9471
|
opJournal
|
|
9463
9472
|
},
|
|
9464
9473
|
{
|
|
@@ -9484,6 +9493,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9484
9493
|
settings,
|
|
9485
9494
|
bus,
|
|
9486
9495
|
onOp: machineOpObserver.observer,
|
|
9496
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
9487
9497
|
onHomeSandboxRebound
|
|
9488
9498
|
},
|
|
9489
9499
|
{
|
|
@@ -9572,6 +9582,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9572
9582
|
db,
|
|
9573
9583
|
settings,
|
|
9574
9584
|
bus,
|
|
9585
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
9575
9586
|
onHomeSandboxLost: publishSandboxLost,
|
|
9576
9587
|
onHomeSandboxRebound
|
|
9577
9588
|
},
|
|
@@ -9630,8 +9641,23 @@ function createRunAgentTurnActivity(services) {
|
|
|
9630
9641
|
attemptId: input.attemptId,
|
|
9631
9642
|
turn
|
|
9632
9643
|
});
|
|
9644
|
+
const personalConnectionDelegations = turn.personalConnectionDelegations;
|
|
9645
|
+
const delegatedMembershipChecks = /* @__PURE__ */ new Map();
|
|
9646
|
+
const delegatedOwnerHasMembership = async (subjectId) => {
|
|
9647
|
+
const existing = delegatedMembershipChecks.get(subjectId);
|
|
9648
|
+
if (existing) return await existing;
|
|
9649
|
+
const check = getWorkspaceGrant(db, subjectId, input.workspaceId).then(Boolean);
|
|
9650
|
+
delegatedMembershipChecks.set(subjectId, check);
|
|
9651
|
+
return await check;
|
|
9652
|
+
};
|
|
9653
|
+
const resolveFrozenCredential = withFrozenPersonalConnectionDelegations({
|
|
9654
|
+
resolveCredential: rawResolveCredential,
|
|
9655
|
+
settings: runSettings,
|
|
9656
|
+
personalConnectionDelegations,
|
|
9657
|
+
ownerHasWorkspaceMembership: delegatedOwnerHasMembership
|
|
9658
|
+
});
|
|
9633
9659
|
const resolveCredential = async (request) => {
|
|
9634
|
-
const result = await
|
|
9660
|
+
const result = await resolveFrozenCredential(request);
|
|
9635
9661
|
if (result.status === "ok") {
|
|
9636
9662
|
registerSecretRedactions(
|
|
9637
9663
|
headerSecretRedactions("MCP", result.headers, Object.keys(result.headers))
|
|
@@ -9639,7 +9665,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9639
9665
|
}
|
|
9640
9666
|
return result;
|
|
9641
9667
|
};
|
|
9642
|
-
const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn
|
|
9668
|
+
const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
|
|
9643
9669
|
preparedTools = await waitForTurnOperation(
|
|
9644
9670
|
withCodex(
|
|
9645
9671
|
() => runtime.prepareTools(runSettings, turnTools, {
|
|
@@ -9970,6 +9996,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9970
9996
|
settings,
|
|
9971
9997
|
bus,
|
|
9972
9998
|
onOp: machineOpObserver.observer,
|
|
9999
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
9973
10000
|
onHomeSandboxLost: publishSandboxLost,
|
|
9974
10001
|
onHomeSandboxRebound
|
|
9975
10002
|
},
|
|
@@ -12724,6 +12751,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
12724
12751
|
deleted: expiredSessionListSnapshots
|
|
12725
12752
|
});
|
|
12726
12753
|
}
|
|
12754
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
12727
12755
|
if (!settings.sandboxOwnershipEnabled) {
|
|
12728
12756
|
if (parentUpdates.claimed > 0) {
|
|
12729
12757
|
observability.info("system-update outbox reconciled", parentUpdates);
|
|
@@ -12737,7 +12765,6 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
12737
12765
|
modalOrphansTerminated: 0
|
|
12738
12766
|
};
|
|
12739
12767
|
}
|
|
12740
|
-
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
12741
12768
|
const rotationsRequested = await requestDueSandboxRotationsGlobal(
|
|
12742
12769
|
db,
|
|
12743
12770
|
settings.sandboxRotationLeadMs,
|
|
@@ -13453,54 +13480,65 @@ async function refreshQueueLeaseAndCreditGauges(db, observability) {
|
|
|
13453
13480
|
error: error instanceof Error ? error.message : String(error)
|
|
13454
13481
|
});
|
|
13455
13482
|
}
|
|
13456
|
-
|
|
13483
|
+
await refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
|
|
13457
13484
|
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
13458
|
-
}
|
|
13459
|
-
|
|
13460
|
-
|
|
13461
|
-
|
|
13462
|
-
|
|
13463
|
-
|
|
13464
|
-
|
|
13465
|
-
|
|
13466
|
-
|
|
13467
|
-
|
|
13468
|
-
|
|
13469
|
-
|
|
13470
|
-
|
|
13471
|
-
|
|
13472
|
-
|
|
13473
|
-
|
|
13474
|
-
|
|
13475
|
-
|
|
13476
|
-
|
|
13477
|
-
|
|
13478
|
-
|
|
13479
|
-
|
|
13480
|
-
|
|
13481
|
-
|
|
13482
|
-
|
|
13483
|
-
|
|
13484
|
-
|
|
13485
|
-
|
|
13486
|
-
|
|
13487
|
-
|
|
13488
|
-
|
|
13489
|
-
|
|
13485
|
+
});
|
|
13486
|
+
await refreshSandboxInventoryGauge(
|
|
13487
|
+
observability,
|
|
13488
|
+
"checkpoint_artifacts",
|
|
13489
|
+
"checkpoint-artifact",
|
|
13490
|
+
async () => {
|
|
13491
|
+
recordSandboxCheckpointArtifactGauges(
|
|
13492
|
+
observability,
|
|
13493
|
+
await countSandboxCheckpointArtifactsByState(db)
|
|
13494
|
+
);
|
|
13495
|
+
}
|
|
13496
|
+
);
|
|
13497
|
+
await refreshSandboxInventoryGauge(
|
|
13498
|
+
observability,
|
|
13499
|
+
"rotation_backlog",
|
|
13500
|
+
"rotation-backlog",
|
|
13501
|
+
async () => {
|
|
13502
|
+
recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
|
|
13503
|
+
}
|
|
13504
|
+
);
|
|
13505
|
+
await refreshSandboxInventoryGauge(
|
|
13506
|
+
observability,
|
|
13507
|
+
"retained_processes",
|
|
13508
|
+
"retained-process",
|
|
13509
|
+
async () => {
|
|
13510
|
+
recordRetainedProcessInventoryGauges(
|
|
13511
|
+
observability,
|
|
13512
|
+
await countActiveRetainedProcessesByOwnerState(db)
|
|
13513
|
+
);
|
|
13514
|
+
}
|
|
13515
|
+
);
|
|
13516
|
+
await refreshSandboxInventoryGauge(
|
|
13517
|
+
observability,
|
|
13518
|
+
"expired_drains",
|
|
13519
|
+
"expired-draining",
|
|
13520
|
+
async () => {
|
|
13521
|
+
recordExpiredDrainingSandboxLeaseGauges(
|
|
13522
|
+
observability,
|
|
13523
|
+
await countExpiredDrainingSandboxLeases(db)
|
|
13524
|
+
);
|
|
13525
|
+
}
|
|
13526
|
+
);
|
|
13490
13527
|
try {
|
|
13491
|
-
|
|
13492
|
-
observability,
|
|
13493
|
-
await countExpiredDrainingSandboxLeases(db)
|
|
13494
|
-
);
|
|
13528
|
+
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
13495
13529
|
} catch (error) {
|
|
13496
|
-
observability.warn("sandbox reaper:
|
|
13530
|
+
observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
|
|
13497
13531
|
error: error instanceof Error ? error.message : String(error)
|
|
13498
13532
|
});
|
|
13499
13533
|
}
|
|
13534
|
+
}
|
|
13535
|
+
async function refreshSandboxInventoryGauge(observability, domain, diagnosticName, refresh) {
|
|
13500
13536
|
try {
|
|
13501
|
-
|
|
13537
|
+
await refresh();
|
|
13538
|
+
recordSandboxInventoryProjectionSuccess(observability, domain);
|
|
13502
13539
|
} catch (error) {
|
|
13503
|
-
observability
|
|
13540
|
+
recordSandboxInventoryProjectionFailure(observability, domain);
|
|
13541
|
+
observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {
|
|
13504
13542
|
error: error instanceof Error ? error.message : String(error)
|
|
13505
13543
|
});
|
|
13506
13544
|
}
|
|
@@ -13997,6 +14035,7 @@ import {
|
|
|
13997
14035
|
enqueueSessionWorkflowWakeIfRunnable as enqueueSessionWorkflowWakeIfRunnable2,
|
|
13998
14036
|
getBillingBalance as getBillingBalance5,
|
|
13999
14037
|
getScheduledTask,
|
|
14038
|
+
getScheduledTaskPersonalConnectionDelegations,
|
|
14000
14039
|
getRig,
|
|
14001
14040
|
getVariableSet,
|
|
14002
14041
|
isCodexBilledTurn as isCodexBilledTurn2,
|
|
@@ -14023,6 +14062,11 @@ function createScheduledTaskActivities(services) {
|
|
|
14023
14062
|
if (!task) {
|
|
14024
14063
|
return { action: "deleted" };
|
|
14025
14064
|
}
|
|
14065
|
+
const taskPersonalConnectionDelegations = await getScheduledTaskPersonalConnectionDelegations(
|
|
14066
|
+
db,
|
|
14067
|
+
task.workspaceId,
|
|
14068
|
+
task.id
|
|
14069
|
+
);
|
|
14026
14070
|
const slackBotConnection = task.agentConfig.slackBotConnectionId ? await requireOpenGeniSlackBotConnection(
|
|
14027
14071
|
db,
|
|
14028
14072
|
task.workspaceId,
|
|
@@ -14168,6 +14212,7 @@ function createScheduledTaskActivities(services) {
|
|
|
14168
14212
|
variableSetId: task.variableSetId ?? null,
|
|
14169
14213
|
rigId: frozenRigId,
|
|
14170
14214
|
rigVersionId: frozenRigVersionId,
|
|
14215
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
14171
14216
|
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
14172
14217
|
// The durable agent config was privilege-checked when the task
|
|
14173
14218
|
// was created/updated. Preserve that explicit policy if a broader
|
|
@@ -14250,7 +14295,8 @@ function createScheduledTaskActivities(services) {
|
|
|
14250
14295
|
task.id,
|
|
14251
14296
|
run.id
|
|
14252
14297
|
),
|
|
14253
|
-
lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id }
|
|
14298
|
+
lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
14299
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations
|
|
14254
14300
|
},
|
|
14255
14301
|
async (tx, wakeEventId) => {
|
|
14256
14302
|
if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
|
|
@@ -14349,7 +14395,8 @@ function createScheduledTaskActivities(services) {
|
|
|
14349
14395
|
task.id,
|
|
14350
14396
|
run.id
|
|
14351
14397
|
),
|
|
14352
|
-
lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id }
|
|
14398
|
+
lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
14399
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations
|
|
14353
14400
|
},
|
|
14354
14401
|
async (tx, wakeEventId) => {
|
|
14355
14402
|
if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
|