@opengeni/worker-bundle 1.0.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -14
- package/dist/activities/agent-turn/admission.d.ts +3 -2
- package/dist/activities/agent-turn/compaction-prep.d.ts +1 -0
- package/dist/activities/agent-turn/errors.d.ts +3 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +6 -1
- package/dist/activities/agent-turn/model-usage.d.ts +10 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/turn-context.d.ts +5 -0
- package/dist/activities/goals.d.ts +1 -17
- package/dist/activities/types.d.ts +1 -1
- package/dist/{activities-control-BNPTRTOE.js → activities-control-QSDPXLLD.js} +26 -10
- package/dist/activities-control-QSDPXLLD.js.map +1 -0
- package/dist/{activities-turn-5LGZOSVU.js → activities-turn-BMY3CDKL.js} +191 -35
- package/dist/activities-turn-BMY3CDKL.js.map +1 -0
- package/dist/{chunk-LJX7Z2KP.js → chunk-JKMG5ZHH.js} +55 -167
- package/dist/chunk-JKMG5ZHH.js.map +1 -0
- package/dist/{chunk-3ZTRWDMN.js → chunk-XONQEVAD.js} +1 -1
- package/dist/chunk-XONQEVAD.js.map +1 -0
- package/dist/index.js +3 -4
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/workflow-bundle.js +1 -1
- package/package.json +21 -21
- package/src/activities/agent-run-admission.ts +1 -98
- package/src/activities/agent-turn/admission.ts +11 -5
- package/src/activities/agent-turn/claim.ts +12 -0
- package/src/activities/agent-turn/compaction-prep.ts +35 -1
- package/src/activities/agent-turn/errors.ts +51 -19
- package/src/activities/agent-turn/failure-settlement.ts +31 -1
- package/src/activities/agent-turn/finalization-monitor.ts +14 -1
- package/src/activities/agent-turn/finalization.ts +5 -0
- package/src/activities/agent-turn/model-usage.ts +38 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/run.ts +1 -0
- package/src/activities/agent-turn/stream-attempt.ts +32 -16
- package/src/activities/agent-turn/tool-environment.ts +9 -2
- package/src/activities/agent-turn/turn-context.ts +4 -0
- package/src/activities/goals.ts +17 -113
- package/src/activities/knowledge-indexing.ts +2 -2
- package/src/activities/scheduled-tasks.ts +18 -3
- package/src/activities/types.ts +1 -0
- package/src/index.ts +0 -2
- package/src/observability-metrics.ts +1 -0
- package/dist/activities-control-BNPTRTOE.js.map +0 -1
- package/dist/activities-turn-5LGZOSVU.js.map +0 -1
- package/dist/chunk-3ZTRWDMN.js.map +0 -1
- package/dist/chunk-LJX7Z2KP.js.map +0 -1
|
@@ -99,7 +99,7 @@ import {
|
|
|
99
99
|
withFirstPartyTools,
|
|
100
100
|
wrapLazyTurnBoxWithRouting,
|
|
101
101
|
wrapTurnBoxWithRouting
|
|
102
|
-
} from "./chunk-
|
|
102
|
+
} from "./chunk-JKMG5ZHH.js";
|
|
103
103
|
import {
|
|
104
104
|
StreamTimingMetrics,
|
|
105
105
|
makeMachineOpObserver,
|
|
@@ -132,7 +132,7 @@ import {
|
|
|
132
132
|
runtimeMetricsHooksForObservability,
|
|
133
133
|
sandboxAutomaticRecoveryOutcome,
|
|
134
134
|
turnLifecycleMetricsFor
|
|
135
|
-
} from "./chunk-
|
|
135
|
+
} from "./chunk-XONQEVAD.js";
|
|
136
136
|
import {
|
|
137
137
|
__export
|
|
138
138
|
} from "./chunk-PZ5AY32C.js";
|
|
@@ -163,7 +163,7 @@ import {
|
|
|
163
163
|
} from "@opengeni/db";
|
|
164
164
|
import {
|
|
165
165
|
sandboxOperationMetricObserver as sandboxOperationMetricObserver2,
|
|
166
|
-
turnExecutionTelemetryKey as
|
|
166
|
+
turnExecutionTelemetryKey as turnExecutionTelemetryKey3,
|
|
167
167
|
withTraceContext,
|
|
168
168
|
withMcpTelemetry
|
|
169
169
|
} from "@opengeni/observability";
|
|
@@ -3670,6 +3670,7 @@ import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
|
|
|
3670
3670
|
import { environmentsEncryptionKeyBytes } from "@opengeni/config";
|
|
3671
3671
|
|
|
3672
3672
|
// src/activities/agent-turn/finalization-monitor.ts
|
|
3673
|
+
import { turnExecutionTelemetryKey } from "@opengeni/observability";
|
|
3673
3674
|
var TURN_FINALIZATION_STAGES = [
|
|
3674
3675
|
"tool_writers",
|
|
3675
3676
|
"credential_renewals",
|
|
@@ -3701,6 +3702,16 @@ function diagnose(operation) {
|
|
|
3701
3702
|
}
|
|
3702
3703
|
function startTurnFinalizationMonitor(input) {
|
|
3703
3704
|
const { observability, details } = input;
|
|
3705
|
+
const correlation = {};
|
|
3706
|
+
diagnose(() => {
|
|
3707
|
+
if (input.execution) {
|
|
3708
|
+
correlation.correlationId = turnExecutionTelemetryKey(
|
|
3709
|
+
input.execution.workspaceId,
|
|
3710
|
+
input.execution.sessionId,
|
|
3711
|
+
input.execution.attemptId
|
|
3712
|
+
);
|
|
3713
|
+
}
|
|
3714
|
+
});
|
|
3704
3715
|
if (!initialized.has(observability)) {
|
|
3705
3716
|
for (const stage2 of TURN_FINALIZATION_STAGES) {
|
|
3706
3717
|
diagnose(() => observability.incrementGauge({ ...INFLIGHT, labels: { stage: stage2 }, amount: 0 }));
|
|
@@ -3743,7 +3754,8 @@ function startTurnFinalizationMonitor(input) {
|
|
|
3743
3754
|
observability.error("turn finalization stalled; containing worker", {
|
|
3744
3755
|
surface: "turn_finalization",
|
|
3745
3756
|
outcome: "containment",
|
|
3746
|
-
reason: next
|
|
3757
|
+
reason: next,
|
|
3758
|
+
...correlation
|
|
3747
3759
|
});
|
|
3748
3760
|
} catch {
|
|
3749
3761
|
}
|
|
@@ -3756,7 +3768,8 @@ function startTurnFinalizationMonitor(input) {
|
|
|
3756
3768
|
observability.warn("turn finalization is taking longer than expected", {
|
|
3757
3769
|
surface: "turn_finalization",
|
|
3758
3770
|
outcome: "slow",
|
|
3759
|
-
reason: next
|
|
3771
|
+
reason: next,
|
|
3772
|
+
...correlation
|
|
3760
3773
|
});
|
|
3761
3774
|
}),
|
|
3762
3775
|
input.slowAfterMs ?? 3e4
|
|
@@ -4618,7 +4631,12 @@ async function finalizeTurnAttempt(deps) {
|
|
|
4618
4631
|
} catch {
|
|
4619
4632
|
}
|
|
4620
4633
|
},
|
|
4621
|
-
requestWorkerDrain: deps.requestWorkerDrain
|
|
4634
|
+
requestWorkerDrain: deps.requestWorkerDrain,
|
|
4635
|
+
execution: {
|
|
4636
|
+
workspaceId: deps.input.workspaceId,
|
|
4637
|
+
sessionId: deps.input.sessionId,
|
|
4638
|
+
attemptId: deps.input.attemptId
|
|
4639
|
+
}
|
|
4622
4640
|
});
|
|
4623
4641
|
try {
|
|
4624
4642
|
monitor.enter("tool_writers");
|
|
@@ -5259,7 +5277,7 @@ import {
|
|
|
5259
5277
|
} from "@opengeni/codex";
|
|
5260
5278
|
|
|
5261
5279
|
// src/activities/agent-turn/admission.ts
|
|
5262
|
-
import { checkWorkspaceAllowance,
|
|
5280
|
+
import { checkWorkspaceAllowance, getSpendableCreditBalance, sumUsageQuantity } from "@opengeni/db";
|
|
5263
5281
|
import {
|
|
5264
5282
|
configuredStaticUsageLimits
|
|
5265
5283
|
} from "@opengeni/config";
|
|
@@ -5268,6 +5286,7 @@ import { directPersonalConnectionSubjectId } from "@opengeni/core";
|
|
|
5268
5286
|
import { createHash as createHash7 } from "crypto";
|
|
5269
5287
|
|
|
5270
5288
|
// src/activities/agent-turn/model-usage.ts
|
|
5289
|
+
import { canonicalizeConfiguredModelId } from "@opengeni/config";
|
|
5271
5290
|
import {
|
|
5272
5291
|
applyCreditDebitUpToBalance,
|
|
5273
5292
|
recordUsageEvent,
|
|
@@ -5297,6 +5316,12 @@ function modelUsageSourceKey(input) {
|
|
|
5297
5316
|
}
|
|
5298
5317
|
return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
|
|
5299
5318
|
}
|
|
5319
|
+
function aggregateCreditPolicyRevision(input) {
|
|
5320
|
+
if (input.chargesOpenGeniCredits && input.responseRevisions.size > 1 && (input.totalTokens ?? 0) > 0) {
|
|
5321
|
+
throw new Error("Aggregate model usage spans different credit policy revisions");
|
|
5322
|
+
}
|
|
5323
|
+
return input.responseRevisions.size === 1 ? input.responseRevisions.values().next().value : input.lastAdmittedRevision;
|
|
5324
|
+
}
|
|
5300
5325
|
async function recordCompletedModelCallBeforeOwnershipFences(input) {
|
|
5301
5326
|
await input.renewLease();
|
|
5302
5327
|
await input.recordUsage();
|
|
@@ -5346,6 +5371,9 @@ async function processModelResponseTerminalEvent(input) {
|
|
|
5346
5371
|
if (!terminal) {
|
|
5347
5372
|
return { status: "not_response" };
|
|
5348
5373
|
}
|
|
5374
|
+
if (!terminal.usage && input.event.type === "raw_model_stream_event" && input.event.data.type !== "response_done") {
|
|
5375
|
+
return { status: "not_response" };
|
|
5376
|
+
}
|
|
5349
5377
|
const responseOrdinal = input.state.responseCount + 1;
|
|
5350
5378
|
const sourceKey = modelUsageSourceKey({
|
|
5351
5379
|
responseId: terminal.responseId,
|
|
@@ -5378,6 +5406,7 @@ async function processModelResponseTerminalEvent(input) {
|
|
|
5378
5406
|
turnId: input.turnId,
|
|
5379
5407
|
turnAttemptId: input.turnAttemptId,
|
|
5380
5408
|
model: input.model,
|
|
5409
|
+
creditPolicyRevision: input.creditPolicyRevision,
|
|
5381
5410
|
externallyBilled: input.externallyBilled,
|
|
5382
5411
|
...input.chargesOpenGeniCredits !== void 0 ? { chargesOpenGeniCredits: input.chargesOpenGeniCredits } : {},
|
|
5383
5412
|
...input.countsTowardTokenCap !== void 0 ? { countsTowardTokenCap: input.countsTowardTokenCap } : {},
|
|
@@ -5478,6 +5507,7 @@ async function processCompactionModelUsageEvent(input) {
|
|
|
5478
5507
|
turnId: input.turnId,
|
|
5479
5508
|
turnAttemptId: input.turnAttemptId,
|
|
5480
5509
|
model: input.model,
|
|
5510
|
+
creditPolicyRevision: input.creditPolicyRevision,
|
|
5481
5511
|
externallyBilled: input.externallyBilled,
|
|
5482
5512
|
...input.chargesOpenGeniCredits !== void 0 ? { chargesOpenGeniCredits: input.chargesOpenGeniCredits } : {},
|
|
5483
5513
|
...input.countsTowardTokenCap !== void 0 ? { countsTowardTokenCap: input.countsTowardTokenCap } : {},
|
|
@@ -5745,6 +5775,8 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
5745
5775
|
workspaceId: input.workspaceId,
|
|
5746
5776
|
type: "model_usage_debit",
|
|
5747
5777
|
requestedAmountMicros: costMicros,
|
|
5778
|
+
modelId: canonicalizeConfiguredModelId(settings, input.model),
|
|
5779
|
+
creditPolicyRevision: input.creditPolicyRevision,
|
|
5748
5780
|
sourceType: "model_response",
|
|
5749
5781
|
sourceId: `${input.turnId}:${input.sourceKey}`,
|
|
5750
5782
|
idempotencyKey: `credit:model_usage_debit:${input.turnId}:${input.sourceKey}`,
|
|
@@ -5946,7 +5978,7 @@ var AllowanceExhaustedError = class extends Error {
|
|
|
5946
5978
|
};
|
|
5947
5979
|
async function ensureRunAllowedBetweenModelCalls(input) {
|
|
5948
5980
|
try {
|
|
5949
|
-
await ensureRunAllowed(
|
|
5981
|
+
return await ensureRunAllowed(
|
|
5950
5982
|
input.settings,
|
|
5951
5983
|
input.db,
|
|
5952
5984
|
input.accountId,
|
|
@@ -5955,7 +5987,8 @@ async function ensureRunAllowedBetweenModelCalls(input) {
|
|
|
5955
5987
|
input.entitlements,
|
|
5956
5988
|
input.chargesOpenGeniCredits,
|
|
5957
5989
|
input.countsTowardTokenCap,
|
|
5958
|
-
input.initiatingHumanSubjectId
|
|
5990
|
+
input.initiatingHumanSubjectId,
|
|
5991
|
+
input.modelId
|
|
5959
5992
|
);
|
|
5960
5993
|
} catch (limitError) {
|
|
5961
5994
|
let serializedRunState = null;
|
|
@@ -5970,7 +6003,8 @@ async function ensureRunAllowedBetweenModelCalls(input) {
|
|
|
5970
6003
|
);
|
|
5971
6004
|
}
|
|
5972
6005
|
}
|
|
5973
|
-
async function ensureRunAllowed(settings, db, accountId, workspaceId, isExternallyBilledTurn, entitlements, chargesOpenGeniCredits = !isExternallyBilledTurn, countsTowardTokenCap = !isExternallyBilledTurn, initiatingHumanSubjectId = null) {
|
|
6006
|
+
async function ensureRunAllowed(settings, db, accountId, workspaceId, isExternallyBilledTurn, entitlements, chargesOpenGeniCredits = !isExternallyBilledTurn, countsTowardTokenCap = !isExternallyBilledTurn, initiatingHumanSubjectId = null, modelId) {
|
|
6007
|
+
let creditPolicyRevision;
|
|
5974
6008
|
if (chargesOpenGeniCredits && entitlements && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
5975
6009
|
const decision = await entitlements.admitRun({
|
|
5976
6010
|
accountId,
|
|
@@ -5982,7 +6016,8 @@ async function ensureRunAllowed(settings, db, accountId, workspaceId, isExternal
|
|
|
5982
6016
|
throw new Error(decision.reason || "insufficient Opengeni credits");
|
|
5983
6017
|
}
|
|
5984
6018
|
} else if (chargesOpenGeniCredits && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
5985
|
-
const balance = await
|
|
6019
|
+
const balance = await getSpendableCreditBalance(db, accountId, modelId);
|
|
6020
|
+
creditPolicyRevision = balance.creditPolicyRevision;
|
|
5986
6021
|
if (balance.balanceMicros <= 0) {
|
|
5987
6022
|
throw new Error("insufficient Opengeni credits");
|
|
5988
6023
|
}
|
|
@@ -6020,6 +6055,48 @@ async function ensureRunAllowed(settings, db, accountId, workspaceId, isExternal
|
|
|
6020
6055
|
}
|
|
6021
6056
|
}
|
|
6022
6057
|
}
|
|
6058
|
+
return creditPolicyRevision;
|
|
6059
|
+
}
|
|
6060
|
+
|
|
6061
|
+
// src/activities/agent-turn/provider-recovery-metrics.ts
|
|
6062
|
+
function providerRecoveryCause(code) {
|
|
6063
|
+
if (code === "provider_rate_limited") return "rate_limited";
|
|
6064
|
+
if (code === "provider_unavailable") return "unavailable";
|
|
6065
|
+
if (code === "upstream_connectivity_unavailable") return "connectivity";
|
|
6066
|
+
return null;
|
|
6067
|
+
}
|
|
6068
|
+
function readProviderRecoveryObservation(metadata) {
|
|
6069
|
+
const cause = providerRecoveryCause(metadata.providerRecoveryReason);
|
|
6070
|
+
const startedAt = typeof metadata.providerRecoveryStartedAt === "string" ? Date.parse(metadata.providerRecoveryStartedAt) : Number.NaN;
|
|
6071
|
+
return cause && Number.isFinite(startedAt) ? { cause, startedAt } : void 0;
|
|
6072
|
+
}
|
|
6073
|
+
function recordProviderRecoveryOutcome(observability, input) {
|
|
6074
|
+
if (!input.route) return;
|
|
6075
|
+
const labels = { ...input.route, cause: input.cause, outcome: input.outcome };
|
|
6076
|
+
try {
|
|
6077
|
+
observability.incrementCounter({
|
|
6078
|
+
name: "opengeni_model_recovery_total",
|
|
6079
|
+
help: "Observed model recovery decisions and successful resumptions.",
|
|
6080
|
+
labels
|
|
6081
|
+
});
|
|
6082
|
+
if (input.delayMs !== void 0) {
|
|
6083
|
+
observability.observeHistogram({
|
|
6084
|
+
name: "opengeni_model_recovery_delay_seconds",
|
|
6085
|
+
help: "Scheduled model recovery delay, including provider hints and jitter.",
|
|
6086
|
+
labels: { ...input.route, cause: input.cause },
|
|
6087
|
+
value: Math.max(0, input.delayMs) / 1e3
|
|
6088
|
+
});
|
|
6089
|
+
}
|
|
6090
|
+
if (input.elapsedMs !== void 0) {
|
|
6091
|
+
observability.observeHistogram({
|
|
6092
|
+
name: "opengeni_model_recovery_duration_seconds",
|
|
6093
|
+
help: "Elapsed recovery episode including backoff, preparation and model requests.",
|
|
6094
|
+
labels,
|
|
6095
|
+
value: Math.max(0, input.elapsedMs) / 1e3
|
|
6096
|
+
});
|
|
6097
|
+
}
|
|
6098
|
+
} catch {
|
|
6099
|
+
}
|
|
6023
6100
|
}
|
|
6024
6101
|
|
|
6025
6102
|
// src/activities/agent-turn/failure-settlement.ts
|
|
@@ -6091,9 +6168,13 @@ async function settleTurnFailure(deps) {
|
|
|
6091
6168
|
try {
|
|
6092
6169
|
return await settleTurnFailureInAttempt(deps);
|
|
6093
6170
|
} catch (error51) {
|
|
6171
|
+
if (deps.control.activityStatus === "recovering" && error51 instanceof ApplicationFailure && error51.type === "OpenGeniPostClaimDatabaseRecovery")
|
|
6172
|
+
throw error51;
|
|
6094
6173
|
if (deps.attempt.turnId && deps.attempt.triggerEventId) {
|
|
6095
6174
|
const recovery = postClaimDatabaseRecoveryFailure({
|
|
6096
|
-
|
|
6175
|
+
// A failed rollback/terminal write must not erase no-replay evidence
|
|
6176
|
+
// from the failure we were settling (notably unknown tool effects).
|
|
6177
|
+
error: new AggregateError([error51, deps.error], "Turn failure settlement failed"),
|
|
6097
6178
|
turnId: deps.attempt.turnId,
|
|
6098
6179
|
triggerEventId: deps.attempt.triggerEventId,
|
|
6099
6180
|
executionGeneration: deps.attempt.executionGeneration,
|
|
@@ -7395,7 +7476,8 @@ async function settleTurnFailureInAttempt(deps) {
|
|
|
7395
7476
|
const recoveryResult = providerRecoveryResult({
|
|
7396
7477
|
failureCode: failure2.code,
|
|
7397
7478
|
attemptNumber: nextProviderRecoveryCount,
|
|
7398
|
-
retryAfterMs: providerRetryAfterMs(error51)
|
|
7479
|
+
retryAfterMs: providerRetryAfterMs(error51),
|
|
7480
|
+
jitterSample: Math.random()
|
|
7399
7481
|
});
|
|
7400
7482
|
const setupRecoveryExhausted = earlyCommandStartUnavailable && recoveryResult.status === "exhausted" && attempt.providerRecoveryCount === SANDBOX_SETUP_RECOVERY_LIMIT;
|
|
7401
7483
|
try {
|
|
@@ -7459,9 +7541,27 @@ async function settleTurnFailureInAttempt(deps) {
|
|
|
7459
7541
|
control.turnMetricOutcome = "recovering";
|
|
7460
7542
|
control.activityStatus = "recovering";
|
|
7461
7543
|
control.activityError = error51;
|
|
7544
|
+
const recoveryCause2 = providerRecoveryCause(failure2.code);
|
|
7545
|
+
if (recoveryCause2) {
|
|
7546
|
+
recordProviderRecoveryOutcome(observability, {
|
|
7547
|
+
route: attempt.modelMetricRoute,
|
|
7548
|
+
cause: recoveryCause2,
|
|
7549
|
+
outcome: "scheduled",
|
|
7550
|
+
delayMs: recoveryResult.continueDelayMs
|
|
7551
|
+
});
|
|
7552
|
+
}
|
|
7462
7553
|
return claimedResult(recoveryResult);
|
|
7463
7554
|
}
|
|
7464
7555
|
failure2 = providerRecoveryExhaustedFailure(failure2, recoveryResult);
|
|
7556
|
+
const recoveryCause = providerRecoveryCause(failure2.code);
|
|
7557
|
+
if (recoveryCause) {
|
|
7558
|
+
recordProviderRecoveryOutcome(observability, {
|
|
7559
|
+
route: attempt.modelMetricRoute,
|
|
7560
|
+
cause: recoveryCause,
|
|
7561
|
+
outcome: "exhausted",
|
|
7562
|
+
...attempt.providerRecoveryObservation ? { elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt } : {}
|
|
7563
|
+
});
|
|
7564
|
+
}
|
|
7465
7565
|
if (earlyRecoverableSetup) {
|
|
7466
7566
|
control.activityStatus = "failed";
|
|
7467
7567
|
control.turnMetricOutcome = "failed";
|
|
@@ -8835,6 +8935,8 @@ async function runTurnStreamAttempt(deps) {
|
|
|
8835
8935
|
);
|
|
8836
8936
|
let parallelSessionTitle = null;
|
|
8837
8937
|
let parallelSessionTitleFinished = false;
|
|
8938
|
+
let creditPolicyRevision;
|
|
8939
|
+
let titleCreditPolicyRevision;
|
|
8838
8940
|
const finishParallelSessionTitle = async () => {
|
|
8839
8941
|
if (parallelSessionTitleFinished) return;
|
|
8840
8942
|
parallelSessionTitleFinished = true;
|
|
@@ -8843,6 +8945,7 @@ async function runTurnStreamAttempt(deps) {
|
|
|
8843
8945
|
if (generated.usage) {
|
|
8844
8946
|
await processSessionTitleModelUsageEvent({
|
|
8845
8947
|
usage: generated.usage,
|
|
8948
|
+
creditPolicyRevision: titleCreditPolicyRevision,
|
|
8846
8949
|
state: sessionTitleUsageState,
|
|
8847
8950
|
dispatchId: modelUsageDispatchId,
|
|
8848
8951
|
settings,
|
|
@@ -9009,7 +9112,8 @@ async function runTurnStreamAttempt(deps) {
|
|
|
9009
9112
|
let finalReplyNudged = false;
|
|
9010
9113
|
const revalidateModelCallAdmission = async () => {
|
|
9011
9114
|
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
9012
|
-
await ensureRunAllowedBetweenModelCalls({
|
|
9115
|
+
creditPolicyRevision = await ensureRunAllowedBetweenModelCalls({
|
|
9116
|
+
modelId: resolvedModel?.configured.id ?? turn.model,
|
|
9013
9117
|
settings,
|
|
9014
9118
|
db,
|
|
9015
9119
|
accountId: input.accountId,
|
|
@@ -9031,6 +9135,7 @@ async function runTurnStreamAttempt(deps) {
|
|
|
9031
9135
|
signal: runtimeCancellationSignal,
|
|
9032
9136
|
admit: revalidateModelCallAdmission
|
|
9033
9137
|
});
|
|
9138
|
+
const responseCreditPolicyRevisions = /* @__PURE__ */ new Set();
|
|
9034
9139
|
const responseCountBeforeStream = modelResponseState.responseCount;
|
|
9035
9140
|
eventing.batcher = null;
|
|
9036
9141
|
let currentToolBatchCallIds = /* @__PURE__ */ new Set();
|
|
@@ -9200,7 +9305,10 @@ async function runTurnStreamAttempt(deps) {
|
|
|
9200
9305
|
attempt.modelRequestStarted = true;
|
|
9201
9306
|
return await runtime.runStream(agent, runInput, eventing.modelRunSettings, {
|
|
9202
9307
|
beforeModelRequest: modelCallAdmission.beforeModelRequest,
|
|
9203
|
-
onModelResponse:
|
|
9308
|
+
onModelResponse: (event) => {
|
|
9309
|
+
responseCreditPolicyRevisions.add(creditPolicyRevision);
|
|
9310
|
+
return modelCallAdmission.onModelResponse(event);
|
|
9311
|
+
},
|
|
9204
9312
|
signal: runtimeCancellationSignal,
|
|
9205
9313
|
sandboxEnvironment,
|
|
9206
9314
|
onModelVisibleContext: async (snapshot) => {
|
|
@@ -9390,6 +9498,7 @@ async function runTurnStreamAttempt(deps) {
|
|
|
9390
9498
|
}
|
|
9391
9499
|
const generatedImageReceipt = generatedImage ? await media.retainNativeGeneratedImage(generatedImage) : null;
|
|
9392
9500
|
const responseResult = await processModelResponseTerminalEvent({
|
|
9501
|
+
creditPolicyRevision,
|
|
9393
9502
|
event: next.value,
|
|
9394
9503
|
state: modelResponseState,
|
|
9395
9504
|
dispatchId: modelUsageDispatchId,
|
|
@@ -9452,6 +9561,15 @@ async function runTurnStreamAttempt(deps) {
|
|
|
9452
9561
|
]);
|
|
9453
9562
|
attempt.providerRecoveryCount = 0;
|
|
9454
9563
|
}
|
|
9564
|
+
if (attempt.providerRecoveryObservation) {
|
|
9565
|
+
recordProviderRecoveryOutcome(observability, {
|
|
9566
|
+
route: attempt.modelMetricRoute,
|
|
9567
|
+
cause: attempt.providerRecoveryObservation.cause,
|
|
9568
|
+
outcome: "recovered",
|
|
9569
|
+
elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt
|
|
9570
|
+
});
|
|
9571
|
+
attempt.providerRecoveryObservation = void 0;
|
|
9572
|
+
}
|
|
9455
9573
|
const rawStreamHistory = eventing.stream.state.history;
|
|
9456
9574
|
if (Array.isArray(rawStreamHistory)) {
|
|
9457
9575
|
await media.retainNativeGeneratedImagesFromHistory(
|
|
@@ -9464,18 +9582,6 @@ async function runTurnStreamAttempt(deps) {
|
|
|
9464
9582
|
await historySink.reconcileConversationTruth();
|
|
9465
9583
|
turnLifecycleMetricsFor(observability).progress({ attemptId: input.attemptId });
|
|
9466
9584
|
modelCheckpointMemoryCollector.schedule(observability);
|
|
9467
|
-
await ensureRunAllowedBetweenModelCalls({
|
|
9468
|
-
settings,
|
|
9469
|
-
db,
|
|
9470
|
-
accountId: input.accountId,
|
|
9471
|
-
workspaceId: input.workspaceId,
|
|
9472
|
-
isExternallyBilledTurn: billingState.isExternallyBilledTurn,
|
|
9473
|
-
entitlements,
|
|
9474
|
-
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
9475
|
-
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
9476
|
-
initiatingHumanSubjectId: turn.initiatingHumanSubjectId,
|
|
9477
|
-
serializedRunState: () => media.compactMediaRunState(String(eventing.stream.state.toString()))
|
|
9478
|
-
});
|
|
9479
9585
|
}
|
|
9480
9586
|
modelCallAdmission.settle(next.value);
|
|
9481
9587
|
const durableSdkEvent = generatedImageReceipt ? compactGeneratedImageSdkEvent(next.value, generatedImageReceipt) : next.value;
|
|
@@ -9769,6 +9875,12 @@ async function runTurnStreamAttempt(deps) {
|
|
|
9769
9875
|
if (!streamSawPerResponseUsage) {
|
|
9770
9876
|
const aggregateUsage = eventing.stream.state.usage;
|
|
9771
9877
|
const normalizedAggregateUsage = normalizeModelCallUsage2(aggregateUsage);
|
|
9878
|
+
const aggregatePolicyRevision = aggregateCreditPolicyRevision({
|
|
9879
|
+
responseRevisions: responseCreditPolicyRevisions,
|
|
9880
|
+
lastAdmittedRevision: creditPolicyRevision,
|
|
9881
|
+
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
9882
|
+
totalTokens: normalizedAggregateUsage.totalTokens
|
|
9883
|
+
});
|
|
9772
9884
|
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
9773
9885
|
const aggregateSourceKey = modelUsageSourceKey({
|
|
9774
9886
|
responseId: null,
|
|
@@ -9789,6 +9901,7 @@ async function runTurnStreamAttempt(deps) {
|
|
|
9789
9901
|
leaseLostMessage: "Provider credential lease expired during the active turn",
|
|
9790
9902
|
recordUsage: async () => {
|
|
9791
9903
|
const billing = await recordModelUsageAndDebitCredits(settings, db, {
|
|
9904
|
+
creditPolicyRevision: aggregatePolicyRevision,
|
|
9792
9905
|
accountId: input.accountId,
|
|
9793
9906
|
workspaceId: input.workspaceId,
|
|
9794
9907
|
sessionId: input.sessionId,
|
|
@@ -10125,6 +10238,7 @@ async function runTurnStreamAttempt(deps) {
|
|
|
10125
10238
|
turnExecutionPolicy.providerId,
|
|
10126
10239
|
turnExecutionPolicy.latencyMode
|
|
10127
10240
|
);
|
|
10241
|
+
titleCreditPolicyRevision = creditPolicyRevision;
|
|
10128
10242
|
parallelSessionTitle = startParallelSessionTitleGeneration({
|
|
10129
10243
|
signal: runtimeCancellationSignal,
|
|
10130
10244
|
generate: async (signal) => await withSessionTitleProviderRequestContext(
|
|
@@ -10330,7 +10444,7 @@ import {
|
|
|
10330
10444
|
requireSession
|
|
10331
10445
|
} from "@opengeni/db";
|
|
10332
10446
|
import { appendAndPublishTurnEventsFenced as appendAndPublishTurnEventsFenced2, publishDurableSessionEvents as publishDurableSessionEvents4 } from "@opengeni/events";
|
|
10333
|
-
import { linkCurrentSpanToAdmission, turnExecutionTelemetryKey } from "@opengeni/observability";
|
|
10447
|
+
import { linkCurrentSpanToAdmission, turnExecutionTelemetryKey as turnExecutionTelemetryKey2 } from "@opengeni/observability";
|
|
10334
10448
|
import {
|
|
10335
10449
|
assertTurnExecutionPolicyMatchesConfigV1,
|
|
10336
10450
|
settingsForAcceptedSubscriptionTurn,
|
|
@@ -10684,6 +10798,7 @@ async function claimTurnAttempt(deps) {
|
|
|
10684
10798
|
attempt.dispatchId = dispatchId;
|
|
10685
10799
|
attempt.executionGeneration = turn.executionGeneration;
|
|
10686
10800
|
attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
10801
|
+
attempt.providerRecoveryObservation = readProviderRecoveryObservation(turn.metadata ?? {});
|
|
10687
10802
|
const authRecovery = turn.metadata?.claudeAuthRecovery;
|
|
10688
10803
|
attempt.claudeAuthRecovery = authRecovery && typeof authRecovery === "object" && "credentialId" in authRecovery && typeof authRecovery.credentialId === "string" && "credentialVersion" in authRecovery && typeof authRecovery.credentialVersion === "number" && Number.isSafeInteger(authRecovery.credentialVersion) && authRecovery.credentialVersion > 0 ? {
|
|
10689
10804
|
credentialId: authRecovery.credentialId,
|
|
@@ -10793,6 +10908,7 @@ async function claimTurnAttempt(deps) {
|
|
|
10793
10908
|
latencyMode: turn.latencyMode
|
|
10794
10909
|
}
|
|
10795
10910
|
);
|
|
10911
|
+
capabilitySettings = { ...capabilitySettings, openaiMaxRetries: 0 };
|
|
10796
10912
|
runtime.configure(capabilitySettings);
|
|
10797
10913
|
const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
|
|
10798
10914
|
capabilitySettings,
|
|
@@ -10804,6 +10920,10 @@ async function claimTurnAttempt(deps) {
|
|
|
10804
10920
|
}
|
|
10805
10921
|
);
|
|
10806
10922
|
const turnExecutionPolicy = verifiedExecutionPolicy.policy;
|
|
10923
|
+
attempt.modelMetricRoute = {
|
|
10924
|
+
provider: turnExecutionPolicy.providerId,
|
|
10925
|
+
model: turnExecutionPolicy.productModelId
|
|
10926
|
+
};
|
|
10807
10927
|
assertSessionAllowsProductModel(session, turnExecutionPolicy.productModelId);
|
|
10808
10928
|
const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
|
|
10809
10929
|
billingState.isExternallyBilledTurn = billingIdentity.externallyBilled;
|
|
@@ -10857,7 +10977,8 @@ async function claimTurnAttempt(deps) {
|
|
|
10857
10977
|
entitlements,
|
|
10858
10978
|
billingState.chargesOpenGeniCredits,
|
|
10859
10979
|
billingState.countsTowardTokenCap,
|
|
10860
|
-
turn.initiatingHumanSubjectId
|
|
10980
|
+
turn.initiatingHumanSubjectId,
|
|
10981
|
+
turnExecutionPolicy.productModelId
|
|
10861
10982
|
),
|
|
10862
10983
|
cancellationSignal,
|
|
10863
10984
|
void 0
|
|
@@ -11007,7 +11128,7 @@ async function claimTurnAttempt(deps) {
|
|
|
11007
11128
|
throwIfTurnCancelled();
|
|
11008
11129
|
recordTurnStartupPhase(observability, {
|
|
11009
11130
|
phase: "claim_and_policy",
|
|
11010
|
-
executionCorrelationId:
|
|
11131
|
+
executionCorrelationId: turnExecutionTelemetryKey2(
|
|
11011
11132
|
input.workspaceId,
|
|
11012
11133
|
input.sessionId,
|
|
11013
11134
|
input.attemptId
|
|
@@ -12745,9 +12866,11 @@ async function prepareCompaction(deps) {
|
|
|
12745
12866
|
};
|
|
12746
12867
|
const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel) ? input.sessionId : void 0;
|
|
12747
12868
|
const compactionUsageState = createCompactionModelUsageEventState(claimedModelUsageSourceKeys);
|
|
12869
|
+
let compactionCreditPolicyRevision;
|
|
12748
12870
|
const recordCompactionUsage = async (usage) => {
|
|
12749
12871
|
await processCompactionModelUsageEvent({
|
|
12750
12872
|
usage,
|
|
12873
|
+
creditPolicyRevision: compactionCreditPolicyRevision,
|
|
12751
12874
|
state: compactionUsageState,
|
|
12752
12875
|
dispatchId: modelUsageDispatchId,
|
|
12753
12876
|
settings,
|
|
@@ -12775,7 +12898,7 @@ async function prepareCompaction(deps) {
|
|
|
12775
12898
|
});
|
|
12776
12899
|
};
|
|
12777
12900
|
const compactionSummarizerFor = (systemInstructions) => {
|
|
12778
|
-
const
|
|
12901
|
+
const summarizeModel = resolvedModel ? (s, m) => withProviderRequestContext(
|
|
12779
12902
|
() => summarizeContextForCompaction(s, m, {
|
|
12780
12903
|
client: resolvedModel.client,
|
|
12781
12904
|
provider: resolvedModel.provider,
|
|
@@ -12796,6 +12919,21 @@ async function prepareCompaction(deps) {
|
|
|
12796
12919
|
...systemInstructions ? { systemInstructions } : {},
|
|
12797
12920
|
...promptCacheKey ? { promptCacheKey } : {}
|
|
12798
12921
|
});
|
|
12922
|
+
const summarize = async (s, m) => {
|
|
12923
|
+
compactionCreditPolicyRevision = await ensureRunAllowedBetweenModelCalls({
|
|
12924
|
+
settings: s,
|
|
12925
|
+
db,
|
|
12926
|
+
accountId: input.accountId,
|
|
12927
|
+
workspaceId: input.workspaceId,
|
|
12928
|
+
modelId: resolvedModel?.configured.id ?? turn.model,
|
|
12929
|
+
isExternallyBilledTurn: billingState.isExternallyBilledTurn,
|
|
12930
|
+
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
12931
|
+
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
12932
|
+
initiatingHumanSubjectId: turn.initiatingHumanSubjectId,
|
|
12933
|
+
entitlements: deps.entitlements
|
|
12934
|
+
});
|
|
12935
|
+
return await summarizeModel(s, m);
|
|
12936
|
+
};
|
|
12799
12937
|
summarize.estimatePrefixTokens = () => {
|
|
12800
12938
|
if (resolvedModel?.provider.api === "chat") {
|
|
12801
12939
|
return estimateSerializedValueTokens(systemInstructions ?? "");
|
|
@@ -12822,6 +12960,18 @@ async function prepareCompaction(deps) {
|
|
|
12822
12960
|
await publishDurableSessionEvents6(bus, input.workspaceId, input.sessionId, events);
|
|
12823
12961
|
};
|
|
12824
12962
|
const publishCompactionOutcomeEvents = async (events) => {
|
|
12963
|
+
if (events.some((event) => event.type === "session.context.compacted")) {
|
|
12964
|
+
attempt.providerRecoveryCount = 0;
|
|
12965
|
+
if (attempt.providerRecoveryObservation) {
|
|
12966
|
+
recordProviderRecoveryOutcome(observability, {
|
|
12967
|
+
route: attempt.modelMetricRoute,
|
|
12968
|
+
cause: attempt.providerRecoveryObservation.cause,
|
|
12969
|
+
outcome: "recovered",
|
|
12970
|
+
elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt
|
|
12971
|
+
});
|
|
12972
|
+
attempt.providerRecoveryObservation = void 0;
|
|
12973
|
+
}
|
|
12974
|
+
}
|
|
12825
12975
|
await publishDurableSessionEvents6(
|
|
12826
12976
|
bus,
|
|
12827
12977
|
input.workspaceId,
|
|
@@ -33172,6 +33322,7 @@ import {
|
|
|
33172
33322
|
import { SandboxChannelAService as SandboxChannelAService3 } from "@opengeni/runtime/sandbox";
|
|
33173
33323
|
import { sandboxRunAs as sandboxRunAs4 } from "@opengeni/runtime";
|
|
33174
33324
|
import {
|
|
33325
|
+
bundledSkillSelectionForAgentConfig,
|
|
33175
33326
|
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS as DEFAULT_FIRST_PARTY_MCP_PERMISSIONS2,
|
|
33176
33327
|
resolveAgentToolFamilies
|
|
33177
33328
|
} from "@opengeni/contracts";
|
|
@@ -34928,7 +35079,9 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
34928
35079
|
session.firstPartyMcpPermissions,
|
|
34929
35080
|
linkedAuthority
|
|
34930
35081
|
);
|
|
34931
|
-
const toolFamilies = resolveAgentToolFamilies(session.agent
|
|
35082
|
+
const toolFamilies = resolveAgentToolFamilies(session.agent, {
|
|
35083
|
+
sandboxAttached: (activeSandboxBackend ?? groupBoxBackend) !== "none"
|
|
35084
|
+
});
|
|
34932
35085
|
const selectedFirstPartyMcpTools = toolFamilies.firstPartyTools(
|
|
34933
35086
|
allowedFirstPartyMcpToolsForSession(runSettings, session.firstPartyMcpTools)
|
|
34934
35087
|
);
|
|
@@ -34988,7 +35141,9 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
34988
35141
|
})
|
|
34989
35142
|
]);
|
|
34990
35143
|
const bundledSkills = loadConfiguredBundledSkills({
|
|
34991
|
-
|
|
35144
|
+
// Rows that could not freeze the "none" default at create (scheduled
|
|
35145
|
+
// generated sessions, pre-existing rows) get the same rule here.
|
|
35146
|
+
bundledSkillIds: bundledSkillSelectionForAgentConfig(session.bundledSkillIds, session.agent),
|
|
34992
35147
|
firstPartyTools: selectedFirstPartyMcpTools,
|
|
34993
35148
|
videoGenerationEnabled: skillConfiguration.defaultModelId !== null && skillConfiguration.enabledModelIds.length > 0
|
|
34994
35149
|
});
|
|
@@ -38394,6 +38549,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
38394
38549
|
)
|
|
38395
38550
|
);
|
|
38396
38551
|
const compactionPrep = await prepareCompaction({
|
|
38552
|
+
entitlements,
|
|
38397
38553
|
input,
|
|
38398
38554
|
settings: capabilitySettings,
|
|
38399
38555
|
db,
|
|
@@ -39205,7 +39361,7 @@ These are publication receipts for the human's earlier answer. Do not call the r
|
|
|
39205
39361
|
const servicesStartedAt = performance.now();
|
|
39206
39362
|
const resolvedServices = await services();
|
|
39207
39363
|
const servicesDurationSeconds = (performance.now() - servicesStartedAt) / 1e3;
|
|
39208
|
-
const correlationId =
|
|
39364
|
+
const correlationId = turnExecutionTelemetryKey3(
|
|
39209
39365
|
input.workspaceId,
|
|
39210
39366
|
input.sessionId,
|
|
39211
39367
|
input.attemptId
|
|
@@ -40351,4 +40507,4 @@ export {
|
|
|
40351
40507
|
createTurnActivities,
|
|
40352
40508
|
createTurnActivitiesFromServices
|
|
40353
40509
|
};
|
|
40354
|
-
//# sourceMappingURL=activities-turn-
|
|
40510
|
+
//# sourceMappingURL=activities-turn-BMY3CDKL.js.map
|