@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +10 -2
- package/dist/activities/agent-turn/agent-build.d.ts +1 -4
- package/dist/activities/agent-turn/claim.d.ts +3 -4
- package/dist/activities/agent-turn/codex.d.ts +8 -7
- package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
- package/dist/activities/agent-turn/errors.d.ts +13 -0
- package/dist/activities/agent-turn/finalization.d.ts +1 -3
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +10 -1
- package/dist/activities/agent-turn/quiescence.d.ts +39 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
- package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
- package/dist/activities/agent-turn/turn-context.d.ts +3 -8
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
- package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +2 -1
- package/dist/activities/codex-rotation.d.ts +2 -0
- package/dist/activities/goals.d.ts +11 -0
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/session-state.d.ts +2 -1
- package/dist/activities/types.d.ts +8 -0
- package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
- package/dist/activities-control-SGU2WEVB.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
- package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
- package/dist/chunk-S2QVRVJW.js.map +1 -0
- package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
- package/dist/chunk-TJIPCHMP.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +69 -6
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +29 -0
- package/dist/personal-github-git-credentials.d.ts +1 -0
- package/dist/unhandled-rejection-boundary.d.ts +29 -0
- package/dist/workflow-bundle.js +26 -3
- package/dist/workflows/activities.d.ts +5 -0
- package/dist/workflows/automations.d.ts +5 -0
- package/package.json +18 -18
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +28 -6
- package/src/activities/agent-turn/agent-build.ts +15 -31
- package/src/activities/agent-turn/claim.ts +53 -72
- package/src/activities/agent-turn/codex.ts +21 -12
- package/src/activities/agent-turn/compaction-prep.ts +52 -1
- package/src/activities/agent-turn/errors.ts +78 -0
- package/src/activities/agent-turn/failure-settlement.ts +30 -19
- package/src/activities/agent-turn/finalization.ts +24 -21
- package/src/activities/agent-turn/history.ts +5 -1
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +88 -42
- package/src/activities/agent-turn/quiescence.ts +86 -0
- package/src/activities/agent-turn/run-credentials.ts +12 -1
- package/src/activities/agent-turn/run.ts +20 -14
- package/src/activities/agent-turn/sandbox-establish.ts +46 -11
- package/src/activities/agent-turn/sandbox-route.ts +21 -25
- package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
- package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
- package/src/activities/agent-turn/stream-attempt.ts +106 -14
- package/src/activities/agent-turn/tool-environment.ts +72 -14
- package/src/activities/agent-turn/tool-policy.ts +7 -6
- package/src/activities/agent-turn/turn-context.ts +12 -14
- package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
- package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +72 -2
- package/src/activities/codex-capacity.ts +13 -5
- package/src/activities/codex-rotation.ts +17 -0
- package/src/activities/goals.ts +117 -35
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/retained-screenshots.ts +104 -21
- package/src/activities/sandbox-lease.ts +7 -4
- package/src/activities/scheduled-tasks.ts +248 -30
- package/src/activities/session-state.ts +8 -1
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +6 -0
- package/src/activities.ts +1 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +8 -4
- package/src/http.ts +3 -0
- package/src/index.ts +38 -4
- package/src/observability-metrics.ts +202 -0
- package/src/personal-github-git-credentials.ts +45 -4
- package/src/sandbox-resume.ts +1 -1
- package/src/sandbox-routing.ts +7 -4
- package/src/unhandled-rejection-boundary.ts +73 -0
- package/src/workflows/activities.ts +15 -0
- package/src/workflows/automations.ts +21 -2
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
- package/dist/activities/recording.d.ts +0 -90
- package/dist/activities-control-VZIIV7RX.js.map +0 -1
- package/dist/activities-turn-5RGQDBFV.js.map +0 -1
- package/dist/chunk-NAVSAZ56.js.map +0 -1
- package/dist/chunk-O4HDX5SC.js.map +0 -1
- package/src/activities/recording.ts +0 -279
|
@@ -15,11 +15,13 @@ import {
|
|
|
15
15
|
} from "@opengeni/runtime";
|
|
16
16
|
import {
|
|
17
17
|
calculateGatewayReportedCostBreakdown,
|
|
18
|
+
calculateGatewayReportedProviderCostMicros,
|
|
18
19
|
calculateModelUsageCostBreakdown,
|
|
19
20
|
configuredModelPricingSchedules,
|
|
21
|
+
resolveModelProvider,
|
|
20
22
|
responseSatisfiesLatencyMode,
|
|
21
|
-
OPENGENI_GATEWAY_MODELS,
|
|
22
23
|
OPENGENI_GATEWAY_PROVIDER_ID,
|
|
24
|
+
WORKSPACE_GATEWAY_PROVIDER_ID,
|
|
23
25
|
type ModelUsageInput,
|
|
24
26
|
type ModelProviderApi,
|
|
25
27
|
type Settings,
|
|
@@ -191,6 +193,8 @@ export async function processModelResponseTerminalEvent(input: {
|
|
|
191
193
|
latencyMode?: LatencyMode;
|
|
192
194
|
metricProvider: string;
|
|
193
195
|
externallyBilled: boolean;
|
|
196
|
+
chargesOpenGeniCredits?: boolean;
|
|
197
|
+
countsTowardTokenCap?: boolean;
|
|
194
198
|
servingCredentialId: string | null;
|
|
195
199
|
priorSessionCredentialId: string | null;
|
|
196
200
|
emittedSourceKeys: Set<string>;
|
|
@@ -251,9 +255,14 @@ export async function processModelResponseTerminalEvent(input: {
|
|
|
251
255
|
turnAttemptId: input.turnAttemptId,
|
|
252
256
|
model: input.model,
|
|
253
257
|
externallyBilled: input.externallyBilled,
|
|
258
|
+
...(input.chargesOpenGeniCredits !== undefined
|
|
259
|
+
? { chargesOpenGeniCredits: input.chargesOpenGeniCredits }
|
|
260
|
+
: {}),
|
|
261
|
+
...(input.countsTowardTokenCap !== undefined
|
|
262
|
+
? { countsTowardTokenCap: input.countsTowardTokenCap }
|
|
263
|
+
: {}),
|
|
254
264
|
usage: responseUsage.usage,
|
|
255
265
|
normalizedUsage,
|
|
256
|
-
gatewayManaged: input.provider === OPENGENI_GATEWAY_PROVIDER_ID,
|
|
257
266
|
gatewayBilling: responseUsage.gatewayBilling,
|
|
258
267
|
sourceKey,
|
|
259
268
|
...(input.latencyMode ? { latencyMode: input.latencyMode } : {}),
|
|
@@ -272,6 +281,8 @@ export async function processModelResponseTerminalEvent(input: {
|
|
|
272
281
|
sourceKey,
|
|
273
282
|
usage: responseUsage,
|
|
274
283
|
normalizedUsage,
|
|
284
|
+
...(billing ? { billingPath: billing.billingPath } : {}),
|
|
285
|
+
...(billing?.upstreamProvider ? { upstreamProvider: billing.upstreamProvider } : {}),
|
|
275
286
|
servingAccountHash: accountContext.servingAccountHash,
|
|
276
287
|
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
277
288
|
emittedSourceKeys: input.emittedSourceKeys,
|
|
@@ -345,6 +356,8 @@ export async function processCompactionModelUsageEvent(input: {
|
|
|
345
356
|
providerApi: ModelProviderApi;
|
|
346
357
|
model: string;
|
|
347
358
|
externallyBilled: boolean;
|
|
359
|
+
chargesOpenGeniCredits?: boolean;
|
|
360
|
+
countsTowardTokenCap?: boolean;
|
|
348
361
|
servingCredentialId: string | null;
|
|
349
362
|
priorSessionCredentialId: string | null;
|
|
350
363
|
emittedSourceKeys: Set<string>;
|
|
@@ -388,9 +401,14 @@ export async function processCompactionModelUsageEvent(input: {
|
|
|
388
401
|
turnAttemptId: input.turnAttemptId,
|
|
389
402
|
model: input.model,
|
|
390
403
|
externallyBilled: input.externallyBilled,
|
|
404
|
+
...(input.chargesOpenGeniCredits !== undefined
|
|
405
|
+
? { chargesOpenGeniCredits: input.chargesOpenGeniCredits }
|
|
406
|
+
: {}),
|
|
407
|
+
...(input.countsTowardTokenCap !== undefined
|
|
408
|
+
? { countsTowardTokenCap: input.countsTowardTokenCap }
|
|
409
|
+
: {}),
|
|
391
410
|
usage: input.usage.usage,
|
|
392
411
|
normalizedUsage,
|
|
393
|
-
gatewayManaged: input.provider === OPENGENI_GATEWAY_PROVIDER_ID,
|
|
394
412
|
gatewayBilling: input.usage.gatewayBilling,
|
|
395
413
|
sourceKey,
|
|
396
414
|
observability: input.observability,
|
|
@@ -408,6 +426,8 @@ export async function processCompactionModelUsageEvent(input: {
|
|
|
408
426
|
sourceKey,
|
|
409
427
|
usage: input.usage,
|
|
410
428
|
normalizedUsage,
|
|
429
|
+
...(billing ? { billingPath: billing.billingPath } : {}),
|
|
430
|
+
...(billing?.upstreamProvider ? { upstreamProvider: billing.upstreamProvider } : {}),
|
|
411
431
|
servingAccountHash: accountContext.servingAccountHash,
|
|
412
432
|
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
413
433
|
emittedSourceKeys: input.emittedSourceKeys,
|
|
@@ -449,6 +469,10 @@ export async function emitModelCallUsage(input: {
|
|
|
449
469
|
sourceKey: string;
|
|
450
470
|
usage: ModelResponseUsage | { usage?: unknown | null } | null;
|
|
451
471
|
normalizedUsage?: ModelCallUsageNormalization;
|
|
472
|
+
/** Accepted billing authority persisted for Insights repair after a soft fact-write failure. */
|
|
473
|
+
billingPath?: ModelUsageBillingRecord["billingPath"];
|
|
474
|
+
/** Validated Gateway endpoint provider persisted for exact Insights repair. */
|
|
475
|
+
upstreamProvider?: string;
|
|
452
476
|
// Prompt-cache research dimensions (log-only; NEVER on a metric label or a
|
|
453
477
|
// durable event). The opaque serving-account tag and whether it changed since
|
|
454
478
|
// the session's previous call — the account-switch hypothesis for cache misses.
|
|
@@ -480,6 +504,8 @@ export async function emitModelCallUsage(input: {
|
|
|
480
504
|
providerApi: input.providerApi,
|
|
481
505
|
model: input.model,
|
|
482
506
|
sourceKey: input.sourceKey,
|
|
507
|
+
...(input.billingPath ? { billingPath: input.billingPath } : {}),
|
|
508
|
+
...(input.upstreamProvider ? { upstreamProvider: input.upstreamProvider } : {}),
|
|
483
509
|
...telemetry,
|
|
484
510
|
},
|
|
485
511
|
},
|
|
@@ -572,7 +598,8 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
572
598
|
turnAttemptId: string;
|
|
573
599
|
model: string;
|
|
574
600
|
externallyBilled: boolean;
|
|
575
|
-
|
|
601
|
+
chargesOpenGeniCredits?: boolean;
|
|
602
|
+
countsTowardTokenCap?: boolean;
|
|
576
603
|
gatewayBilling?: ModelResponseUsage["gatewayBilling"];
|
|
577
604
|
usage?: ModelUsageInput | ModelCallUsageInput | null;
|
|
578
605
|
normalizedUsage?: ModelCallUsageNormalization;
|
|
@@ -589,19 +616,35 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
589
616
|
const inputTokens = sanitizedUsage.inputTokens ?? 0;
|
|
590
617
|
const outputTokens = sanitizedUsage.outputTokens ?? 0;
|
|
591
618
|
const totalTokens = sanitizedUsage.totalTokens ?? 0;
|
|
592
|
-
const
|
|
619
|
+
const chargesOpenGeniCredits = input.chargesOpenGeniCredits ?? !input.externallyBilled;
|
|
620
|
+
const countsTowardTokenCap = input.countsTowardTokenCap ?? !input.externallyBilled;
|
|
621
|
+
const resolvedGatewayModel = input.gatewayBilling
|
|
622
|
+
? resolveModelProvider(settings, input.model)
|
|
623
|
+
: undefined;
|
|
624
|
+
const gatewayProviderId = resolvedGatewayModel?.provider.id;
|
|
625
|
+
const gatewayBilling =
|
|
626
|
+
gatewayProviderId === OPENGENI_GATEWAY_PROVIDER_ID ||
|
|
627
|
+
gatewayProviderId === WORKSPACE_GATEWAY_PROVIDER_ID
|
|
628
|
+
? input.gatewayBilling
|
|
629
|
+
: undefined;
|
|
630
|
+
const allowedProviders = resolvedGatewayModel?.model.requestPolicy?.gateway.only;
|
|
631
|
+
const unpinnedWorkspaceGatewayModel =
|
|
632
|
+
gatewayProviderId === WORKSPACE_GATEWAY_PROVIDER_ID && allowedProviders === undefined;
|
|
593
633
|
if (gatewayBilling) {
|
|
594
|
-
const gatewayModel = Object.values(OPENGENI_GATEWAY_MODELS).find(
|
|
595
|
-
(candidate) => candidate.productId === input.model,
|
|
596
|
-
);
|
|
597
634
|
if (
|
|
598
|
-
!
|
|
599
|
-
!
|
|
635
|
+
!unpinnedWorkspaceGatewayModel &&
|
|
636
|
+
(!allowedProviders ||
|
|
637
|
+
!(allowedProviders as readonly string[]).includes(gatewayBilling.finalProvider))
|
|
600
638
|
) {
|
|
601
639
|
throw new Error(
|
|
602
640
|
`AI Gateway reported unapproved provider ${gatewayBilling.finalProvider} for ${input.model}`,
|
|
603
641
|
);
|
|
604
642
|
}
|
|
643
|
+
if (unpinnedWorkspaceGatewayModel && chargesOpenGeniCredits) {
|
|
644
|
+
throw new Error(
|
|
645
|
+
`Workspace Gateway custom model ${input.model} cannot charge OpenGeni credits without pinned pricing`,
|
|
646
|
+
);
|
|
647
|
+
}
|
|
605
648
|
}
|
|
606
649
|
const pricingSchedules = configuredModelPricingSchedules(settings);
|
|
607
650
|
const configuredPricingModel = pricingSchedules[input.model]
|
|
@@ -610,12 +653,19 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
610
653
|
? input.model.slice("codex/".length)
|
|
611
654
|
: null;
|
|
612
655
|
const pricingBreakdown = gatewayBilling
|
|
613
|
-
?
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
656
|
+
? unpinnedWorkspaceGatewayModel
|
|
657
|
+
? {
|
|
658
|
+
providerCostMicros: calculateGatewayReportedProviderCostMicros(
|
|
659
|
+
gatewayBilling.inferenceCostUsd,
|
|
660
|
+
),
|
|
661
|
+
creditCostMicros: 0,
|
|
662
|
+
}
|
|
663
|
+
: calculateGatewayReportedCostBreakdown(
|
|
664
|
+
settings,
|
|
665
|
+
input.model,
|
|
666
|
+
gatewayBilling.inferenceCostUsd,
|
|
667
|
+
{ inputTokens },
|
|
668
|
+
)
|
|
619
669
|
: configuredPricingModel
|
|
620
670
|
? calculateModelUsageCostBreakdown(settings, configuredPricingModel, sanitizedUsage, {
|
|
621
671
|
latencyMode: input.latencyMode ?? "standard",
|
|
@@ -634,17 +684,27 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
634
684
|
: estimatedProviderCostMicros !== null
|
|
635
685
|
? ("configured_list_price" as const)
|
|
636
686
|
: null;
|
|
637
|
-
//
|
|
638
|
-
//
|
|
639
|
-
//
|
|
640
|
-
//
|
|
641
|
-
//
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
687
|
+
// Provider settlement and workspace-facing cost are separate. Externally
|
|
688
|
+
// metered subscription/workspace turns remain exempt from the OpenGeni token
|
|
689
|
+
// cap, while a deployment-funded free model still records model.tokens. Every
|
|
690
|
+
// non-credit path records a zero-cost marker and never consults pricing for a
|
|
691
|
+
// debit.
|
|
692
|
+
if (countsTowardTokenCap && totalTokens > 0) {
|
|
693
|
+
await recordUsageEvent(db, {
|
|
694
|
+
accountId: input.accountId,
|
|
695
|
+
workspaceId: input.workspaceId,
|
|
696
|
+
eventType: "model.tokens",
|
|
697
|
+
quantity: totalTokens,
|
|
698
|
+
unit: "tokens",
|
|
699
|
+
sourceResourceType: "model_response",
|
|
700
|
+
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
701
|
+
sessionId: input.sessionId,
|
|
702
|
+
turnId: input.turnId,
|
|
703
|
+
turnAttemptId: input.turnAttemptId,
|
|
704
|
+
idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`,
|
|
705
|
+
});
|
|
706
|
+
}
|
|
707
|
+
if (!chargesOpenGeniCredits) {
|
|
648
708
|
await recordUsageEvent(db, {
|
|
649
709
|
accountId: input.accountId,
|
|
650
710
|
workspaceId: input.workspaceId,
|
|
@@ -664,23 +724,9 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
664
724
|
estimatedProviderCostMicros,
|
|
665
725
|
pricingSource,
|
|
666
726
|
normalizedUsage,
|
|
727
|
+
...(gatewayBilling ? { upstreamProvider: gatewayBilling.finalProvider } : {}),
|
|
667
728
|
};
|
|
668
729
|
}
|
|
669
|
-
if (totalTokens > 0) {
|
|
670
|
-
await recordUsageEvent(db, {
|
|
671
|
-
accountId: input.accountId,
|
|
672
|
-
workspaceId: input.workspaceId,
|
|
673
|
-
eventType: "model.tokens",
|
|
674
|
-
quantity: totalTokens,
|
|
675
|
-
unit: "tokens",
|
|
676
|
-
sourceResourceType: "model_response",
|
|
677
|
-
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
678
|
-
sessionId: input.sessionId,
|
|
679
|
-
turnId: input.turnId,
|
|
680
|
-
turnAttemptId: input.turnAttemptId,
|
|
681
|
-
idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`,
|
|
682
|
-
});
|
|
683
|
-
}
|
|
684
730
|
const shouldDebit = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
685
731
|
if (!shouldDebit || (totalTokens === 0 && !gatewayBilling)) {
|
|
686
732
|
return {
|
|
@@ -79,6 +79,32 @@ export function assertSessionAttemptQuiescenceRecoveryDurable(input: {
|
|
|
79
79
|
|
|
80
80
|
export const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
|
|
81
81
|
export const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
|
|
82
|
+
export const TURN_QUIESCENCE_WATCHDOG_MS = 5 * 60_000;
|
|
83
|
+
|
|
84
|
+
/** A fenced activity must never heartbeat forever while its physical writer
|
|
85
|
+
* drain is stuck. The last-resort worker exit stops every in-process writer;
|
|
86
|
+
* Kubernetes replaces the pod and the workflow's heartbeat-lease reconciler
|
|
87
|
+
* then commits the exact quiescence receipt. */
|
|
88
|
+
export function armTurnQuiescenceWatchdog(input: {
|
|
89
|
+
enabled: boolean;
|
|
90
|
+
timeoutMs?: number;
|
|
91
|
+
onTimeout?: () => void;
|
|
92
|
+
terminateWorker: () => void;
|
|
93
|
+
}): () => void {
|
|
94
|
+
if (!input.enabled) return () => undefined;
|
|
95
|
+
let armed = true;
|
|
96
|
+
const timer = setTimeout(() => {
|
|
97
|
+
if (!armed) return;
|
|
98
|
+
input.onTimeout?.();
|
|
99
|
+
input.terminateWorker();
|
|
100
|
+
}, input.timeoutMs ?? TURN_QUIESCENCE_WATCHDOG_MS);
|
|
101
|
+
timer.unref?.();
|
|
102
|
+
return () => {
|
|
103
|
+
if (!armed) return;
|
|
104
|
+
armed = false;
|
|
105
|
+
clearTimeout(timer);
|
|
106
|
+
};
|
|
107
|
+
}
|
|
82
108
|
|
|
83
109
|
/** Persist the authoritative receipt or durably hand the exact physical proof
|
|
84
110
|
* to Temporal. This retries signal delivery, not DB eligibility or workflow
|
|
@@ -253,6 +279,66 @@ export async function waitForTurnStreamCleanup(
|
|
|
253
279
|
await waitForTurnFinalizerStep(providerCompleted, signal);
|
|
254
280
|
}
|
|
255
281
|
|
|
282
|
+
export class IncompleteAgentStreamError extends Error {
|
|
283
|
+
readonly name = "IncompleteAgentStreamError";
|
|
284
|
+
|
|
285
|
+
constructor(message = "Agent stream ended without a successful terminal output") {
|
|
286
|
+
super(message);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Prove that SDK completion authority did not report a failure or runtime
|
|
292
|
+
* cancellation. The Agents SDK closes its ReadableStream immediately when its
|
|
293
|
+
* run signal aborts, so iterator EOF is not completion authority: the separate
|
|
294
|
+
* `completed` promise carries the eventual run failure.
|
|
295
|
+
*
|
|
296
|
+
* Only Temporal cancellation detaches cleanup. Internal runtime cancellation
|
|
297
|
+
* such as a provider-deadline sandbox rotation must escape to the turn failure
|
|
298
|
+
* settlement path, which checkpoints and redispatches the same logical turn.
|
|
299
|
+
*/
|
|
300
|
+
export async function assertSuccessfulAgentStreamCompletion(input: {
|
|
301
|
+
batcherFlush: Promise<unknown>;
|
|
302
|
+
stream: {
|
|
303
|
+
completed: Promise<void>;
|
|
304
|
+
error: unknown;
|
|
305
|
+
};
|
|
306
|
+
temporalCancellationSignal: AbortSignal | undefined;
|
|
307
|
+
runtimeCancellationSignal: AbortSignal;
|
|
308
|
+
}): Promise<void> {
|
|
309
|
+
await waitForTurnStreamCleanup(
|
|
310
|
+
input.batcherFlush,
|
|
311
|
+
input.stream.completed,
|
|
312
|
+
input.temporalCancellationSignal,
|
|
313
|
+
);
|
|
314
|
+
|
|
315
|
+
if (input.runtimeCancellationSignal.aborted) {
|
|
316
|
+
throw (
|
|
317
|
+
input.runtimeCancellationSignal.reason ??
|
|
318
|
+
new IncompleteAgentStreamError("Agent stream was cancelled without a terminal reason")
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
if (input.stream.error !== null && input.stream.error !== undefined) {
|
|
322
|
+
throw input.stream.error;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/** Call after any more-specific empty-stream recovery has had priority. */
|
|
327
|
+
export function assertAgentStreamNotCancelled(cancelled: boolean): void {
|
|
328
|
+
if (cancelled) {
|
|
329
|
+
throw new IncompleteAgentStreamError("Agent stream was cancelled without a terminal result");
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** Interruption streams do not produce a normal final output. Call this only
|
|
334
|
+
* after the interruption branch has settled any required action. */
|
|
335
|
+
export function requireAgentStreamFinalOutput(finalOutput: unknown | undefined): unknown {
|
|
336
|
+
if (finalOutput === undefined) {
|
|
337
|
+
throw new IncompleteAgentStreamError();
|
|
338
|
+
}
|
|
339
|
+
return finalOutput;
|
|
340
|
+
}
|
|
341
|
+
|
|
256
342
|
/**
|
|
257
343
|
* Terminal settlement closes the active attempt before turn finalization. The
|
|
258
344
|
* ordinary workspace admission therefore correctly returns `attempt_fenced`
|
|
@@ -56,6 +56,7 @@ import {
|
|
|
56
56
|
sandboxEstablishPolicyDecision,
|
|
57
57
|
ensureTurnModalRegistryImage,
|
|
58
58
|
sandboxArtifactRuntimeAdmission,
|
|
59
|
+
sandboxSettingsForRoute,
|
|
59
60
|
} from "./sandbox-route";
|
|
60
61
|
import type { ClaimTurnOk } from "./claim";
|
|
61
62
|
import type { GovernanceModelOk } from "./governance-model";
|
|
@@ -134,6 +135,16 @@ export async function prepareRunCredentials(deps: PrepareRunCredentialsDeps) {
|
|
|
134
135
|
connectionScope,
|
|
135
136
|
runWorkspaceMutationForSandbox,
|
|
136
137
|
} = deps;
|
|
138
|
+
// A machine-home row carries selfhosted path defaults, but clearing its
|
|
139
|
+
// pointer explicitly selects the deployment-managed group. Build the stable
|
|
140
|
+
// manifest environment with that effective provider just as Channel-A and
|
|
141
|
+
// viewer readiness do; otherwise the verified managed box and the agent can
|
|
142
|
+
// disagree on backend-aware HOME/token-file paths.
|
|
143
|
+
const sandboxEnvironmentSettings = sandboxSettingsForRoute({
|
|
144
|
+
runSettings,
|
|
145
|
+
machinePrimary,
|
|
146
|
+
groupBoxBackend,
|
|
147
|
+
});
|
|
137
148
|
|
|
138
149
|
const runCredentialResolver =
|
|
139
150
|
effectiveRunCredentialBackend === "none"
|
|
@@ -321,7 +332,7 @@ export async function prepareRunCredentials(deps: PrepareRunCredentialsDeps) {
|
|
|
321
332
|
codemodeTokenExpiresAt: sandboxCodemodeTokenExpiresAt,
|
|
322
333
|
} = await waitForTurnOperation(
|
|
323
334
|
sandboxEnvironmentForRun(
|
|
324
|
-
|
|
335
|
+
sandboxEnvironmentSettings,
|
|
325
336
|
turnResources,
|
|
326
337
|
// Rig default sets merged BELOW the session set (session wins); rig-less
|
|
327
338
|
// turns pass exactly workspaceVariableSet?.values (byte-for-byte today).
|
|
@@ -45,6 +45,7 @@ import { createTurnCredentialLeases } from "./credential-leases";
|
|
|
45
45
|
import { createTurnMediaArtifacts } from "./media-artifacts";
|
|
46
46
|
import { createTurnHistorySink } from "./history-sink";
|
|
47
47
|
import { checkpointHistoryBeforeProviderDispatch } from "./provider-dispatch-barrier";
|
|
48
|
+
import { providerRecoveryCountAfterModelRequestPhase } from "./errors";
|
|
48
49
|
import { sandboxRunAs } from "@opengeni/runtime";
|
|
49
50
|
import { randomUUID } from "node:crypto";
|
|
50
51
|
import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
|
|
@@ -88,6 +89,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
88
89
|
return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
|
|
89
90
|
const {
|
|
90
91
|
settings,
|
|
92
|
+
catalogSourceSettings = settings,
|
|
91
93
|
db,
|
|
92
94
|
bus,
|
|
93
95
|
runtime,
|
|
@@ -124,7 +126,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
124
126
|
billingState,
|
|
125
127
|
sandboxState,
|
|
126
128
|
renewals,
|
|
127
|
-
recordingState,
|
|
128
129
|
eventing,
|
|
129
130
|
workspaceRefs,
|
|
130
131
|
providerTurn,
|
|
@@ -222,7 +223,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
222
223
|
activityContext,
|
|
223
224
|
sandboxRotationController,
|
|
224
225
|
sandboxState,
|
|
225
|
-
recordingState,
|
|
226
226
|
eventing,
|
|
227
227
|
attempt,
|
|
228
228
|
});
|
|
@@ -230,9 +230,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
230
230
|
requireResolvedSandboxForMutation,
|
|
231
231
|
runWorkspaceMutationForSandbox,
|
|
232
232
|
stopLeaseHeartbeat,
|
|
233
|
-
abandonActiveRecording,
|
|
234
233
|
flushRuntimeBatcher,
|
|
235
|
-
maybeStartOnTurnRecording,
|
|
236
234
|
finalizeTurnOpStreamOps,
|
|
237
235
|
} = sandboxRuntime;
|
|
238
236
|
|
|
@@ -305,10 +303,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
305
303
|
const claimed = await claimTurnAttempt({
|
|
306
304
|
input,
|
|
307
305
|
settings,
|
|
306
|
+
catalogSourceSettings,
|
|
308
307
|
db,
|
|
309
308
|
bus,
|
|
310
309
|
runtime,
|
|
311
|
-
objectStorage,
|
|
312
310
|
observability,
|
|
313
311
|
entitlements,
|
|
314
312
|
wakeSessionWorkflow,
|
|
@@ -320,13 +318,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
320
318
|
attempt,
|
|
321
319
|
billingState,
|
|
322
320
|
sandboxState,
|
|
323
|
-
recordingState,
|
|
324
321
|
eventing,
|
|
325
322
|
leases,
|
|
326
323
|
media,
|
|
327
324
|
claimedResult,
|
|
328
325
|
acknowledgeLostAttemptOwnership,
|
|
329
|
-
abandonActiveRecording,
|
|
330
326
|
});
|
|
331
327
|
if ("exit" in claimed) return claimed.exit;
|
|
332
328
|
if (!eventing.publish || !eventing.settle) {
|
|
@@ -350,6 +346,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
350
346
|
trigger,
|
|
351
347
|
humanInputResume,
|
|
352
348
|
interactionInterventionResume,
|
|
349
|
+
attachPendingUpdatesAfterOpenSuffix,
|
|
353
350
|
throwIfWorkerShuttingDown,
|
|
354
351
|
throwIfTurnCancelled,
|
|
355
352
|
opJournal,
|
|
@@ -605,6 +602,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
605
602
|
},
|
|
606
603
|
},
|
|
607
604
|
]);
|
|
605
|
+
attempt.providerRecoveryCount = providerRecoveryCountAfterModelRequestPhase(
|
|
606
|
+
attempt.providerRecoveryCount,
|
|
607
|
+
event.phase,
|
|
608
|
+
);
|
|
608
609
|
} catch (error) {
|
|
609
610
|
auditOutcome = "failed";
|
|
610
611
|
throw error;
|
|
@@ -750,6 +751,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
750
751
|
},
|
|
751
752
|
},
|
|
752
753
|
]);
|
|
754
|
+
attempt.providerRecoveryCount = providerRecoveryCountAfterModelRequestPhase(
|
|
755
|
+
attempt.providerRecoveryCount,
|
|
756
|
+
event.phase,
|
|
757
|
+
);
|
|
753
758
|
} catch (error) {
|
|
754
759
|
auditOutcome = "failed";
|
|
755
760
|
throw error;
|
|
@@ -793,7 +798,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
793
798
|
);
|
|
794
799
|
const compactionPrep = await prepareCompaction({
|
|
795
800
|
input,
|
|
796
|
-
settings,
|
|
801
|
+
settings: capabilitySettings,
|
|
797
802
|
db,
|
|
798
803
|
bus,
|
|
799
804
|
observability,
|
|
@@ -834,6 +839,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
834
839
|
compactionModeOptions,
|
|
835
840
|
compactionOnlyTurn,
|
|
836
841
|
compactionSummarizerFor,
|
|
842
|
+
settleDeferredSteerAfterCompaction,
|
|
837
843
|
} = compactionPrep.ok;
|
|
838
844
|
|
|
839
845
|
const toolPolicy = await prepareTurnToolPolicy({
|
|
@@ -1058,6 +1064,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1058
1064
|
];
|
|
1059
1065
|
const toolRuntime = await prepareTurnToolRuntime({
|
|
1060
1066
|
input,
|
|
1067
|
+
catalogSourceSettings,
|
|
1061
1068
|
db,
|
|
1062
1069
|
bus,
|
|
1063
1070
|
runtime,
|
|
@@ -1112,8 +1119,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1112
1119
|
eventing,
|
|
1113
1120
|
attempt,
|
|
1114
1121
|
sandboxState,
|
|
1115
|
-
recordingState,
|
|
1116
|
-
maybeStartOnTurnRecording,
|
|
1117
1122
|
providerTurn,
|
|
1118
1123
|
media,
|
|
1119
1124
|
leases,
|
|
@@ -1241,7 +1246,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1241
1246
|
};
|
|
1242
1247
|
const postAgentCompaction = await runPostAgentCompaction({
|
|
1243
1248
|
input,
|
|
1244
|
-
settings,
|
|
1249
|
+
settings: capabilitySettings,
|
|
1245
1250
|
db,
|
|
1246
1251
|
bus,
|
|
1247
1252
|
observability,
|
|
@@ -1278,6 +1283,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1278
1283
|
compactionModeOptions,
|
|
1279
1284
|
compactionOnlyTurn,
|
|
1280
1285
|
compactionSummarizerFor,
|
|
1286
|
+
settleDeferredSteerAfterCompaction,
|
|
1281
1287
|
agent,
|
|
1282
1288
|
});
|
|
1283
1289
|
if ("exit" in postAgentCompaction) return postAgentCompaction.exit;
|
|
@@ -1405,7 +1411,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1405
1411
|
);
|
|
1406
1412
|
return await runTurnStreamAttempt({
|
|
1407
1413
|
input,
|
|
1408
|
-
settings,
|
|
1414
|
+
settings: capabilitySettings,
|
|
1409
1415
|
db,
|
|
1410
1416
|
runtime,
|
|
1411
1417
|
objectStorage,
|
|
@@ -1456,12 +1462,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1456
1462
|
turn,
|
|
1457
1463
|
trigger,
|
|
1458
1464
|
humanInputResume,
|
|
1465
|
+
attachPendingUpdatesAfterOpenSuffix,
|
|
1459
1466
|
agent,
|
|
1460
1467
|
resolvedModel,
|
|
1461
1468
|
providerApi,
|
|
1462
1469
|
runSettings,
|
|
1463
1470
|
turnTools,
|
|
1464
1471
|
compactSummarizer,
|
|
1472
|
+
settleDeferredSteerAfterCompaction,
|
|
1465
1473
|
compactionModelHistoryProjector,
|
|
1466
1474
|
generatedImageHistoryProjector,
|
|
1467
1475
|
modelHistoryProjector,
|
|
@@ -1525,14 +1533,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1525
1533
|
attempt,
|
|
1526
1534
|
sandboxState,
|
|
1527
1535
|
renewals,
|
|
1528
|
-
recordingState,
|
|
1529
1536
|
eventing,
|
|
1530
1537
|
providerTurn,
|
|
1531
1538
|
workspaceRefs,
|
|
1532
1539
|
runWorkspaceMutationForSandbox,
|
|
1533
1540
|
requireResolvedSandboxForMutation,
|
|
1534
1541
|
stopLeaseHeartbeat,
|
|
1535
|
-
abandonActiveRecording,
|
|
1536
1542
|
turnCompletionMemoryCollector,
|
|
1537
1543
|
});
|
|
1538
1544
|
}
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
resolveConnectedMachineWorkspaceRoot,
|
|
23
23
|
} from "@opengeni/runtime";
|
|
24
24
|
import { type Settings } from "@opengeni/config";
|
|
25
|
+
import { managedSessionGroupBackend, managedSessionGroupOs } from "@opengeni/core";
|
|
25
26
|
import { mergeResourceRefs } from "../common";
|
|
26
27
|
import {
|
|
27
28
|
mintSandboxCodemodeToken,
|
|
@@ -42,6 +43,7 @@ import {
|
|
|
42
43
|
makeMachineOpObserver,
|
|
43
44
|
recordSandboxLogicalProvision,
|
|
44
45
|
recordSandboxProvisionAttempt,
|
|
46
|
+
recordSandboxSharedPreparation,
|
|
45
47
|
recordTurnStartupPhase,
|
|
46
48
|
runtimeMetricsHooksForObservability,
|
|
47
49
|
} from "../../observability-metrics";
|
|
@@ -62,11 +64,13 @@ import {
|
|
|
62
64
|
sandboxEstablishPolicyDecision,
|
|
63
65
|
shouldPrefetchManagedSandbox,
|
|
64
66
|
reconcileActiveSandboxPointer,
|
|
67
|
+
sandboxSettingsForRoute,
|
|
65
68
|
} from "./sandbox-route";
|
|
66
69
|
import type { ClaimTurnOk } from "./claim";
|
|
67
70
|
import type { GovernanceModelOk } from "./governance-model";
|
|
68
71
|
import type { SandboxTurnRuntime } from "./sandbox-runtime";
|
|
69
72
|
import type { AttemptIdentityState, EventingState, SandboxRuntimeState } from "./turn-context";
|
|
73
|
+
import { createSharedRigSetupCoordinator } from "./sandbox-shared-preparation";
|
|
70
74
|
|
|
71
75
|
export type SandboxRouteDeps = {
|
|
72
76
|
input: RunAgentTurnInput;
|
|
@@ -249,20 +253,32 @@ export async function resolveSandboxRoute(deps: SandboxRouteDeps): Promise<Sandb
|
|
|
249
253
|
// common path this is runSettings.sandboxBackend. A selfhosted home turn
|
|
250
254
|
// that is NOT machine-primary falls back to the deployment cloud backend
|
|
251
255
|
// so swap-away / flag-off degrade to a real group box.
|
|
252
|
-
const groupBoxBackend: Settings["sandboxBackend"] =
|
|
253
|
-
runSettings.sandboxBackend
|
|
254
|
-
|
|
255
|
-
|
|
256
|
+
const groupBoxBackend: Settings["sandboxBackend"] = !machinePrimary
|
|
257
|
+
? (managedSessionGroupBackend(settings.sandboxBackend, runSettings.sandboxBackend) ??
|
|
258
|
+
runSettings.sandboxBackend)
|
|
259
|
+
: runSettings.sandboxBackend;
|
|
260
|
+
const groupRouteSettings = sandboxSettingsForRoute({
|
|
261
|
+
runSettings,
|
|
262
|
+
machinePrimary,
|
|
263
|
+
groupBoxBackend,
|
|
264
|
+
});
|
|
265
|
+
if (groupRouteSettings !== runSettings) {
|
|
266
|
+
// `modelRunSettings` drives SandboxAgent manifest construction and the
|
|
267
|
+
// legacy SDK-owned client path. Keep its provider/model context overrides,
|
|
268
|
+
// but replace the stale durable machine-home label with the route that will
|
|
269
|
+
// actually execute.
|
|
270
|
+
eventing.modelRunSettings = {
|
|
271
|
+
...eventing.modelRunSettings,
|
|
272
|
+
sandboxBackend: groupRouteSettings.sandboxBackend,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
256
275
|
sandboxState.startupMilestoneBackend = activeSandboxBackend ?? groupBoxBackend;
|
|
257
276
|
media.sandboxFileDownloadBackend = sandboxState.startupMilestoneBackend;
|
|
258
277
|
// Durable lease identity is always the logical OCI/base image. A provider
|
|
259
278
|
// image id in runSettings is only a physical cold-create optimization and
|
|
260
279
|
// must never become cross-surface shared-state fencing.
|
|
261
280
|
const groupBoxImage = rigProviderImageSourceImage(logicalSandboxSettings, groupBoxBackend);
|
|
262
|
-
const sandboxCreationBackend: Settings["sandboxBackend"] =
|
|
263
|
-
settings.sandboxOwnershipEnabled && runSettings.sandboxBackend !== "none"
|
|
264
|
-
? groupBoxBackend
|
|
265
|
-
: runSettings.sandboxBackend;
|
|
281
|
+
const sandboxCreationBackend: Settings["sandboxBackend"] = groupRouteSettings.sandboxBackend;
|
|
266
282
|
const effectiveRunCredentialBackend = activeSandboxBackend ?? groupBoxBackend;
|
|
267
283
|
|
|
268
284
|
return {
|
|
@@ -315,6 +331,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
|
|
|
315
331
|
rigVersion,
|
|
316
332
|
turnResources,
|
|
317
333
|
} = deps;
|
|
334
|
+
const groupBoxOs = managedSessionGroupOs(runSettings.sandboxBackend, session.sandboxOs);
|
|
318
335
|
const {
|
|
319
336
|
releaseLateSandbox,
|
|
320
337
|
onHomeSandboxRebound,
|
|
@@ -577,7 +594,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
|
|
|
577
594
|
sandboxGroupId: lazyGroupId,
|
|
578
595
|
sessionId: input.sessionId,
|
|
579
596
|
backend: groupBoxBackend,
|
|
580
|
-
os:
|
|
597
|
+
os: groupBoxOs,
|
|
581
598
|
environment: sandboxEnvironment,
|
|
582
599
|
...(groupBoxImage
|
|
583
600
|
? {
|
|
@@ -656,7 +673,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
|
|
|
656
673
|
// deployment default), never a "selfhosted" box (which would throw
|
|
657
674
|
// for lack of a bound agentId).
|
|
658
675
|
backend: groupBoxBackend,
|
|
659
|
-
os:
|
|
676
|
+
os: groupBoxOs,
|
|
660
677
|
environment: sandboxEnvironment,
|
|
661
678
|
// IMAGE IS SHARED STATE (B3): the container image
|
|
662
679
|
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
@@ -881,7 +898,7 @@ export async function bindLazySandboxProvisioner(
|
|
|
881
898
|
// recursion that HANGS the turn — caught live on staging 2026-07-08). The SDK
|
|
882
899
|
// already holds the proxy directly (injected as lazyOwnedSandbox.session), so it
|
|
883
900
|
// gets per-op routing; the worker-side handle (resolvedSandbox: release,
|
|
884
|
-
// heartbeat,
|
|
901
|
+
// heartbeat, workspace capture) wants the real box, unproxied.
|
|
885
902
|
return provisioned;
|
|
886
903
|
},
|
|
887
904
|
{
|
|
@@ -1007,6 +1024,24 @@ export async function bindLazySandboxProvisioner(
|
|
|
1007
1024
|
),
|
|
1008
1025
|
}
|
|
1009
1026
|
: {}),
|
|
1027
|
+
...(sandboxState.sandboxGroupId && sandboxState.sandboxHolderId
|
|
1028
|
+
? {
|
|
1029
|
+
coordinateSharedRigSetup: createSharedRigSetupCoordinator({
|
|
1030
|
+
db,
|
|
1031
|
+
accountId: input.accountId,
|
|
1032
|
+
workspaceId: input.workspaceId,
|
|
1033
|
+
sandboxGroupId: sandboxState.sandboxGroupId,
|
|
1034
|
+
attemptId: input.attemptId,
|
|
1035
|
+
holderId: sandboxState.sandboxHolderId,
|
|
1036
|
+
sandbox: provisioned,
|
|
1037
|
+
...(runtimeCancellationSignal
|
|
1038
|
+
? { signal: runtimeCancellationSignal }
|
|
1039
|
+
: {}),
|
|
1040
|
+
observe: (measurement) =>
|
|
1041
|
+
recordSandboxSharedPreparation(observability, measurement),
|
|
1042
|
+
}),
|
|
1043
|
+
}
|
|
1044
|
+
: {}),
|
|
1010
1045
|
},
|
|
1011
1046
|
),
|
|
1012
1047
|
(measurement) => {
|