@opengeni/worker-bundle 1.4.4 → 1.4.5-canary.37458084490001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/errors.d.ts +69 -2
- package/dist/activities/agent-turn/governance-model.d.ts +2 -1
- package/dist/activities/agent-turn/model-usage.d.ts +22 -0
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -0
- package/dist/activities/capabilities.d.ts +2 -0
- package/dist/activities/types.d.ts +3 -0
- package/dist/{activities-control-VRFS65FT.js → activities-control-KBOSUF5Y.js} +36 -12
- package/dist/activities-control-KBOSUF5Y.js.map +1 -0
- package/dist/{activities-turn-P2YAW2QW.js → activities-turn-QOR26CTC.js} +230 -39
- package/dist/activities-turn-QOR26CTC.js.map +1 -0
- package/dist/artifact-materializer-entry.js +0 -0
- package/dist/artifact-outbox-entry.js +0 -0
- package/dist/{chunk-HVNNCWEU.js → chunk-CMIUQ2D3.js} +114 -13
- package/dist/chunk-CMIUQ2D3.js.map +1 -0
- package/dist/{chunk-BICGFGO7.js → chunk-WX5PX2C6.js} +208 -20
- package/dist/chunk-WX5PX2C6.js.map +1 -0
- package/dist/index.js +5 -5
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +59 -4
- package/dist/workflow-bundle.js +72 -39
- package/package.json +21 -21
- package/src/activities/agent-turn/admission.ts +7 -8
- package/src/activities/agent-turn/claim.ts +20 -1
- package/src/activities/agent-turn/errors.ts +242 -13
- package/src/activities/agent-turn/failure-settlement.ts +19 -7
- package/src/activities/agent-turn/governance-model.ts +3 -0
- package/src/activities/agent-turn/model-usage.ts +114 -3
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +1 -1
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/run.ts +14 -1
- package/src/activities/agent-turn/stream-attempt.ts +68 -1
- package/src/activities/agent-turn/turn-context.ts +5 -0
- package/src/activities/automations.ts +5 -2
- package/src/activities/capabilities.ts +58 -2
- package/src/activities/session-state.ts +41 -5
- package/src/activities/types.ts +3 -1
- package/src/index.ts +2 -2
- package/src/observability-metrics.ts +184 -15
- package/src/workflows/session.ts +22 -12
- package/dist/activities-control-VRFS65FT.js.map +0 -1
- package/dist/activities-turn-P2YAW2QW.js.map +0 -1
- package/dist/chunk-BICGFGO7.js.map +0 -1
- package/dist/chunk-HVNNCWEU.js.map +0 -1
|
@@ -4,9 +4,12 @@ import { ApplicationFailure } from "@temporalio/activity";
|
|
|
4
4
|
import type { CodexAccountStatus } from "@opengeni/db";
|
|
5
5
|
import type { EscapedMcpTimeoutRecoveryDetail } from "../types.js";
|
|
6
6
|
import { type MandatoryHistoryPersistenceStage } from "./quiescence.js";
|
|
7
|
+
import { type ProviderCondition } from "./provider-recovery-copy.js";
|
|
8
|
+
export { PROVIDER_OVERLOAD_RECOVERY_CODE, MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES, PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS, } from "./provider-recovery-policy.js";
|
|
7
9
|
export declare const PROVIDER_BACKPRESSURE_DELAY_MS = 60000;
|
|
8
10
|
export declare const PROVIDER_CONNECTIVITY_BACKOFF_MS: readonly [2000, 5000, 15000, 30000, 60000];
|
|
9
11
|
export declare const MAX_AUTOMATIC_PROVIDER_RECOVERIES: 5;
|
|
12
|
+
export declare function providerRecoveryLimit(failureCode: unknown): number;
|
|
10
13
|
/**
|
|
11
14
|
* Minimum wait per rate-limited recovery. Providers such as Azure OpenAI often
|
|
12
15
|
* answer a per-minute token limit with a `retry-after` of about a second, which
|
|
@@ -23,17 +26,44 @@ export declare class PostCompactionContinuationEmptyError extends Error {
|
|
|
23
26
|
export type ProviderRecoveryResult = {
|
|
24
27
|
status: "recovering";
|
|
25
28
|
continueDelayMs: number;
|
|
29
|
+
maxProviderRecoveryCount?: number;
|
|
26
30
|
} | {
|
|
27
31
|
status: "exhausted";
|
|
28
32
|
providerRecoveryCount: number;
|
|
29
33
|
maxProviderRecoveryCount: number;
|
|
34
|
+
providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
|
|
30
35
|
};
|
|
31
36
|
export declare function providerRecoveryResult(input: {
|
|
32
37
|
failureCode: string | undefined;
|
|
33
38
|
attemptNumber: number;
|
|
34
39
|
retryAfterMs?: number | null;
|
|
35
40
|
jitterSample?: number;
|
|
41
|
+
/** Durable first-failure clock, never a new clock on each replacement attempt. */
|
|
42
|
+
recoveryStartedAt?: number | undefined;
|
|
43
|
+
now?: number;
|
|
36
44
|
}): ProviderRecoveryResult;
|
|
45
|
+
/** A timer/queue delay cannot authorize a provider call after the recovery window. */
|
|
46
|
+
export declare class ProviderOverloadRecoveryExpiredError extends Error {
|
|
47
|
+
readonly failure: {
|
|
48
|
+
error: string;
|
|
49
|
+
code: string;
|
|
50
|
+
providerCondition: "overloaded";
|
|
51
|
+
} & {
|
|
52
|
+
retryable: false;
|
|
53
|
+
recoveryExhausted: true;
|
|
54
|
+
providerRecoveryCount: number;
|
|
55
|
+
maxProviderRecoveryCount: number;
|
|
56
|
+
lastRetryableError: string;
|
|
57
|
+
providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
|
|
58
|
+
};
|
|
59
|
+
constructor(count: number, reason: "deadline" | "retry_limit" | "invalid_clock");
|
|
60
|
+
}
|
|
61
|
+
export declare function assertProviderOverloadRecoveryActive(input: {
|
|
62
|
+
failureCode: unknown;
|
|
63
|
+
providerRecoveryCount: number;
|
|
64
|
+
recoveryStartedAt?: number | undefined;
|
|
65
|
+
now?: number;
|
|
66
|
+
}): void;
|
|
37
67
|
export declare function providerRecoveryExhaustedFailure<T extends Record<string, unknown> & {
|
|
38
68
|
error: string;
|
|
39
69
|
}>(failure: T, recovery: Extract<ProviderRecoveryResult, {
|
|
@@ -44,6 +74,23 @@ export declare function providerRecoveryExhaustedFailure<T extends Record<string
|
|
|
44
74
|
providerRecoveryCount: number;
|
|
45
75
|
maxProviderRecoveryCount: number;
|
|
46
76
|
lastRetryableError: string;
|
|
77
|
+
providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
|
|
78
|
+
};
|
|
79
|
+
/**
|
|
80
|
+
* Name the exact accepted model route on model-provider recovery evidence so a
|
|
81
|
+
* person learns which model is affected. Display labels only: the route's
|
|
82
|
+
* credentials, base URL and upstream ids never enter the event.
|
|
83
|
+
*/
|
|
84
|
+
export declare function withModelRoutePresentation<T extends {
|
|
85
|
+
code?: string | undefined;
|
|
86
|
+
}>(failure: T, route: {
|
|
87
|
+
model: string;
|
|
88
|
+
modelLabel: string;
|
|
89
|
+
providerLabel: string;
|
|
90
|
+
} | undefined): T & {
|
|
91
|
+
model?: string;
|
|
92
|
+
modelLabel?: string;
|
|
93
|
+
providerLabel?: string;
|
|
47
94
|
};
|
|
48
95
|
export declare function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number;
|
|
49
96
|
export declare function providerRecoveryCountAfterModelRequestPhase(currentCount: number, phase: string): number;
|
|
@@ -100,6 +147,7 @@ export declare function postClaimDatabaseRecoveryFailure(input: {
|
|
|
100
147
|
providerRecovery?: {
|
|
101
148
|
failureCode: string;
|
|
102
149
|
providerRecoveryCount: number;
|
|
150
|
+
continueDelayMs?: number;
|
|
103
151
|
};
|
|
104
152
|
}): ApplicationFailure | null;
|
|
105
153
|
/**
|
|
@@ -276,6 +324,16 @@ export declare function classifyXaiCredentialFailure(error: unknown): XaiCredent
|
|
|
276
324
|
export declare function agentRunFailurePayload(error: unknown, options?: {
|
|
277
325
|
isCodexTurn?: boolean;
|
|
278
326
|
}): ReturnType<typeof baseAgentRunFailurePayload>;
|
|
327
|
+
/** HTTP 529, Anthropic `overloaded_error`, or explicit provider "overloaded" wording. */
|
|
328
|
+
export declare function isProviderOverloadError(error: unknown, failure?: {
|
|
329
|
+
error?: string;
|
|
330
|
+
detail?: string;
|
|
331
|
+
}): boolean;
|
|
332
|
+
/** Retry authority must not depend on the looser presentation-only overload label. */
|
|
333
|
+
export declare function providerRecoveryCode(error: unknown, failure: {
|
|
334
|
+
code?: string;
|
|
335
|
+
retryable?: boolean;
|
|
336
|
+
}): string | undefined;
|
|
279
337
|
/** Keep Anthropic provider text on terminal failures, never recovery events. */
|
|
280
338
|
export declare function agentRunRecoveryFailurePayload(error: unknown, failure: ReturnType<typeof agentRunFailurePayload>): ReturnType<typeof agentRunFailurePayload>;
|
|
281
339
|
declare function baseAgentRunFailurePayload(error: unknown, options?: {
|
|
@@ -303,8 +361,18 @@ declare function baseAgentRunFailurePayload(error: unknown, options?: {
|
|
|
303
361
|
quotaScope?: ProviderQuotaScope;
|
|
304
362
|
/** Closed Codex plan key on `codex_plan_entitlement` / `codex_request_rejected`. */
|
|
305
363
|
planType?: string | null;
|
|
306
|
-
/** Product model id a `codex_plan_entitlement` failure refers to. */
|
|
364
|
+
/** Product model id a `codex_plan_entitlement` or model-provider recovery failure refers to. */
|
|
307
365
|
model?: string | null;
|
|
366
|
+
/** Display label of the turn's model on model-provider recovery evidence. */
|
|
367
|
+
modelLabel?: string;
|
|
368
|
+
/** Display label of the serving provider on model-provider recovery evidence. */
|
|
369
|
+
providerLabel?: string;
|
|
370
|
+
/** Closed transient provider condition on `provider_unavailable` / `provider_rate_limited`. */
|
|
371
|
+
providerCondition?: ProviderCondition;
|
|
372
|
+
recoveryExhausted?: boolean;
|
|
373
|
+
providerRecoveryCount?: number;
|
|
374
|
+
maxProviderRecoveryCount?: number;
|
|
375
|
+
providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
|
|
308
376
|
};
|
|
309
377
|
export type CodexCredentialFailure = {
|
|
310
378
|
/**
|
|
@@ -352,4 +420,3 @@ export declare const CODEX_USAGE_LIMIT_MAX_RESUME_MS: number;
|
|
|
352
420
|
export declare function classifyClaudeCredentialFailure(error: unknown): (XaiCredentialFailure & {
|
|
353
421
|
requestId?: string;
|
|
354
422
|
}) | null;
|
|
355
|
-
export {};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { materializeRigVersionForAttempt } from "@opengeni/db";
|
|
1
|
+
import { materializeRigVersionForAttempt, freezeAgentLearningPolicy } from "@opengeni/db";
|
|
2
2
|
import { renderWorkspaceGovernanceContext, type OpenGeniRuntime } from "@opengeni/runtime";
|
|
3
3
|
import { type Settings } from "@opengeni/config";
|
|
4
4
|
import { createModelHistoryAttachmentProjector } from "../run-input.js";
|
|
@@ -9,6 +9,7 @@ import { lazyToolTransportForTurn, openAiHostedImageProviderBindingForTurn, mode
|
|
|
9
9
|
import type { ClaimTurnOk } from "./claim.js";
|
|
10
10
|
import type { EventingState, WorkspaceRefState } from "./turn-context.js";
|
|
11
11
|
export type GovernanceModelDeps = {
|
|
12
|
+
learningPolicy: Awaited<ReturnType<typeof freezeAgentLearningPolicy>>;
|
|
12
13
|
input: RunAgentTurnInput;
|
|
13
14
|
db: ActivityServices["db"];
|
|
14
15
|
observability: ActivityServices["observability"];
|
|
@@ -227,7 +227,29 @@ export declare function recordModelUsageAndDebitCredits(settings: Settings, db:
|
|
|
227
227
|
sourceKey: string;
|
|
228
228
|
latencyMode?: LatencyMode;
|
|
229
229
|
observability?: ActivityServices["observability"];
|
|
230
|
+
/** Configured provider id for metric labels; never billing authority. */
|
|
231
|
+
metricProvider?: string;
|
|
230
232
|
}): Promise<ModelUsageBillingRecord | null>;
|
|
233
|
+
/**
|
|
234
|
+
* The bounded `model` metric label: the deployment catalog product id for a
|
|
235
|
+
* catalog model, or `custom` for a workspace-owned model (workspace gateway or
|
|
236
|
+
* a customer-key provider) and for anything the deployment catalog does not
|
|
237
|
+
* list. Metrics only; never billing or routing authority.
|
|
238
|
+
*/
|
|
239
|
+
export declare function modelMetricProductId(settings: Settings, provider: string | undefined, model: string): string;
|
|
240
|
+
/**
|
|
241
|
+
* Per-model usage and estimated provider cost for one authoritative response.
|
|
242
|
+
* Call only after the durable usage event was accepted as current (the same
|
|
243
|
+
* fence as the Insights fact). Best-effort: never throws.
|
|
244
|
+
*/
|
|
245
|
+
export declare function recordAuthoritativeModelUsageMetrics(input: {
|
|
246
|
+
observability: ActivityServices["observability"];
|
|
247
|
+
settings: Settings;
|
|
248
|
+
provider: string;
|
|
249
|
+
model: string;
|
|
250
|
+
externallyBilled: boolean;
|
|
251
|
+
billing: ModelUsageBillingRecord;
|
|
252
|
+
}): void;
|
|
231
253
|
/** Soft-fail Insights fact write — never throws into the billing/emit path. */
|
|
232
254
|
export declare function recordAuthoritativeModelCallFact(input: {
|
|
233
255
|
db: ActivityServices["db"];
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Person-facing copy for finite same-turn provider recovery. Presentation only:
|
|
3
|
+
* retry authority, pacing and the recovery budget stay in `errors.ts`.
|
|
4
|
+
*
|
|
5
|
+
* `@opengeni/react` (`lib/provider-recovery.ts`) composes the same sentences
|
|
6
|
+
* from the structured event fields, so live retry status and historical
|
|
7
|
+
* failures read identically in every surface. Keep the two in step.
|
|
8
|
+
*/
|
|
9
|
+
/** Closed provider condition recorded on model-provider recovery payloads. */
|
|
10
|
+
export type ProviderCondition = "overloaded" | "unavailable" | "unresponsive" | "rate_limited";
|
|
11
|
+
/** Failure codes whose transient condition belongs to the turn's model route. */
|
|
12
|
+
export declare const MODEL_PROVIDER_RECOVERY_CODES: ReadonlySet<string>;
|
|
13
|
+
export type ProviderRecoveryCopyInput = {
|
|
14
|
+
code?: string | null | undefined;
|
|
15
|
+
providerCondition?: ProviderCondition | null | undefined;
|
|
16
|
+
modelLabel?: string | null | undefined;
|
|
17
|
+
providerLabel?: string | null | undefined;
|
|
18
|
+
};
|
|
19
|
+
/** One present-tense clause naming what is unavailable, without a trailing period. */
|
|
20
|
+
export declare function providerRecoverySubject(input: ProviderRecoveryCopyInput): string;
|
|
21
|
+
/** Terminal copy once the finite automatic recovery budget is spent. */
|
|
22
|
+
export declare function providerRecoveryExhaustedMessage(input: ProviderRecoveryCopyInput & {
|
|
23
|
+
providerRecoveryCount: number;
|
|
24
|
+
}): string;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/** Closed policy facts shared with the workflow's content-free recovery wire. */
|
|
2
|
+
export declare const PROVIDER_OVERLOAD_RECOVERY_CODE = "provider_overloaded";
|
|
3
|
+
export declare const MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES = 15;
|
|
4
|
+
export declare const PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS: number;
|
|
5
|
+
export declare function validProviderOverloadRecoveryDelay(code: unknown, delay: unknown): boolean;
|
|
6
|
+
export declare function readProviderRecoveryStartedAt(metadata: Record<string, unknown>): number | undefined;
|
|
@@ -43,11 +43,20 @@ export type AttemptIdentityState = {
|
|
|
43
43
|
triggerEventId: string | undefined;
|
|
44
44
|
executionGeneration: number;
|
|
45
45
|
providerRecoveryCount: number;
|
|
46
|
+
/** Durable recovery policy reason; never inferred from display wording. */
|
|
47
|
+
providerRecoveryPolicyCode?: string | undefined;
|
|
48
|
+
providerRecoveryStartedAt?: number | undefined;
|
|
46
49
|
providerRecoveryObservation?: import("./provider-recovery-metrics.js").ProviderRecoveryObservation | undefined;
|
|
47
50
|
modelMetricRoute?: {
|
|
48
51
|
provider: string;
|
|
49
52
|
model: string;
|
|
50
53
|
};
|
|
54
|
+
/** Public display labels of the accepted model route, for recovery copy only. */
|
|
55
|
+
modelRoutePresentation?: {
|
|
56
|
+
model: string;
|
|
57
|
+
modelLabel: string;
|
|
58
|
+
providerLabel: string;
|
|
59
|
+
};
|
|
51
60
|
claudeAuthRecovery?: {
|
|
52
61
|
credentialId: string;
|
|
53
62
|
credentialVersion: number;
|
|
@@ -24,4 +24,6 @@ export declare function withCodexProvider(settings: Settings): Settings;
|
|
|
24
24
|
export declare function withXaiSubscriptionProvider(settings: Settings): Settings;
|
|
25
25
|
export declare function settingsWithWorkspaceGatewayCredential(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
|
|
26
26
|
export declare function settingsWithWorkspaceOpenRouterCredential(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
|
|
27
|
+
/** Worker-only: overlay the workspace Opper catalog and, when active, its decrypted key. */
|
|
28
|
+
export declare function settingsWithWorkspaceOpperCredential(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
|
|
27
29
|
export declare function settingsWithOrganizationProviderCredentials(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
|
|
@@ -226,6 +226,7 @@ export type FailSessionAttemptResult = {
|
|
|
226
226
|
action: "failed";
|
|
227
227
|
} | {
|
|
228
228
|
action: "recovering";
|
|
229
|
+
continueDelayMs?: number;
|
|
229
230
|
} | {
|
|
230
231
|
action: "unclaimed";
|
|
231
232
|
} | {
|
|
@@ -254,6 +255,8 @@ export type PostClaimDatabaseRecoveryDetail = {
|
|
|
254
255
|
providerRecoveryCount?: number;
|
|
255
256
|
/** Safe classified provider cause paired with providerRecoveryCount. */
|
|
256
257
|
providerFailureCode?: string;
|
|
258
|
+
/** Preserve a confirmed-overload Retry-After through checkpoint DB outages. */
|
|
259
|
+
providerRecoveryContinueDelayMs?: number;
|
|
257
260
|
/** Persist the no-replay setup marker, not a new provider/setup attempt. */
|
|
258
261
|
sandboxSetupOutcomeUnknown?: true;
|
|
259
262
|
/** Preserve the exhausted budget for a command proven not dispatched. */
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
import {
|
|
9
9
|
INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
|
|
10
10
|
MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
11
|
+
PROVIDER_OVERLOAD_RECOVERY_CODE,
|
|
11
12
|
assertReusableSessionRevivable,
|
|
12
13
|
captureConnectedCommandOutput,
|
|
13
14
|
createCodexCapacityActivities,
|
|
@@ -23,6 +24,7 @@ import {
|
|
|
23
24
|
notifyParentOfChildIdle,
|
|
24
25
|
persistWorkspaceArchiveCandidate,
|
|
25
26
|
providerRecoveryCountFromMetadata,
|
|
27
|
+
providerRecoveryLimit,
|
|
26
28
|
publishDurableSessionEvents,
|
|
27
29
|
putVersion1TarArchiveOrInline,
|
|
28
30
|
reconcileAutomaticSessionTitleFanout,
|
|
@@ -31,10 +33,11 @@ import {
|
|
|
31
33
|
resolveIncidentTelemetryResponderMetadata,
|
|
32
34
|
rigProviderImageSourceImage,
|
|
33
35
|
scheduledUserMessagePayload,
|
|
36
|
+
validProviderOverloadRecoveryDelay,
|
|
34
37
|
warnDrainSnapshotFailure,
|
|
35
38
|
withFirstPartyTools,
|
|
36
39
|
workflowIdForSession
|
|
37
|
-
} from "./chunk-
|
|
40
|
+
} from "./chunk-WX5PX2C6.js";
|
|
38
41
|
import {
|
|
39
42
|
inspectOpenSandboxKubernetesInventory,
|
|
40
43
|
recordActiveUserGauges,
|
|
@@ -64,7 +67,7 @@ import {
|
|
|
64
67
|
recordVerifiedSignupTrialSwitchGauge,
|
|
65
68
|
recordWorkerDeathRecoveryMetrics,
|
|
66
69
|
runtimeMetricsHooksForObservability
|
|
67
|
-
} from "./chunk-
|
|
70
|
+
} from "./chunk-CMIUQ2D3.js";
|
|
68
71
|
import "./chunk-PZ5AY32C.js";
|
|
69
72
|
|
|
70
73
|
// src/activities/knowledge-indexing.ts
|
|
@@ -712,7 +715,8 @@ function createBrowserDeadlineCheckpointActivities(services, connectController =
|
|
|
712
715
|
import {
|
|
713
716
|
resolveTurnExecutionPolicyV1,
|
|
714
717
|
WORKSPACE_GATEWAY_MODEL_ID_PREFIX,
|
|
715
|
-
WORKSPACE_OPENROUTER_MODEL_ID_PREFIX
|
|
718
|
+
WORKSPACE_OPENROUTER_MODEL_ID_PREFIX,
|
|
719
|
+
WORKSPACE_OPPER_MODEL_ID_PREFIX
|
|
716
720
|
} from "@opengeni/config";
|
|
717
721
|
import {
|
|
718
722
|
applySessionAgentConfigWriteThrough,
|
|
@@ -800,7 +804,7 @@ function createAutomationActivities(services, options = {}) {
|
|
|
800
804
|
return { action: "failed", reason: "credential_restriction_violation" };
|
|
801
805
|
}
|
|
802
806
|
const requestedModel = template.model ?? deploymentCatalogSettings.openaiModel;
|
|
803
|
-
const catalogSettings = requestedModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) ? (await resolveWorkspaceCatalogSettings(service.db, catalogSourceSettings, {
|
|
807
|
+
const catalogSettings = requestedModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPPER_MODEL_ID_PREFIX) ? (await resolveWorkspaceCatalogSettings(service.db, catalogSourceSettings, {
|
|
804
808
|
accountId: input.accountId,
|
|
805
809
|
workspaceId: input.workspaceId,
|
|
806
810
|
retainedProductModelId: requestedModel
|
|
@@ -950,7 +954,7 @@ function createAutomationActivities(services, options = {}) {
|
|
|
950
954
|
firstPartyMcpPermissions: template.firstPartyMcpPermissions,
|
|
951
955
|
firstPartyMcpTools: agentWriteThrough.firstPartyMcpTools,
|
|
952
956
|
...agentWriteThrough.config ? { agentConfig: agentWriteThrough.config } : {},
|
|
953
|
-
retainWorkspaceCustomModel: model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX),
|
|
957
|
+
retainWorkspaceCustomModel: model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPPER_MODEL_ID_PREFIX),
|
|
954
958
|
createIdempotencyKey: `automation-run:${run.id}`,
|
|
955
959
|
subjectId: accepted.serviceSubjectId
|
|
956
960
|
});
|
|
@@ -8172,6 +8176,7 @@ import {
|
|
|
8172
8176
|
} from "@opengeni/db";
|
|
8173
8177
|
import { publishDurableSessionEvents as publishDurableSessionEvents3 } from "@opengeni/events";
|
|
8174
8178
|
import { CancelledFailure } from "@temporalio/activity";
|
|
8179
|
+
import { interactionInterventionMetricObserver } from "@opengeni/observability";
|
|
8175
8180
|
var WORKER_DEATH_MAX_REDISPATCHES = 3;
|
|
8176
8181
|
function createSessionStateActivities(services, overrides = {}) {
|
|
8177
8182
|
const settleSessionAttemptInterruptionsFn = overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
|
|
@@ -8286,13 +8291,14 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
8286
8291
|
const postClaimIdentityMatches = Boolean(
|
|
8287
8292
|
postClaimRecovery && turn.id === postClaimRecovery.turnId && turn.triggerEventId === postClaimRecovery.triggerEventId && turn.executionGeneration === postClaimRecovery.executionGeneration
|
|
8288
8293
|
);
|
|
8289
|
-
if (postClaimRecovery
|
|
8294
|
+
if (postClaimRecovery && !postClaimIdentityMatches) {
|
|
8290
8295
|
return { action: "stale" };
|
|
8291
8296
|
}
|
|
8292
8297
|
const providerRecoveryCount = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryCount : void 0;
|
|
8293
8298
|
const providerFailureCode = postClaimIdentityMatches ? postClaimRecovery?.providerFailureCode : void 0;
|
|
8294
8299
|
const hasProviderRecoveryCount = providerRecoveryCount !== void 0;
|
|
8295
8300
|
const hasProviderFailureCode = providerFailureCode !== void 0;
|
|
8301
|
+
const providerRecoveryContinueDelayMs = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryContinueDelayMs : void 0;
|
|
8296
8302
|
const setupOutcomeUnknown = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupOutcomeUnknown === true;
|
|
8297
8303
|
const setupRecoveryExhausted = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupRecoveryExhausted === true;
|
|
8298
8304
|
if (setupOutcomeUnknown && setupRecoveryExhausted || (setupOutcomeUnknown || setupRecoveryExhausted) && (hasProviderRecoveryCount || hasProviderFailureCode) || setupRecoveryExhausted && turn.metadata?.providerRecoveryCount !== SANDBOX_SETUP_RECOVERY_LIMIT) {
|
|
@@ -8301,7 +8307,10 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
8301
8307
|
if (hasProviderRecoveryCount !== hasProviderFailureCode) {
|
|
8302
8308
|
return { action: "stale" };
|
|
8303
8309
|
}
|
|
8304
|
-
if (
|
|
8310
|
+
if (!validProviderOverloadRecoveryDelay(providerFailureCode, providerRecoveryContinueDelayMs)) {
|
|
8311
|
+
return { action: "stale" };
|
|
8312
|
+
}
|
|
8313
|
+
if (providerRecoveryCount !== void 0 && (!Number.isSafeInteger(providerRecoveryCount) || providerRecoveryCount <= 0 || providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 || providerRecoveryCount > providerRecoveryLimit(providerFailureCode) || typeof providerFailureCode !== "string" || !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))) {
|
|
8305
8314
|
return { action: "stale" };
|
|
8306
8315
|
}
|
|
8307
8316
|
const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : input.preClaimFailureDisposition === "retryable" ? "legacy_retryable_preclaim_database_failure" : null;
|
|
@@ -8316,7 +8325,7 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
8316
8325
|
...setupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
|
|
8317
8326
|
...providerRecoveryCount !== void 0 ? { providerRecoveryCount } : {},
|
|
8318
8327
|
detail: {
|
|
8319
|
-
code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? recoveredClaimCode,
|
|
8328
|
+
code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode === PROVIDER_OVERLOAD_RECOVERY_CODE ? "provider_unavailable" : providerFailureCode ?? recoveredClaimCode,
|
|
8320
8329
|
retryable: !setupOutcomeUnknown && !setupRecoveryExhausted,
|
|
8321
8330
|
...setupOutcomeUnknown ? { setupOutcome: "unknown", replay: "blocked" } : {},
|
|
8322
8331
|
...setupRecoveryExhausted ? {
|
|
@@ -8327,7 +8336,12 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
8327
8336
|
} : {},
|
|
8328
8337
|
...providerRecoveryCount !== void 0 ? {
|
|
8329
8338
|
databaseFailureCode: recoveredClaimCode,
|
|
8330
|
-
providerRecoveryCount
|
|
8339
|
+
providerRecoveryCount,
|
|
8340
|
+
...providerFailureCode === PROVIDER_OVERLOAD_RECOVERY_CODE ? {
|
|
8341
|
+
providerCondition: "overloaded",
|
|
8342
|
+
maxProviderRecoveryCount: providerRecoveryLimit(providerFailureCode),
|
|
8343
|
+
continueDelayMs: providerRecoveryContinueDelayMs
|
|
8344
|
+
} : {}
|
|
8331
8345
|
} : {},
|
|
8332
8346
|
recoverySource: "workflow_activity_failure"
|
|
8333
8347
|
},
|
|
@@ -8341,7 +8355,10 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
8341
8355
|
countQueuedTurnsFn,
|
|
8342
8356
|
recordTurnsQueuedGaugeFn
|
|
8343
8357
|
);
|
|
8344
|
-
return {
|
|
8358
|
+
return {
|
|
8359
|
+
action: "recovering",
|
|
8360
|
+
...providerRecoveryContinueDelayMs !== void 0 ? { continueDelayMs: providerRecoveryContinueDelayMs } : {}
|
|
8361
|
+
};
|
|
8345
8362
|
}
|
|
8346
8363
|
const trigger = await getSessionEventFn(db, input.workspaceId, turn.triggerEventId);
|
|
8347
8364
|
const result = await applySessionTurnSettlementFn(db, input.workspaceId, {
|
|
@@ -8659,8 +8676,15 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
8659
8676
|
};
|
|
8660
8677
|
}
|
|
8661
8678
|
async function expireSessionInteractionIntervention(input) {
|
|
8662
|
-
const { db, bus } = await services();
|
|
8679
|
+
const { db, bus, observability } = await services();
|
|
8663
8680
|
const result = await expireSessionInteractionInterventionFn(db, input);
|
|
8681
|
+
if (result.action === "expired" && result.intervention) {
|
|
8682
|
+
interactionInterventionMetricObserver(observability)({
|
|
8683
|
+
kind: result.intervention.kind,
|
|
8684
|
+
outcome: "expired",
|
|
8685
|
+
waitMs: Date.now() - result.intervention.createdAt.getTime()
|
|
8686
|
+
});
|
|
8687
|
+
}
|
|
8664
8688
|
if (result.events.length > 0) {
|
|
8665
8689
|
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
8666
8690
|
}
|
|
@@ -8859,4 +8883,4 @@ export {
|
|
|
8859
8883
|
createControlActivities,
|
|
8860
8884
|
createControlActivitiesFromServices
|
|
8861
8885
|
};
|
|
8862
|
-
//# sourceMappingURL=activities-control-
|
|
8886
|
+
//# sourceMappingURL=activities-control-KBOSUF5Y.js.map
|