@opengeni/worker-bundle 0.23.4 → 0.26.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +17 -2
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +8 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -2
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +1 -1
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +15 -1
- package/dist/activities/agent-turn/quiescence.d.ts +11 -6
- package/dist/activities/agent-turn/run.d.ts +9 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
- package/dist/activities/agent-turn/session-title.d.ts +21 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +4 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +40 -5
- package/dist/activities/agent-turn/turn-context.d.ts +13 -2
- package/dist/activities/agent-turn.d.ts +1 -1
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +3 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/goals.d.ts +12 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +19 -1
- package/dist/{activities-control-52W5IDWM.js → activities-control-CTEMLTHG.js} +339 -195
- package/dist/activities-control-CTEMLTHG.js.map +1 -0
- package/dist/activities-control.d.ts +4 -0
- package/dist/{activities-turn-A2G2KLSY.js → activities-turn-S5IU347A.js} +1846 -1136
- package/dist/activities-turn-S5IU347A.js.map +1 -0
- package/dist/activities.d.ts +4 -0
- package/dist/{chunk-GFNXVZVQ.js → chunk-D4ZEFOYH.js} +107 -7
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
- package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
- package/dist/{chunk-IPVMCAW5.js → chunk-TAUE3GYT.js} +225 -154
- package/dist/chunk-TAUE3GYT.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.js +43 -7
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +35 -8
- package/dist/sandbox-reaper-timeout.d.ts +2 -2
- package/dist/sandbox-resume.d.ts +6 -1
- package/dist/workflow-bundle.js +96 -10
- package/dist/workflows/activities.d.ts +6 -1
- package/dist/workflows/automations.d.ts +5 -0
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +47 -6
- package/src/activities/agent-turn/agent-build.ts +20 -26
- package/src/activities/agent-turn/claim.ts +59 -6
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/codex.ts +2 -2
- package/src/activities/agent-turn/compaction-prep.ts +8 -1
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +29 -1
- package/src/activities/agent-turn/failure-settlement.ts +451 -323
- package/src/activities/agent-turn/finalization.ts +31 -17
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +120 -45
- package/src/activities/agent-turn/quiescence.ts +32 -17
- package/src/activities/agent-turn/run.ts +98 -9
- package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
- package/src/activities/agent-turn/session-title.ts +55 -2
- package/src/activities/agent-turn/stream-attempt.ts +281 -136
- package/src/activities/agent-turn/tool-environment.ts +132 -39
- package/src/activities/agent-turn/tool-policy.ts +71 -12
- package/src/activities/agent-turn/turn-context.ts +21 -2
- package/src/activities/agent-turn.ts +5 -1
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +129 -2
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +89 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/context-compaction.ts +3 -0
- package/src/activities/goals.ts +116 -39
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/mcp-credentials.ts +53 -0
- package/src/activities/rig-verification.ts +5 -3
- package/src/activities/sandbox-lease.ts +29 -12
- package/src/activities/scheduled-tasks.ts +217 -110
- package/src/activities/session-state.ts +17 -0
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +22 -8
- package/src/activities.ts +2 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +2 -83
- package/src/index.ts +34 -4
- package/src/observability-metrics.ts +158 -14
- package/src/sandbox-reaper-timeout.ts +10 -4
- package/src/sandbox-resume.ts +58 -8
- package/src/sandbox-routing.ts +71 -16
- package/src/workflows/activities.ts +16 -0
- package/src/workflows/automations.ts +21 -2
- package/src/workflows/session.ts +85 -7
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-52W5IDWM.js.map +0 -1
- package/dist/activities-turn-A2G2KLSY.js.map +0 -1
- package/dist/chunk-GFNXVZVQ.js.map +0 -1
- package/dist/chunk-IPVMCAW5.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -141
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
fetchCodexUsageForAccount,
|
|
4
4
|
type CodexAccountStatus,
|
|
5
5
|
} from "@opengeni/db";
|
|
6
|
-
import { type
|
|
6
|
+
import { type ResolvedModelProvider, type Settings } from "@opengeni/config";
|
|
7
7
|
import { codexAccountNeedsLiveCapacityRefresh } from "../codex-rotation";
|
|
8
8
|
import {
|
|
9
9
|
refreshCodexUsageAndRepairCapacityWaiters,
|
|
@@ -19,7 +19,7 @@ export function codexWorkspaceMetricKey(workspaceId: string): string {
|
|
|
19
19
|
/** Stable public request identity across partial resumes and activity retries. */
|
|
20
20
|
export function acceptsPromptCacheKeyForTurn(
|
|
21
21
|
resolvedModel: {
|
|
22
|
-
provider: { kind:
|
|
22
|
+
provider: { kind: ResolvedModelProvider["kind"]; builtin?: boolean };
|
|
23
23
|
} | null,
|
|
24
24
|
): boolean {
|
|
25
25
|
if (!resolvedModel) {
|
|
@@ -20,7 +20,10 @@ import type {
|
|
|
20
20
|
RunAgentTurnInput,
|
|
21
21
|
RunAgentTurnResult,
|
|
22
22
|
} from "../types";
|
|
23
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
recordContextCompaction,
|
|
25
|
+
recordContextCompactionStarted,
|
|
26
|
+
} from "../../observability-metrics";
|
|
24
27
|
import { createTurnCredentialLeases } from "./credential-leases";
|
|
25
28
|
import { createTurnMediaArtifacts } from "./media-artifacts";
|
|
26
29
|
import { type SessionEvent } from "@opengeni/contracts";
|
|
@@ -188,6 +191,8 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
|
|
|
188
191
|
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
189
192
|
model: resolvedModel?.configured.id ?? turn.model,
|
|
190
193
|
externallyBilled: billingState.isExternallyBilledTurn,
|
|
194
|
+
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
195
|
+
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
191
196
|
turnAttemptId: input.attemptId,
|
|
192
197
|
servingCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
193
198
|
priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
|
|
@@ -286,6 +291,8 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
|
|
|
286
291
|
const compactionModeOptions = {
|
|
287
292
|
codexCompactionMode: session.codexCompactionMode,
|
|
288
293
|
isCodexSubscriptionTurn: billingState.isCodexTurn,
|
|
294
|
+
onCompactionStarted: (trigger: "auto" | "operator" | "proactive" | "overflow") =>
|
|
295
|
+
recordContextCompactionStarted(observability, trigger),
|
|
289
296
|
publishLiveEvents: publishCompactionLiveEvents,
|
|
290
297
|
...(remoteCompactionRequester ? { requestRemoteCompactionV2: remoteCompactionRequester } : {}),
|
|
291
298
|
} as const;
|
|
@@ -11,6 +11,15 @@ import { safeErrorDiagnostic } from "./errors";
|
|
|
11
11
|
export type LeaseRenewReason = "timer" | "runtime_event" | "model_usage";
|
|
12
12
|
export type LeaseLossReason = "deadline" | "not_found";
|
|
13
13
|
|
|
14
|
+
export class CodexCredentialLeaseLostError extends Error {
|
|
15
|
+
readonly code = "codex_credential_lease_lost";
|
|
16
|
+
|
|
17
|
+
constructor(readonly reason: LeaseLossReason) {
|
|
18
|
+
super("Codex credential lease is not usable for provider dispatch");
|
|
19
|
+
this.name = "CodexCredentialLeaseLostError";
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
14
23
|
export type TurnCredentialLeaseDeps = {
|
|
15
24
|
db: SharedActivityServices["db"];
|
|
16
25
|
observability: SharedActivityServices["observability"];
|
|
@@ -28,6 +37,7 @@ export type TurnCredentialLeaseDeps = {
|
|
|
28
37
|
export class CodexTurnLease {
|
|
29
38
|
held = false;
|
|
30
39
|
lost = false;
|
|
40
|
+
lossReason: LeaseLossReason | null = null;
|
|
31
41
|
holderId: string | null = null;
|
|
32
42
|
generation: number | null = null;
|
|
33
43
|
// Monotonic worker deadline, not a comparison between the Postgres and
|
|
@@ -43,6 +53,7 @@ export class CodexTurnLease {
|
|
|
43
53
|
markLost = (reason: LeaseLossReason): void => {
|
|
44
54
|
if (this.lost) return;
|
|
45
55
|
this.lost = true;
|
|
56
|
+
this.lossReason = reason;
|
|
46
57
|
this.deps.observability.incrementCounter({
|
|
47
58
|
name: "opengeni_codex_lease_renewals_total",
|
|
48
59
|
help: "Codex lease renewal checkpoints by outcome and reason.",
|
|
@@ -55,6 +66,22 @@ export class CodexTurnLease {
|
|
|
55
66
|
});
|
|
56
67
|
};
|
|
57
68
|
|
|
69
|
+
/** Fail closed at the last worker-confirmed lease deadline before dispatch. */
|
|
70
|
+
assertUsable = (): void => {
|
|
71
|
+
if (this.lost) {
|
|
72
|
+
throw new CodexCredentialLeaseLostError(this.lossReason ?? "not_found");
|
|
73
|
+
}
|
|
74
|
+
if (
|
|
75
|
+
!this.held ||
|
|
76
|
+
this.holderId === null ||
|
|
77
|
+
this.generation === null ||
|
|
78
|
+
codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)
|
|
79
|
+
) {
|
|
80
|
+
this.markLost("deadline");
|
|
81
|
+
throw new CodexCredentialLeaseLostError("deadline");
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
|
|
58
85
|
renew = async (reason: LeaseRenewReason): Promise<void> => {
|
|
59
86
|
const turnId = this.deps.getTurnId();
|
|
60
87
|
if (!turnId || !this.held || !this.holderId || this.generation === null || this.lost) {
|
|
@@ -66,6 +93,7 @@ export class CodexTurnLease {
|
|
|
66
93
|
}
|
|
67
94
|
if (this.heartbeatInFlight) return;
|
|
68
95
|
this.heartbeatInFlight = true;
|
|
96
|
+
const priorConfirmedUntilMs = this.confirmedUntilMs;
|
|
69
97
|
const renewalStartedAtMs = performance.now();
|
|
70
98
|
try {
|
|
71
99
|
const renewedUntil = await heartbeatCodexCredentialLeaseUntil(
|
|
@@ -79,6 +107,14 @@ export class CodexTurnLease {
|
|
|
79
107
|
);
|
|
80
108
|
if (!renewedUntil) {
|
|
81
109
|
this.markLost("not_found");
|
|
110
|
+
} else if (
|
|
111
|
+
codexCredentialLeaseDeadlineExpired(priorConfirmedUntilMs) ||
|
|
112
|
+
codexCredentialLeaseDeadlineExpired(renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS)
|
|
113
|
+
) {
|
|
114
|
+
// The database call may have waited on a lock until the prior lease
|
|
115
|
+
// expired. A late successful UPDATE is not proof that this worker
|
|
116
|
+
// owned the credential continuously; do not accept or extend it.
|
|
117
|
+
this.markLost("deadline");
|
|
82
118
|
} else {
|
|
83
119
|
this.confirmedUntilMs = renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS;
|
|
84
120
|
this.deps.observability.incrementCounter({
|
|
@@ -725,7 +725,22 @@ export function isExactStatuslessUpstreamConnectivityMessage(message: string): b
|
|
|
725
725
|
return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
|
|
726
726
|
}
|
|
727
727
|
|
|
728
|
+
function providerSafetyRefusalDiagnostic(error: unknown): string | undefined {
|
|
729
|
+
return collectErrorStrings(error).find(
|
|
730
|
+
(value) =>
|
|
731
|
+
/^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy)$/.test(
|
|
732
|
+
value,
|
|
733
|
+
) || /\bthis request was blocked by our safety systems\b/i.test(value),
|
|
734
|
+
);
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
function isProviderSafetyRefusal(error: unknown): boolean {
|
|
738
|
+
return providerSafetyRefusalDiagnostic(error) !== undefined;
|
|
739
|
+
}
|
|
740
|
+
|
|
728
741
|
export function isTransientProviderError(error: unknown): boolean {
|
|
742
|
+
// A semantic refusal can arrive inside a 5xx transport envelope.
|
|
743
|
+
if (isProviderSafetyRefusal(error)) return false;
|
|
729
744
|
const status =
|
|
730
745
|
typeof error === "object" && error !== null
|
|
731
746
|
? Number(
|
|
@@ -769,6 +784,7 @@ export type XaiCredentialFailure = {
|
|
|
769
784
|
* without an accepted model response; refresh relogin is equally definitive.
|
|
770
785
|
*/
|
|
771
786
|
export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailure | null {
|
|
787
|
+
if (isProviderSafetyRefusal(error)) return null;
|
|
772
788
|
let relogin: unknown = error;
|
|
773
789
|
for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
|
|
774
790
|
if (relogin instanceof XaiSubscriptionReloginRequired) {
|
|
@@ -844,6 +860,16 @@ export function agentRunFailurePayload(
|
|
|
844
860
|
historyPersistenceStage: error.stage,
|
|
845
861
|
};
|
|
846
862
|
}
|
|
863
|
+
const safetyRefusalDiagnostic = providerSafetyRefusalDiagnostic(error);
|
|
864
|
+
if (safetyRefusalDiagnostic !== undefined) {
|
|
865
|
+
return {
|
|
866
|
+
error:
|
|
867
|
+
"The model provider blocked this request through its safety systems. Automatic retries stopped.",
|
|
868
|
+
code: "provider_safety_refusal",
|
|
869
|
+
retryable: false,
|
|
870
|
+
detail: safetyRefusalDiagnostic,
|
|
871
|
+
};
|
|
872
|
+
}
|
|
847
873
|
const message = error instanceof Error ? error.message : String(error);
|
|
848
874
|
const status =
|
|
849
875
|
typeof error === "object" && error !== null
|
|
@@ -1106,6 +1132,8 @@ export function codexCredentialCooldownUntil(
|
|
|
1106
1132
|
* progress and therefore MUST NOT walk the credential pool automatically.
|
|
1107
1133
|
*/
|
|
1108
1134
|
export function classifyCodexCredentialFailure(error: unknown): CodexCredentialFailure | null {
|
|
1135
|
+
// A request safety refusal is not evidence that another account should run it.
|
|
1136
|
+
if (isProviderSafetyRefusal(error)) return null;
|
|
1109
1137
|
// A permanent OAuth refresh failure is definitive and the shared resolver has
|
|
1110
1138
|
// already fenced/stamped the exact credential version. The OpenAI client can
|
|
1111
1139
|
// wrap a rejection from its custom fetch in APIConnectionError, so recognize
|
|
@@ -1207,7 +1235,7 @@ export function codexUsageLimitFailurePayload(
|
|
|
1207
1235
|
info: { resetsInSeconds: number | null },
|
|
1208
1236
|
detail: string,
|
|
1209
1237
|
opts?: { allAccounts?: boolean },
|
|
1210
|
-
): { error: string; code: string; retryable:
|
|
1238
|
+
): { error: string; code: string; retryable: false; detail?: string } {
|
|
1211
1239
|
// P3: when EVERY connected subscription is rate-limited the message names the
|
|
1212
1240
|
// earliest reset across accounts; the single-account message is unchanged.
|
|
1213
1241
|
const error = opts?.allAccounts
|