@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +7 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +6 -5
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +13 -8
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/goals.d.ts +1 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +11 -1
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +94 -9
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +21 -26
- package/src/activities/agent-turn/claim.ts +26 -1
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +73 -12
- package/src/activities/agent-turn/failure-settlement.ts +472 -331
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +29 -17
- package/src/activities/agent-turn/run.ts +22 -5
- package/src/activities/agent-turn/stream-attempt.ts +35 -8
- package/src/activities/agent-turn/tool-environment.ts +42 -32
- package/src/activities/agent-turn/tool-policy.ts +14 -8
- package/src/activities/agent-turn/turn-context.ts +13 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +107 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/goals.ts +9 -9
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +246 -87
- package/src/activities/session-state.ts +20 -1
- package/src/activities/types.ts +16 -8
- package/src/activities/xai-quota.ts +35 -0
- package/src/activities.ts +1 -0
- package/src/github-rest-mcp.ts +0 -82
- package/src/observability-metrics.ts +4 -0
- package/src/sandbox-routing.ts +186 -16
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +114 -8
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
- package/dist/activities-turn-I4OPLQLS.js.map +0 -1
- package/dist/chunk-65JEWP2Q.js.map +0 -1
- package/dist/chunk-RQNH6CGR.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -151
|
@@ -11,6 +11,15 @@ import { safeErrorDiagnostic } from "./errors";
|
|
|
11
11
|
export type LeaseRenewReason = "timer" | "runtime_event" | "model_usage";
|
|
12
12
|
export type LeaseLossReason = "deadline" | "not_found";
|
|
13
13
|
|
|
14
|
+
export class CodexCredentialLeaseLostError extends Error {
|
|
15
|
+
readonly code = "codex_credential_lease_lost";
|
|
16
|
+
|
|
17
|
+
constructor(readonly reason: LeaseLossReason) {
|
|
18
|
+
super("Codex credential lease is not usable for provider dispatch");
|
|
19
|
+
this.name = "CodexCredentialLeaseLostError";
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
14
23
|
export type TurnCredentialLeaseDeps = {
|
|
15
24
|
db: SharedActivityServices["db"];
|
|
16
25
|
observability: SharedActivityServices["observability"];
|
|
@@ -28,6 +37,7 @@ export type TurnCredentialLeaseDeps = {
|
|
|
28
37
|
export class CodexTurnLease {
|
|
29
38
|
held = false;
|
|
30
39
|
lost = false;
|
|
40
|
+
lossReason: LeaseLossReason | null = null;
|
|
31
41
|
holderId: string | null = null;
|
|
32
42
|
generation: number | null = null;
|
|
33
43
|
// Monotonic worker deadline, not a comparison between the Postgres and
|
|
@@ -43,6 +53,7 @@ export class CodexTurnLease {
|
|
|
43
53
|
markLost = (reason: LeaseLossReason): void => {
|
|
44
54
|
if (this.lost) return;
|
|
45
55
|
this.lost = true;
|
|
56
|
+
this.lossReason = reason;
|
|
46
57
|
this.deps.observability.incrementCounter({
|
|
47
58
|
name: "opengeni_codex_lease_renewals_total",
|
|
48
59
|
help: "Codex lease renewal checkpoints by outcome and reason.",
|
|
@@ -55,6 +66,22 @@ export class CodexTurnLease {
|
|
|
55
66
|
});
|
|
56
67
|
};
|
|
57
68
|
|
|
69
|
+
/** Fail closed at the last worker-confirmed lease deadline before dispatch. */
|
|
70
|
+
assertUsable = (): void => {
|
|
71
|
+
if (this.lost) {
|
|
72
|
+
throw new CodexCredentialLeaseLostError(this.lossReason ?? "not_found");
|
|
73
|
+
}
|
|
74
|
+
if (
|
|
75
|
+
!this.held ||
|
|
76
|
+
this.holderId === null ||
|
|
77
|
+
this.generation === null ||
|
|
78
|
+
codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)
|
|
79
|
+
) {
|
|
80
|
+
this.markLost("deadline");
|
|
81
|
+
throw new CodexCredentialLeaseLostError("deadline");
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
|
|
58
85
|
renew = async (reason: LeaseRenewReason): Promise<void> => {
|
|
59
86
|
const turnId = this.deps.getTurnId();
|
|
60
87
|
if (!turnId || !this.held || !this.holderId || this.generation === null || this.lost) {
|
|
@@ -66,6 +93,7 @@ export class CodexTurnLease {
|
|
|
66
93
|
}
|
|
67
94
|
if (this.heartbeatInFlight) return;
|
|
68
95
|
this.heartbeatInFlight = true;
|
|
96
|
+
const priorConfirmedUntilMs = this.confirmedUntilMs;
|
|
69
97
|
const renewalStartedAtMs = performance.now();
|
|
70
98
|
try {
|
|
71
99
|
const renewedUntil = await heartbeatCodexCredentialLeaseUntil(
|
|
@@ -79,6 +107,14 @@ export class CodexTurnLease {
|
|
|
79
107
|
);
|
|
80
108
|
if (!renewedUntil) {
|
|
81
109
|
this.markLost("not_found");
|
|
110
|
+
} else if (
|
|
111
|
+
codexCredentialLeaseDeadlineExpired(priorConfirmedUntilMs) ||
|
|
112
|
+
codexCredentialLeaseDeadlineExpired(renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS)
|
|
113
|
+
) {
|
|
114
|
+
// The database call may have waited on a lock until the prior lease
|
|
115
|
+
// expired. A late successful UPDATE is not proof that this worker
|
|
116
|
+
// owned the credential continuously; do not accept or extend it.
|
|
117
|
+
this.markLost("deadline");
|
|
82
118
|
} else {
|
|
83
119
|
this.confirmedUntilMs = renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS;
|
|
84
120
|
this.deps.observability.incrementCounter({
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ActiveSessionHistoryLimitExceededError,
|
|
3
3
|
ApprovalRunStateLimitExceededError,
|
|
4
|
+
nestedPostgresSqlState,
|
|
5
|
+
safeDatabaseErrorFacts,
|
|
4
6
|
isRetryableDatabaseTransportFailure,
|
|
5
7
|
isSessionEventPersistenceError,
|
|
6
8
|
SandboxLeaseTransitionError,
|
|
@@ -266,22 +268,21 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
|
|
|
266
268
|
* Convert the atomic claim transaction's failure into a small, stable
|
|
267
269
|
* Temporal wire contract. The original error remains in activity diagnostics,
|
|
268
270
|
* but SQL text, parameters, and arbitrary invariant messages never enter
|
|
269
|
-
* workflow history.
|
|
270
|
-
*
|
|
271
|
-
*
|
|
272
|
-
*
|
|
271
|
+
* workflow history. Every database failure is safe to re-read after backoff
|
|
272
|
+
* because the claim transaction contains no model/tool effects. Database
|
|
273
|
+
* constraints and authorization guards can also be repaired by a deployment or
|
|
274
|
+
* authority refresh, so they must preserve pending work. Only a non-database
|
|
275
|
+
* claim invariant is terminal.
|
|
273
276
|
*/
|
|
274
277
|
export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
|
|
275
278
|
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
276
279
|
const retryableCode = retryableDatabaseFailureCode(error);
|
|
277
|
-
// The
|
|
278
|
-
//
|
|
279
|
-
//
|
|
280
|
-
//
|
|
281
|
-
// they commonly represent a lost connection; known constraint, auth, and
|
|
282
|
-
// application SQLSTATEs are permanent and must not create an infinite loop.
|
|
280
|
+
// The workflow retries after a durable re-read and bounded delay. A
|
|
281
|
+
// SessionEventPersistenceError proves the failure happened inside the atomic
|
|
282
|
+
// claim boundary, before provider or tool work; retaining it is safer than
|
|
283
|
+
// destroying accepted machine input on an application SQLSTATE.
|
|
283
284
|
const detail: PreClaimFailureDetail = {
|
|
284
|
-
disposition: retryableCode ? "retryable" : "permanent",
|
|
285
|
+
disposition: persistenceFailure || retryableCode ? "retryable" : "permanent",
|
|
285
286
|
code: retryableCode ?? persistenceFailure?.details.code ?? "claim_invariant",
|
|
286
287
|
};
|
|
287
288
|
return ApplicationFailure.create({
|
|
@@ -725,7 +726,22 @@ export function isExactStatuslessUpstreamConnectivityMessage(message: string): b
|
|
|
725
726
|
return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
|
|
726
727
|
}
|
|
727
728
|
|
|
729
|
+
function providerSafetyRefusalDiagnostic(error: unknown): string | undefined {
|
|
730
|
+
return collectErrorStrings(error).find(
|
|
731
|
+
(value) =>
|
|
732
|
+
/^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy)$/.test(
|
|
733
|
+
value,
|
|
734
|
+
) || /\bthis request was blocked by our safety systems\b/i.test(value),
|
|
735
|
+
);
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
function isProviderSafetyRefusal(error: unknown): boolean {
|
|
739
|
+
return providerSafetyRefusalDiagnostic(error) !== undefined;
|
|
740
|
+
}
|
|
741
|
+
|
|
728
742
|
export function isTransientProviderError(error: unknown): boolean {
|
|
743
|
+
// A semantic refusal can arrive inside a 5xx transport envelope.
|
|
744
|
+
if (isProviderSafetyRefusal(error)) return false;
|
|
729
745
|
const status =
|
|
730
746
|
typeof error === "object" && error !== null
|
|
731
747
|
? Number(
|
|
@@ -769,6 +785,7 @@ export type XaiCredentialFailure = {
|
|
|
769
785
|
* without an accepted model response; refresh relogin is equally definitive.
|
|
770
786
|
*/
|
|
771
787
|
export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailure | null {
|
|
788
|
+
if (isProviderSafetyRefusal(error)) return null;
|
|
772
789
|
let relogin: unknown = error;
|
|
773
790
|
for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
|
|
774
791
|
if (relogin instanceof XaiSubscriptionReloginRequired) {
|
|
@@ -810,6 +827,26 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
|
|
|
810
827
|
return null;
|
|
811
828
|
}
|
|
812
829
|
|
|
830
|
+
// The generic turn-failure boundary also receives application/provider errors.
|
|
831
|
+
// A five-character code or generic severity alone does not establish a driver error.
|
|
832
|
+
function findPostgresDriverError(error: unknown): Record<string, unknown> | null {
|
|
833
|
+
const queue: unknown[] = [error];
|
|
834
|
+
const seen = new Set<unknown>();
|
|
835
|
+
for (let index = 0; index < queue.length && index < 64; index += 1) {
|
|
836
|
+
const current = queue[index];
|
|
837
|
+
if (!current || typeof current !== "object" || seen.has(current)) continue;
|
|
838
|
+
seen.add(current);
|
|
839
|
+
const record = current as Record<string, unknown>;
|
|
840
|
+
if (record.name === "PostgresError") return record;
|
|
841
|
+
for (const key of ["cause", "original", "driverError", "error", "errors"]) {
|
|
842
|
+
const nested = record[key];
|
|
843
|
+
if (Array.isArray(nested)) queue.push(...nested.slice(0, 64));
|
|
844
|
+
else if (nested !== undefined) queue.push(nested);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
return null;
|
|
848
|
+
}
|
|
849
|
+
|
|
813
850
|
export function agentRunFailurePayload(
|
|
814
851
|
error: unknown,
|
|
815
852
|
options: { isCodexTurn?: boolean } = {},
|
|
@@ -844,6 +881,16 @@ export function agentRunFailurePayload(
|
|
|
844
881
|
historyPersistenceStage: error.stage,
|
|
845
882
|
};
|
|
846
883
|
}
|
|
884
|
+
const safetyRefusalDiagnostic = providerSafetyRefusalDiagnostic(error);
|
|
885
|
+
if (safetyRefusalDiagnostic !== undefined) {
|
|
886
|
+
return {
|
|
887
|
+
error:
|
|
888
|
+
"The model provider blocked this request through its safety systems. Automatic retries stopped.",
|
|
889
|
+
code: "provider_safety_refusal",
|
|
890
|
+
retryable: false,
|
|
891
|
+
detail: safetyRefusalDiagnostic,
|
|
892
|
+
};
|
|
893
|
+
}
|
|
847
894
|
const message = error instanceof Error ? error.message : String(error);
|
|
848
895
|
const status =
|
|
849
896
|
typeof error === "object" && error !== null
|
|
@@ -1040,6 +1087,18 @@ export function agentRunFailurePayload(
|
|
|
1040
1087
|
}
|
|
1041
1088
|
return { error: message, code: "provider_unavailable", retryable: true };
|
|
1042
1089
|
}
|
|
1090
|
+
const postgresDriverError = findPostgresDriverError(error);
|
|
1091
|
+
if (postgresDriverError) {
|
|
1092
|
+
const database = safeDatabaseErrorFacts(postgresDriverError);
|
|
1093
|
+
const sqlState = nestedPostgresSqlState(postgresDriverError);
|
|
1094
|
+
if (sqlState !== null || Object.keys(database).length > 0) {
|
|
1095
|
+
return {
|
|
1096
|
+
error: message,
|
|
1097
|
+
sqlState,
|
|
1098
|
+
...(Object.keys(database).length > 0 ? { database } : {}),
|
|
1099
|
+
};
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1043
1102
|
return { error: message };
|
|
1044
1103
|
}
|
|
1045
1104
|
|
|
@@ -1106,6 +1165,8 @@ export function codexCredentialCooldownUntil(
|
|
|
1106
1165
|
* progress and therefore MUST NOT walk the credential pool automatically.
|
|
1107
1166
|
*/
|
|
1108
1167
|
export function classifyCodexCredentialFailure(error: unknown): CodexCredentialFailure | null {
|
|
1168
|
+
// A request safety refusal is not evidence that another account should run it.
|
|
1169
|
+
if (isProviderSafetyRefusal(error)) return null;
|
|
1109
1170
|
// A permanent OAuth refresh failure is definitive and the shared resolver has
|
|
1110
1171
|
// already fenced/stamped the exact credential version. The OpenAI client can
|
|
1111
1172
|
// wrap a rejection from its custom fetch in APIConnectionError, so recognize
|
|
@@ -1207,7 +1268,7 @@ export function codexUsageLimitFailurePayload(
|
|
|
1207
1268
|
info: { resetsInSeconds: number | null },
|
|
1208
1269
|
detail: string,
|
|
1209
1270
|
opts?: { allAccounts?: boolean },
|
|
1210
|
-
): { error: string; code: string; retryable:
|
|
1271
|
+
): { error: string; code: string; retryable: false; detail?: string } {
|
|
1211
1272
|
// P3: when EVERY connected subscription is rate-limited the message names the
|
|
1212
1273
|
// earliest reset across accounts; the single-account message is unchanged.
|
|
1213
1274
|
const error = opts?.allAccounts
|