@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  2. package/dist/activities/agent-turn/claim.d.ts +7 -0
  3. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  4. package/dist/activities/agent-turn/errors.d.ts +6 -5
  5. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  6. package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
  7. package/dist/activities/agent-turn/history-sink.d.ts +6 -2
  8. package/dist/activities/agent-turn/quiescence.d.ts +13 -8
  9. package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
  10. package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
  11. package/dist/activities/agent-turn/turn-context.d.ts +9 -1
  12. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  13. package/dist/activities/codex-capacity.d.ts +32 -3
  14. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  15. package/dist/activities/codex-image-generation.d.ts +1 -1
  16. package/dist/activities/codex-rotation.d.ts +8 -67
  17. package/dist/activities/goals.d.ts +1 -1
  18. package/dist/activities/google-drive-publication.d.ts +2 -13
  19. package/dist/activities/image-generation-operation.d.ts +13 -4
  20. package/dist/activities/sandbox-lease.d.ts +7 -2
  21. package/dist/activities/session-state.d.ts +4 -2
  22. package/dist/activities/types.d.ts +11 -1
  23. package/dist/activities/xai-quota.d.ts +3 -0
  24. package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
  25. package/dist/activities-control-KS3ZPQVY.js.map +1 -0
  26. package/dist/activities-control.d.ts +2 -0
  27. package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
  28. package/dist/activities-turn-GHSFZB64.js.map +1 -0
  29. package/dist/activities.d.ts +2 -0
  30. package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
  31. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  32. package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
  33. package/dist/chunk-O2LQWJIK.js.map +1 -0
  34. package/dist/github-rest-mcp.d.ts +0 -2
  35. package/dist/index.d.ts +1 -0
  36. package/dist/index.js +2 -2
  37. package/dist/index.js.map +1 -1
  38. package/dist/observability-metrics.d.ts +1 -1
  39. package/dist/sandbox-routing.d.ts +6 -0
  40. package/dist/workflow-bundle.js +94 -9
  41. package/dist/workflows/activities.d.ts +1 -1
  42. package/dist/workflows/session.d.ts +3 -0
  43. package/package.json +17 -17
  44. package/src/activities/agent-turn/agent-build.ts +21 -26
  45. package/src/activities/agent-turn/claim.ts +26 -1
  46. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  47. package/src/activities/agent-turn/credential-leases.ts +36 -0
  48. package/src/activities/agent-turn/errors.ts +73 -12
  49. package/src/activities/agent-turn/failure-settlement.ts +472 -331
  50. package/src/activities/agent-turn/history-prefix.ts +51 -0
  51. package/src/activities/agent-turn/history-sink.ts +30 -6
  52. package/src/activities/agent-turn/quiescence.ts +29 -17
  53. package/src/activities/agent-turn/run.ts +22 -5
  54. package/src/activities/agent-turn/stream-attempt.ts +35 -8
  55. package/src/activities/agent-turn/tool-environment.ts +42 -32
  56. package/src/activities/agent-turn/tool-policy.ts +14 -8
  57. package/src/activities/agent-turn/turn-context.ts +13 -0
  58. package/src/activities/agent-turn/xai-capacity.ts +11 -0
  59. package/src/activities/codemode-dispatcher.ts +235 -59
  60. package/src/activities/codex-capacity.ts +107 -10
  61. package/src/activities/codex-fleet-shadow.ts +8 -1
  62. package/src/activities/codex-image-generation.ts +19 -2
  63. package/src/activities/codex-rotation.ts +45 -142
  64. package/src/activities/goals.ts +9 -9
  65. package/src/activities/google-drive-publication.ts +4 -81
  66. package/src/activities/image-generation-operation.ts +28 -3
  67. package/src/activities/run-input.ts +6 -7
  68. package/src/activities/sandbox-lease.ts +246 -87
  69. package/src/activities/session-state.ts +20 -1
  70. package/src/activities/types.ts +16 -8
  71. package/src/activities/xai-quota.ts +35 -0
  72. package/src/activities.ts +1 -0
  73. package/src/github-rest-mcp.ts +0 -82
  74. package/src/observability-metrics.ts +4 -0
  75. package/src/sandbox-routing.ts +186 -16
  76. package/src/workflows/activities.ts +1 -0
  77. package/src/workflows/session.ts +114 -8
  78. package/dist/activities/api-integrations.d.ts +0 -21
  79. package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
  80. package/dist/activities-turn-I4OPLQLS.js.map +0 -1
  81. package/dist/chunk-65JEWP2Q.js.map +0 -1
  82. package/dist/chunk-RQNH6CGR.js.map +0 -1
  83. package/src/activities/api-integrations.ts +0 -151
@@ -11,6 +11,15 @@ import { safeErrorDiagnostic } from "./errors";
11
11
  export type LeaseRenewReason = "timer" | "runtime_event" | "model_usage";
12
12
  export type LeaseLossReason = "deadline" | "not_found";
13
13
 
14
+ export class CodexCredentialLeaseLostError extends Error {
15
+ readonly code = "codex_credential_lease_lost";
16
+
17
+ constructor(readonly reason: LeaseLossReason) {
18
+ super("Codex credential lease is not usable for provider dispatch");
19
+ this.name = "CodexCredentialLeaseLostError";
20
+ }
21
+ }
22
+
14
23
  export type TurnCredentialLeaseDeps = {
15
24
  db: SharedActivityServices["db"];
16
25
  observability: SharedActivityServices["observability"];
@@ -28,6 +37,7 @@ export type TurnCredentialLeaseDeps = {
28
37
  export class CodexTurnLease {
29
38
  held = false;
30
39
  lost = false;
40
+ lossReason: LeaseLossReason | null = null;
31
41
  holderId: string | null = null;
32
42
  generation: number | null = null;
33
43
  // Monotonic worker deadline, not a comparison between the Postgres and
@@ -43,6 +53,7 @@ export class CodexTurnLease {
43
53
  markLost = (reason: LeaseLossReason): void => {
44
54
  if (this.lost) return;
45
55
  this.lost = true;
56
+ this.lossReason = reason;
46
57
  this.deps.observability.incrementCounter({
47
58
  name: "opengeni_codex_lease_renewals_total",
48
59
  help: "Codex lease renewal checkpoints by outcome and reason.",
@@ -55,6 +66,22 @@ export class CodexTurnLease {
55
66
  });
56
67
  };
57
68
 
69
+ /** Fail closed at the last worker-confirmed lease deadline before dispatch. */
70
+ assertUsable = (): void => {
71
+ if (this.lost) {
72
+ throw new CodexCredentialLeaseLostError(this.lossReason ?? "not_found");
73
+ }
74
+ if (
75
+ !this.held ||
76
+ this.holderId === null ||
77
+ this.generation === null ||
78
+ codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)
79
+ ) {
80
+ this.markLost("deadline");
81
+ throw new CodexCredentialLeaseLostError("deadline");
82
+ }
83
+ };
84
+
58
85
  renew = async (reason: LeaseRenewReason): Promise<void> => {
59
86
  const turnId = this.deps.getTurnId();
60
87
  if (!turnId || !this.held || !this.holderId || this.generation === null || this.lost) {
@@ -66,6 +93,7 @@ export class CodexTurnLease {
66
93
  }
67
94
  if (this.heartbeatInFlight) return;
68
95
  this.heartbeatInFlight = true;
96
+ const priorConfirmedUntilMs = this.confirmedUntilMs;
69
97
  const renewalStartedAtMs = performance.now();
70
98
  try {
71
99
  const renewedUntil = await heartbeatCodexCredentialLeaseUntil(
@@ -79,6 +107,14 @@ export class CodexTurnLease {
79
107
  );
80
108
  if (!renewedUntil) {
81
109
  this.markLost("not_found");
110
+ } else if (
111
+ codexCredentialLeaseDeadlineExpired(priorConfirmedUntilMs) ||
112
+ codexCredentialLeaseDeadlineExpired(renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS)
113
+ ) {
114
+ // The database call may have waited on a lock until the prior lease
115
+ // expired. A late successful UPDATE is not proof that this worker
116
+ // owned the credential continuously; do not accept or extend it.
117
+ this.markLost("deadline");
82
118
  } else {
83
119
  this.confirmedUntilMs = renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS;
84
120
  this.deps.observability.incrementCounter({
@@ -1,6 +1,8 @@
1
1
  import {
2
2
  ActiveSessionHistoryLimitExceededError,
3
3
  ApprovalRunStateLimitExceededError,
4
+ nestedPostgresSqlState,
5
+ safeDatabaseErrorFacts,
4
6
  isRetryableDatabaseTransportFailure,
5
7
  isSessionEventPersistenceError,
6
8
  SandboxLeaseTransitionError,
@@ -266,22 +268,21 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
266
268
  * Convert the atomic claim transaction's failure into a small, stable
267
269
  * Temporal wire contract. The original error remains in activity diagnostics,
268
270
  * but SQL text, parameters, and arbitrary invariant messages never enter
269
- * workflow history. Contention and operational database unavailability are
270
- * safe to re-read after backoff because the claim transaction contains no
271
- * model/tool effects; permanent database and state failures require terminal
272
- * settlement.
271
+ * workflow history. Every database failure is safe to re-read after backoff
272
+ * because the claim transaction contains no model/tool effects. Database
273
+ * constraints and authorization guards can also be repaired by a deployment or
274
+ * authority refresh, so they must preserve pending work. Only a non-database
275
+ * claim invariant is terminal.
273
276
  */
274
277
  export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
275
278
  const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
276
279
  const retryableCode = retryableDatabaseFailureCode(error);
277
- // The database transaction itself retries only the two contention failures
278
- // proven safe for immediate replay. Once the activity has failed, the
279
- // workflow may also retry operational outages after a durable re-read and
280
- // bounded delay. Unknown driver/database failures stay recoverable because
281
- // they commonly represent a lost connection; known constraint, auth, and
282
- // application SQLSTATEs are permanent and must not create an infinite loop.
280
+ // The workflow retries after a durable re-read and bounded delay. A
281
+ // SessionEventPersistenceError proves the failure happened inside the atomic
282
+ // claim boundary, before provider or tool work; retaining it is safer than
283
+ // destroying accepted machine input on an application SQLSTATE.
283
284
  const detail: PreClaimFailureDetail = {
284
- disposition: retryableCode ? "retryable" : "permanent",
285
+ disposition: persistenceFailure || retryableCode ? "retryable" : "permanent",
285
286
  code: retryableCode ?? persistenceFailure?.details.code ?? "claim_invariant",
286
287
  };
287
288
  return ApplicationFailure.create({
@@ -725,7 +726,22 @@ export function isExactStatuslessUpstreamConnectivityMessage(message: string): b
725
726
  return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
726
727
  }
727
728
 
729
+ function providerSafetyRefusalDiagnostic(error: unknown): string | undefined {
730
+ return collectErrorStrings(error).find(
731
+ (value) =>
732
+ /^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy)$/.test(
733
+ value,
734
+ ) || /\bthis request was blocked by our safety systems\b/i.test(value),
735
+ );
736
+ }
737
+
738
+ function isProviderSafetyRefusal(error: unknown): boolean {
739
+ return providerSafetyRefusalDiagnostic(error) !== undefined;
740
+ }
741
+
728
742
  export function isTransientProviderError(error: unknown): boolean {
743
+ // A semantic refusal can arrive inside a 5xx transport envelope.
744
+ if (isProviderSafetyRefusal(error)) return false;
729
745
  const status =
730
746
  typeof error === "object" && error !== null
731
747
  ? Number(
@@ -769,6 +785,7 @@ export type XaiCredentialFailure = {
769
785
  * without an accepted model response; refresh relogin is equally definitive.
770
786
  */
771
787
  export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailure | null {
788
+ if (isProviderSafetyRefusal(error)) return null;
772
789
  let relogin: unknown = error;
773
790
  for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
774
791
  if (relogin instanceof XaiSubscriptionReloginRequired) {
@@ -810,6 +827,26 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
810
827
  return null;
811
828
  }
812
829
 
830
+ // The generic turn-failure boundary also receives application/provider errors.
831
+ // A five-character code or generic severity alone does not establish a driver error.
832
+ function findPostgresDriverError(error: unknown): Record<string, unknown> | null {
833
+ const queue: unknown[] = [error];
834
+ const seen = new Set<unknown>();
835
+ for (let index = 0; index < queue.length && index < 64; index += 1) {
836
+ const current = queue[index];
837
+ if (!current || typeof current !== "object" || seen.has(current)) continue;
838
+ seen.add(current);
839
+ const record = current as Record<string, unknown>;
840
+ if (record.name === "PostgresError") return record;
841
+ for (const key of ["cause", "original", "driverError", "error", "errors"]) {
842
+ const nested = record[key];
843
+ if (Array.isArray(nested)) queue.push(...nested.slice(0, 64));
844
+ else if (nested !== undefined) queue.push(nested);
845
+ }
846
+ }
847
+ return null;
848
+ }
849
+
813
850
  export function agentRunFailurePayload(
814
851
  error: unknown,
815
852
  options: { isCodexTurn?: boolean } = {},
@@ -844,6 +881,16 @@ export function agentRunFailurePayload(
844
881
  historyPersistenceStage: error.stage,
845
882
  };
846
883
  }
884
+ const safetyRefusalDiagnostic = providerSafetyRefusalDiagnostic(error);
885
+ if (safetyRefusalDiagnostic !== undefined) {
886
+ return {
887
+ error:
888
+ "The model provider blocked this request through its safety systems. Automatic retries stopped.",
889
+ code: "provider_safety_refusal",
890
+ retryable: false,
891
+ detail: safetyRefusalDiagnostic,
892
+ };
893
+ }
847
894
  const message = error instanceof Error ? error.message : String(error);
848
895
  const status =
849
896
  typeof error === "object" && error !== null
@@ -1040,6 +1087,18 @@ export function agentRunFailurePayload(
1040
1087
  }
1041
1088
  return { error: message, code: "provider_unavailable", retryable: true };
1042
1089
  }
1090
+ const postgresDriverError = findPostgresDriverError(error);
1091
+ if (postgresDriverError) {
1092
+ const database = safeDatabaseErrorFacts(postgresDriverError);
1093
+ const sqlState = nestedPostgresSqlState(postgresDriverError);
1094
+ if (sqlState !== null || Object.keys(database).length > 0) {
1095
+ return {
1096
+ error: message,
1097
+ sqlState,
1098
+ ...(Object.keys(database).length > 0 ? { database } : {}),
1099
+ };
1100
+ }
1101
+ }
1043
1102
  return { error: message };
1044
1103
  }
1045
1104
 
@@ -1106,6 +1165,8 @@ export function codexCredentialCooldownUntil(
1106
1165
  * progress and therefore MUST NOT walk the credential pool automatically.
1107
1166
  */
1108
1167
  export function classifyCodexCredentialFailure(error: unknown): CodexCredentialFailure | null {
1168
+ // A request safety refusal is not evidence that another account should run it.
1169
+ if (isProviderSafetyRefusal(error)) return null;
1109
1170
  // A permanent OAuth refresh failure is definitive and the shared resolver has
1110
1171
  // already fenced/stamped the exact credential version. The OpenAI client can
1111
1172
  // wrap a rejection from its custom fetch in APIConnectionError, so recognize
@@ -1207,7 +1268,7 @@ export function codexUsageLimitFailurePayload(
1207
1268
  info: { resetsInSeconds: number | null },
1208
1269
  detail: string,
1209
1270
  opts?: { allAccounts?: boolean },
1210
- ): { error: string; code: string; retryable: boolean; detail?: string } {
1271
+ ): { error: string; code: string; retryable: false; detail?: string } {
1211
1272
  // P3: when EVERY connected subscription is rate-limited the message names the
1212
1273
  // earliest reset across accounts; the single-account message is unchanged.
1213
1274
  const error = opts?.allAccounts