@opengeni/worker-bundle 0.16.11 → 0.16.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "0.16.11",
3
+ "version": "0.16.13",
4
4
  "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -41,16 +41,16 @@
41
41
  "dependencies": {
42
42
  "@opengeni/agent-proto": "^0.3.0",
43
43
  "@opengeni/codex": "^0.2.10",
44
- "@opengeni/config": "^0.10.11",
45
- "@opengeni/contracts": "^0.38.1",
46
- "@opengeni/core": "^0.20.10",
47
- "@opengeni/db": "^0.27.7",
48
- "@opengeni/documents": "^0.5.3",
49
- "@opengeni/events": "^0.3.73",
50
- "@opengeni/github": "^0.4.27",
51
- "@opengeni/observability": "^0.4.14",
52
- "@opengeni/runtime": "^0.18.10",
53
- "@opengeni/storage": "^0.2.64",
44
+ "@opengeni/config": "^0.10.12",
45
+ "@opengeni/contracts": "^0.38.2",
46
+ "@opengeni/core": "^0.20.12",
47
+ "@opengeni/db": "^0.27.9",
48
+ "@opengeni/documents": "^0.5.5",
49
+ "@opengeni/events": "^0.3.75",
50
+ "@opengeni/github": "^0.4.28",
51
+ "@opengeni/observability": "^0.4.15",
52
+ "@opengeni/runtime": "^0.18.11",
53
+ "@opengeni/storage": "^0.2.65",
54
54
  "@temporalio/activity": "^1.17.0",
55
55
  "@temporalio/client": "^1.17.0",
56
56
  "@temporalio/worker": "^1.17.0",
@@ -357,11 +357,10 @@ import {
357
357
  } from "@opengeni/contracts";
358
358
  import { createHash, randomUUID } from "node:crypto";
359
359
 
360
- // How long the session workflow holds the loop after a retryable provider
361
- // failure before the goal continuation re-enters the model. Azure/OpenAI TPM
362
- // throttling is minute-granular; anything shorter mostly burns continuation
363
- // budget against the same window.
360
+ // Retryable provider connectivity/5xx failures start quickly and back off to
361
+ // this ceiling. Explicit rate limits retain the minute-granular fallback.
364
362
  export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
363
+ const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
365
364
 
366
365
  /** Broad personal lookup is allowed only for a direct human/API command. */
367
366
  export function credentialSubjectIdForTurnInitiator(
@@ -445,16 +444,93 @@ export function legacyTurnExecutionPolicyInput(
445
444
  /** A retryable provider fault recovers the accepted turn itself. Goal state is
446
445
  * irrelevant: autonomous continuation and infrastructure recovery are separate
447
446
  * concerns. */
448
- export function providerRecoveryResult(): {
447
+ export function providerRecoveryResult(input: {
448
+ failureCode: string | undefined;
449
+ attemptNumber: number;
450
+ retryAfterMs?: number | null;
451
+ }): {
449
452
  status: "recovering";
450
453
  continueDelayMs: number;
451
454
  } {
455
+ const providerDelay =
456
+ input.retryAfterMs !== null &&
457
+ input.retryAfterMs !== undefined &&
458
+ Number.isFinite(input.retryAfterMs) &&
459
+ input.retryAfterMs > 0
460
+ ? Math.ceil(input.retryAfterMs)
461
+ : null;
462
+ const continueDelayMs =
463
+ input.failureCode === "provider_rate_limited"
464
+ ? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
465
+ : input.failureCode === "provider_unavailable" ||
466
+ input.failureCode === "upstream_connectivity_unavailable"
467
+ ? Math.max(
468
+ providerDelay ?? 0,
469
+ PROVIDER_CONNECTIVITY_BACKOFF_MS[
470
+ Math.min(
471
+ Math.max(Math.trunc(input.attemptNumber) - 1, 0),
472
+ PROVIDER_CONNECTIVITY_BACKOFF_MS.length - 1,
473
+ )
474
+ ]!,
475
+ )
476
+ : PROVIDER_BACKPRESSURE_DELAY_MS;
452
477
  return {
453
478
  status: "recovering",
454
- continueDelayMs: PROVIDER_BACKPRESSURE_DELAY_MS,
479
+ continueDelayMs,
455
480
  };
456
481
  }
457
482
 
483
+ export function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number {
484
+ const value = metadata.providerRecoveryCount;
485
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
486
+ }
487
+
488
+ function headerValue(headers: unknown, name: string): string | null {
489
+ if (!headers || typeof headers !== "object") return null;
490
+ const getter = (headers as { get?: unknown }).get;
491
+ if (typeof getter === "function") {
492
+ const value = getter.call(headers, name);
493
+ return typeof value === "string" ? value : null;
494
+ }
495
+ const target = name.toLowerCase();
496
+ const entry = Object.entries(headers as Record<string, unknown>).find(
497
+ ([key, value]) => key.toLowerCase() === target && typeof value === "string",
498
+ );
499
+ return typeof entry?.[1] === "string" ? entry[1] : null;
500
+ }
501
+
502
+ /** Read a provider Retry-After hint without retaining response headers/body. */
503
+ export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number | null {
504
+ let current: unknown = error;
505
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
506
+ const value = current as Record<string, unknown>;
507
+ const body =
508
+ value.error && typeof value.error === "object"
509
+ ? (value.error as Record<string, unknown>)
510
+ : null;
511
+ const directSeconds = Number(
512
+ value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
513
+ );
514
+ const header =
515
+ headerValue(value.headers, "retry-after") ??
516
+ headerValue(value.responseHeaders, "retry-after") ??
517
+ headerValue(body?.headers, "retry-after");
518
+ const headerSeconds = header === null ? Number.NaN : Number(header);
519
+ const headerDate =
520
+ header !== null && !Number.isFinite(headerSeconds) ? Date.parse(header) : Number.NaN;
521
+ const seconds = Number.isFinite(directSeconds)
522
+ ? directSeconds
523
+ : Number.isFinite(headerSeconds)
524
+ ? headerSeconds
525
+ : Number.isFinite(headerDate)
526
+ ? Math.max(0, (headerDate - nowMs) / 1_000)
527
+ : Number.NaN;
528
+ if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1_000);
529
+ current = value.cause;
530
+ }
531
+ return null;
532
+ }
533
+
458
534
  /**
459
535
  * Tell the model when durable MCP policy references are unavailable for this
460
536
  * exact turn. The policy projection already bounds and validates these ids;
@@ -2378,6 +2454,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2378
2454
  let isCodexTurn = false;
2379
2455
  let isExternallyBilledTurn = false;
2380
2456
  let executionGeneration = 0;
2457
+ let providerRecoveryCount = 0;
2381
2458
  let modelRequestStarted = false;
2382
2459
  // Still required by credential-loss/capacity settlements, whose own
2383
2460
  // recovery transactions fence against worker-death redispatches.
@@ -3179,6 +3256,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3179
3256
  const turn = claim.turn;
3180
3257
  turnId = turn.id;
3181
3258
  executionGeneration = turn.executionGeneration;
3259
+ providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
3182
3260
  const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
3183
3261
  const policyForAbsent =
3184
3262
  claimedPolicy.kind === "valid"
@@ -8176,7 +8254,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8176
8254
  }
8177
8255
  if (failure.retryable && publish && turnId && turnStartedPublished) {
8178
8256
  try {
8179
- const recoveryResult = providerRecoveryResult();
8257
+ const nextProviderRecoveryCount = providerRecoveryCount + 1;
8258
+ const recoveryResult = providerRecoveryResult({
8259
+ failureCode: failure.code,
8260
+ attemptNumber: nextProviderRecoveryCount,
8261
+ retryAfterMs: providerRetryAfterMs(error),
8262
+ });
8180
8263
  await flushRuntimeBatcher();
8181
8264
  await reconcileConversationTruth({ requireDurable: true });
8182
8265
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
@@ -8185,9 +8268,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8185
8268
  triggerEventId: triggerEventId!,
8186
8269
  attemptId: input.attemptId,
8187
8270
  reason: failure.code ?? "provider_unavailable",
8271
+ providerRecoveryCount: nextProviderRecoveryCount,
8188
8272
  detail: {
8189
8273
  ...failure,
8190
8274
  continueDelayMs: recoveryResult.continueDelayMs,
8275
+ providerRecoveryCount: nextProviderRecoveryCount,
8191
8276
  },
8192
8277
  });
8193
8278
  if (recovery.action === "stale") {
@@ -8810,8 +8895,11 @@ function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean
8810
8895
 
8811
8896
  export function isTransientProviderError(error: unknown): boolean {
8812
8897
  const status =
8813
- typeof error === "object" && error !== null && "status" in error
8814
- ? Number((error as { status?: unknown }).status)
8898
+ typeof error === "object" && error !== null
8899
+ ? Number(
8900
+ (error as { status?: unknown; statusCode?: unknown }).status ??
8901
+ (error as { statusCode?: unknown }).statusCode,
8902
+ )
8815
8903
  : undefined;
8816
8904
  // A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
8817
8905
  // (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
@@ -8857,8 +8945,11 @@ export function agentRunFailurePayload(
8857
8945
  } {
8858
8946
  const message = error instanceof Error ? error.message : String(error);
8859
8947
  const status =
8860
- typeof error === "object" && error !== null && "status" in error
8861
- ? Number((error as { status?: unknown }).status)
8948
+ typeof error === "object" && error !== null
8949
+ ? Number(
8950
+ (error as { status?: unknown; statusCode?: unknown }).status ??
8951
+ (error as { statusCode?: unknown }).statusCode,
8952
+ )
8862
8953
  : undefined;
8863
8954
  const code =
8864
8955
  typeof error === "object" && error !== null && "code" in error
@@ -9062,13 +9153,15 @@ export function classifyCodexCredentialFailure(error: unknown): CodexCredentialF
9062
9153
  value.error && typeof value.error === "object"
9063
9154
  ? (value.error as Record<string, unknown>)
9064
9155
  : null;
9065
- const status = Number(value.status ?? body?.status);
9156
+ const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
9066
9157
  const code = String(value.code ?? body?.code ?? "").toLowerCase();
9067
9158
  const directRetryAfter = Number(
9068
9159
  value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
9069
9160
  );
9070
- const headers = value.headers as { get?: (name: string) => string | null } | undefined;
9071
- const retryAfterHeader = headers?.get?.("retry-after") ?? null;
9161
+ const retryAfterHeader =
9162
+ headerValue(value.headers, "retry-after") ??
9163
+ headerValue(value.responseHeaders, "retry-after") ??
9164
+ headerValue(body?.headers, "retry-after");
9072
9165
  const retryAfterNumber = retryAfterHeader === null ? Number.NaN : Number(retryAfterHeader);
9073
9166
  const retryAfterDate =
9074
9167
  retryAfterHeader !== null && !Number.isFinite(retryAfterNumber)
package/src/index.ts CHANGED
@@ -22,6 +22,7 @@ import {
22
22
  } from "@opengeni/observability";
23
23
  import {
24
24
  Connection,
25
+ isGrpcServiceError,
25
26
  ScheduleAlreadyRunning,
26
27
  ScheduleOverlapPolicy,
27
28
  Client as TemporalClient,
@@ -127,6 +128,13 @@ export function temporalActivityLeaseSettled(
127
128
  );
128
129
  }
129
130
 
131
+ /** The exact Temporal workflow run is absent, so it cannot retain an activity. */
132
+ export function temporalWorkflowExecutionNotFound(error: unknown): boolean {
133
+ // gRPC status code 5 is NOT_FOUND. The typed guard prevents unrelated
134
+ // provider/HTTP errors carrying a numeric code from proving quiescence.
135
+ return isGrpcServiceError(error) && error.code === 5;
136
+ }
137
+
130
138
  type WorkerWorkflowDefinition =
131
139
  | { workflowBundle: WorkflowBundleOption }
132
140
  | { workflowsPath: string };
@@ -312,10 +320,16 @@ export async function createWorkerWorkflowSignaler(
312
320
  // receipt and its exact wake revision atomically.
313
321
  },
314
322
  inspectSessionAttemptActivity: async ({ workflowId, workflowRunId, activityId }) => {
315
- const description = await connection.workflowService.describeWorkflowExecution({
316
- namespace: settings.temporalNamespace,
317
- execution: { workflowId, runId: workflowRunId },
318
- });
323
+ let description;
324
+ try {
325
+ description = await connection.workflowService.describeWorkflowExecution({
326
+ namespace: settings.temporalNamespace,
327
+ execution: { workflowId, runId: workflowRunId },
328
+ });
329
+ } catch (error) {
330
+ if (temporalWorkflowExecutionNotFound(error)) return "settled";
331
+ throw error;
332
+ }
319
333
  const pending = description.pendingActivities?.find(
320
334
  (activity) => activity.activityId === activityId,
321
335
  );