@opengeni/worker-bundle 0.16.11 → 0.16.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -37,10 +37,17 @@ export declare function legacyTurnExecutionPolicyInput(turn: Pick<SessionTurn, "
37
37
  /** A retryable provider fault recovers the accepted turn itself. Goal state is
38
38
  * irrelevant: autonomous continuation and infrastructure recovery are separate
39
39
  * concerns. */
40
- export declare function providerRecoveryResult(): {
40
+ export declare function providerRecoveryResult(input: {
41
+ failureCode: string | undefined;
42
+ attemptNumber: number;
43
+ retryAfterMs?: number | null;
44
+ }): {
41
45
  status: "recovering";
42
46
  continueDelayMs: number;
43
47
  };
48
+ export declare function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number;
49
+ /** Read a provider Retry-After hint without retaining response headers/body. */
50
+ export declare function providerRetryAfterMs(error: unknown, nowMs?: number): number | null;
44
51
  /**
45
52
  * Tell the model when durable MCP policy references are unavailable for this
46
53
  * exact turn. The policy projection already bounds and validates these ids;
package/dist/index.d.ts CHANGED
@@ -35,6 +35,8 @@ type TemporalActivityLease = {
35
35
  } | null;
36
36
  };
37
37
  export declare function temporalActivityLeaseSettled(pending: TemporalActivityLease | null | undefined, nowMs?: number): boolean;
38
+ /** The exact Temporal workflow run is absent, so it cannot retain an activity. */
39
+ export declare function temporalWorkflowExecutionNotFound(error: unknown): boolean;
38
40
  type WorkerWorkflowDefinition = {
39
41
  workflowBundle: WorkflowBundleOption;
40
42
  } | {
package/dist/index.js CHANGED
@@ -23,6 +23,7 @@ import {
23
23
  } from "@opengeni/observability";
24
24
  import {
25
25
  Connection,
26
+ isGrpcServiceError,
26
27
  ScheduleAlreadyRunning,
27
28
  ScheduleOverlapPolicy,
28
29
  Client as TemporalClient
@@ -6443,6 +6444,7 @@ import {
6443
6444
  } from "@opengeni/contracts";
6444
6445
  import { createHash as createHash4, randomUUID as randomUUID2 } from "crypto";
6445
6446
  var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
6447
+ var PROVIDER_CONNECTIVITY_BACKOFF_MS = [2e3, 5e3, 15e3, 3e4, 6e4];
6446
6448
  function credentialSubjectIdForTurnInitiator(turn) {
6447
6449
  return directPersonalConnectionSubjectId(turn);
6448
6450
  }
@@ -6487,12 +6489,54 @@ function legacyTurnExecutionPolicyInput(turn) {
6487
6489
  latencyModeSource: explicit ? "explicit" : "continuation"
6488
6490
  };
6489
6491
  }
6490
- function providerRecoveryResult() {
6492
+ function providerRecoveryResult(input) {
6493
+ const providerDelay = input.retryAfterMs !== null && input.retryAfterMs !== void 0 && Number.isFinite(input.retryAfterMs) && input.retryAfterMs > 0 ? Math.ceil(input.retryAfterMs) : null;
6494
+ const continueDelayMs = input.failureCode === "provider_rate_limited" ? providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS : input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" ? Math.max(
6495
+ providerDelay ?? 0,
6496
+ PROVIDER_CONNECTIVITY_BACKOFF_MS[Math.min(
6497
+ Math.max(Math.trunc(input.attemptNumber) - 1, 0),
6498
+ PROVIDER_CONNECTIVITY_BACKOFF_MS.length - 1
6499
+ )]
6500
+ ) : PROVIDER_BACKPRESSURE_DELAY_MS;
6491
6501
  return {
6492
6502
  status: "recovering",
6493
- continueDelayMs: PROVIDER_BACKPRESSURE_DELAY_MS
6503
+ continueDelayMs
6494
6504
  };
6495
6505
  }
6506
+ function providerRecoveryCountFromMetadata(metadata) {
6507
+ const value = metadata.providerRecoveryCount;
6508
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
6509
+ }
6510
+ function headerValue(headers, name) {
6511
+ if (!headers || typeof headers !== "object") return null;
6512
+ const getter = headers.get;
6513
+ if (typeof getter === "function") {
6514
+ const value = getter.call(headers, name);
6515
+ return typeof value === "string" ? value : null;
6516
+ }
6517
+ const target = name.toLowerCase();
6518
+ const entry = Object.entries(headers).find(
6519
+ ([key, value]) => key.toLowerCase() === target && typeof value === "string"
6520
+ );
6521
+ return typeof entry?.[1] === "string" ? entry[1] : null;
6522
+ }
6523
+ function providerRetryAfterMs(error, nowMs = Date.now()) {
6524
+ let current = error;
6525
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
6526
+ const value = current;
6527
+ const body = value.error && typeof value.error === "object" ? value.error : null;
6528
+ const directSeconds = Number(
6529
+ value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds
6530
+ );
6531
+ const header = headerValue(value.headers, "retry-after") ?? headerValue(value.responseHeaders, "retry-after") ?? headerValue(body?.headers, "retry-after");
6532
+ const headerSeconds = header === null ? Number.NaN : Number(header);
6533
+ const headerDate = header !== null && !Number.isFinite(headerSeconds) ? Date.parse(header) : Number.NaN;
6534
+ const seconds = Number.isFinite(directSeconds) ? directSeconds : Number.isFinite(headerSeconds) ? headerSeconds : Number.isFinite(headerDate) ? Math.max(0, (headerDate - nowMs) / 1e3) : Number.NaN;
6535
+ if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1e3);
6536
+ current = value.cause;
6537
+ }
6538
+ return null;
6539
+ }
6496
6540
  function unavailableMcpTurnInstructions(input) {
6497
6541
  if (input.droppedCount <= 0) {
6498
6542
  return void 0;
@@ -7483,6 +7527,7 @@ function createRunAgentTurnActivity(services) {
7483
7527
  let isCodexTurn = false;
7484
7528
  let isExternallyBilledTurn = false;
7485
7529
  let executionGeneration = 0;
7530
+ let providerRecoveryCount = 0;
7486
7531
  let modelRequestStarted = false;
7487
7532
  let redispatchesAtDispatch = 0;
7488
7533
  const setLastInputTokensFenced = async (lastInputTokens) => {
@@ -8055,6 +8100,7 @@ function createRunAgentTurnActivity(services) {
8055
8100
  const turn = claim.turn;
8056
8101
  turnId = turn.id;
8057
8102
  executionGeneration = turn.executionGeneration;
8103
+ providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
8058
8104
  const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
8059
8105
  const policyForAbsent = claimedPolicy.kind === "valid" ? claimedPolicy.policy : resolveTurnExecutionPolicyV1(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
8060
8106
  const installedPolicy = await installOrReadTurnExecutionPolicyForAttempt(db, {
@@ -11993,7 +12039,12 @@ function createRunAgentTurnActivity(services) {
11993
12039
  }
11994
12040
  if (failure.retryable && publish && turnId && turnStartedPublished) {
11995
12041
  try {
11996
- const recoveryResult = providerRecoveryResult();
12042
+ const nextProviderRecoveryCount = providerRecoveryCount + 1;
12043
+ const recoveryResult = providerRecoveryResult({
12044
+ failureCode: failure.code,
12045
+ attemptNumber: nextProviderRecoveryCount,
12046
+ retryAfterMs: providerRetryAfterMs(error)
12047
+ });
11997
12048
  await flushRuntimeBatcher();
11998
12049
  await reconcileConversationTruth({ requireDurable: true });
11999
12050
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
@@ -12002,9 +12053,11 @@ function createRunAgentTurnActivity(services) {
12002
12053
  triggerEventId,
12003
12054
  attemptId: input.attemptId,
12004
12055
  reason: failure.code ?? "provider_unavailable",
12056
+ providerRecoveryCount: nextProviderRecoveryCount,
12005
12057
  detail: {
12006
12058
  ...failure,
12007
- continueDelayMs: recoveryResult.continueDelayMs
12059
+ continueDelayMs: recoveryResult.continueDelayMs,
12060
+ providerRecoveryCount: nextProviderRecoveryCount
12008
12061
  }
12009
12062
  });
12010
12063
  if (recovery.action === "stale") {
@@ -12479,7 +12532,9 @@ function isExactStatuslessUpstreamConnectivityMessage(message) {
12479
12532
  return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
12480
12533
  }
12481
12534
  function isTransientProviderError(error) {
12482
- const status = typeof error === "object" && error !== null && "status" in error ? Number(error.status) : void 0;
12535
+ const status = typeof error === "object" && error !== null ? Number(
12536
+ error.status ?? error.statusCode
12537
+ ) : void 0;
12483
12538
  if (status !== void 0 && Number.isFinite(status)) {
12484
12539
  return status >= 500 && status < 600;
12485
12540
  }
@@ -12497,7 +12552,9 @@ function isTransientProviderError(error) {
12497
12552
  }
12498
12553
  function agentRunFailurePayload(error, options = {}) {
12499
12554
  const message = error instanceof Error ? error.message : String(error);
12500
- const status = typeof error === "object" && error !== null && "status" in error ? Number(error.status) : void 0;
12555
+ const status = typeof error === "object" && error !== null ? Number(
12556
+ error.status ?? error.statusCode
12557
+ ) : void 0;
12501
12558
  const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
12502
12559
  if (isCodexTransportError(error) && code === "invalid_sse_terminal") {
12503
12560
  return {
@@ -12611,13 +12668,12 @@ function classifyCodexCredentialFailure(error) {
12611
12668
  for (let depth = 0; depth < 6 && cur && typeof cur === "object"; depth++) {
12612
12669
  const value = cur;
12613
12670
  const body = value.error && typeof value.error === "object" ? value.error : null;
12614
- const status = Number(value.status ?? body?.status);
12671
+ const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
12615
12672
  const code = String(value.code ?? body?.code ?? "").toLowerCase();
12616
12673
  const directRetryAfter = Number(
12617
12674
  value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds
12618
12675
  );
12619
- const headers = value.headers;
12620
- const retryAfterHeader = headers?.get?.("retry-after") ?? null;
12676
+ const retryAfterHeader = headerValue(value.headers, "retry-after") ?? headerValue(value.responseHeaders, "retry-after") ?? headerValue(body?.headers, "retry-after");
12621
12677
  const retryAfterNumber = retryAfterHeader === null ? Number.NaN : Number(retryAfterHeader);
12622
12678
  const retryAfterDate = retryAfterHeader !== null && !Number.isFinite(retryAfterNumber) ? Date.parse(retryAfterHeader) : Number.NaN;
12623
12679
  const retryAfter = Number.isFinite(directRetryAfter) ? directRetryAfter : Number.isFinite(retryAfterNumber) ? retryAfterNumber : Number.isFinite(retryAfterDate) ? Math.max(0, (retryAfterDate - Date.now()) / 1e3) : Number.NaN;
@@ -16511,6 +16567,9 @@ function temporalActivityLeaseSettled(pending, nowMs = Date.now()) {
16511
16567
  const heartbeatTimeoutMs = temporalTimeMs(pending.activityOptions?.heartbeatTimeout);
16512
16568
  return heartbeatAt !== null && heartbeatTimeoutMs !== null && nowMs >= heartbeatAt + heartbeatTimeoutMs;
16513
16569
  }
16570
+ function temporalWorkflowExecutionNotFound(error) {
16571
+ return isGrpcServiceError(error) && error.code === 5;
16572
+ }
16514
16573
  function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
16515
16574
  const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
16516
16575
  if (modulePath.endsWith("/src/index.ts")) {
@@ -16644,10 +16703,16 @@ async function createWorkerWorkflowSignaler(settings, db) {
16644
16703
  });
16645
16704
  },
16646
16705
  inspectSessionAttemptActivity: async ({ workflowId, workflowRunId, activityId }) => {
16647
- const description = await connection.workflowService.describeWorkflowExecution({
16648
- namespace: settings.temporalNamespace,
16649
- execution: { workflowId, runId: workflowRunId }
16650
- });
16706
+ let description;
16707
+ try {
16708
+ description = await connection.workflowService.describeWorkflowExecution({
16709
+ namespace: settings.temporalNamespace,
16710
+ execution: { workflowId, runId: workflowRunId }
16711
+ });
16712
+ } catch (error) {
16713
+ if (temporalWorkflowExecutionNotFound(error)) return "settled";
16714
+ throw error;
16715
+ }
16651
16716
  const pending = description.pendingActivities?.find(
16652
16717
  (activity2) => activity2.activityId === activityId
16653
16718
  );
@@ -17013,6 +17078,7 @@ export {
17013
17078
  runOpenGeniWorker,
17014
17079
  startWorker,
17015
17080
  temporalActivityLeaseSettled,
17081
+ temporalWorkflowExecutionNotFound,
17016
17082
  workerOwnsInternalSchedules
17017
17083
  };
17018
17084
  //# sourceMappingURL=index.js.map