@opengeni/worker-bundle 0.16.11 → 0.16.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +8 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +79 -13
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
- package/src/activities/agent-turn.ts +107 -14
- package/src/index.ts +18 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.16.
|
|
3
|
+
"version": "0.16.12",
|
|
4
4
|
"description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -43,10 +43,10 @@
|
|
|
43
43
|
"@opengeni/codex": "^0.2.10",
|
|
44
44
|
"@opengeni/config": "^0.10.11",
|
|
45
45
|
"@opengeni/contracts": "^0.38.1",
|
|
46
|
-
"@opengeni/core": "^0.20.
|
|
47
|
-
"@opengeni/db": "^0.27.
|
|
48
|
-
"@opengeni/documents": "^0.5.
|
|
49
|
-
"@opengeni/events": "^0.3.
|
|
46
|
+
"@opengeni/core": "^0.20.11",
|
|
47
|
+
"@opengeni/db": "^0.27.8",
|
|
48
|
+
"@opengeni/documents": "^0.5.4",
|
|
49
|
+
"@opengeni/events": "^0.3.74",
|
|
50
50
|
"@opengeni/github": "^0.4.27",
|
|
51
51
|
"@opengeni/observability": "^0.4.14",
|
|
52
52
|
"@opengeni/runtime": "^0.18.10",
|
|
@@ -357,11 +357,10 @@ import {
|
|
|
357
357
|
} from "@opengeni/contracts";
|
|
358
358
|
import { createHash, randomUUID } from "node:crypto";
|
|
359
359
|
|
|
360
|
-
//
|
|
361
|
-
//
|
|
362
|
-
// throttling is minute-granular; anything shorter mostly burns continuation
|
|
363
|
-
// budget against the same window.
|
|
360
|
+
// Retryable provider connectivity/5xx failures start quickly and back off to
|
|
361
|
+
// this ceiling. Explicit rate limits retain the minute-granular fallback.
|
|
364
362
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
363
|
+
const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
|
|
365
364
|
|
|
366
365
|
/** Broad personal lookup is allowed only for a direct human/API command. */
|
|
367
366
|
export function credentialSubjectIdForTurnInitiator(
|
|
@@ -445,16 +444,93 @@ export function legacyTurnExecutionPolicyInput(
|
|
|
445
444
|
/** A retryable provider fault recovers the accepted turn itself. Goal state is
|
|
446
445
|
* irrelevant: autonomous continuation and infrastructure recovery are separate
|
|
447
446
|
* concerns. */
|
|
448
|
-
export function providerRecoveryResult(
|
|
447
|
+
export function providerRecoveryResult(input: {
|
|
448
|
+
failureCode: string | undefined;
|
|
449
|
+
attemptNumber: number;
|
|
450
|
+
retryAfterMs?: number | null;
|
|
451
|
+
}): {
|
|
449
452
|
status: "recovering";
|
|
450
453
|
continueDelayMs: number;
|
|
451
454
|
} {
|
|
455
|
+
const providerDelay =
|
|
456
|
+
input.retryAfterMs !== null &&
|
|
457
|
+
input.retryAfterMs !== undefined &&
|
|
458
|
+
Number.isFinite(input.retryAfterMs) &&
|
|
459
|
+
input.retryAfterMs > 0
|
|
460
|
+
? Math.ceil(input.retryAfterMs)
|
|
461
|
+
: null;
|
|
462
|
+
const continueDelayMs =
|
|
463
|
+
input.failureCode === "provider_rate_limited"
|
|
464
|
+
? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
|
|
465
|
+
: input.failureCode === "provider_unavailable" ||
|
|
466
|
+
input.failureCode === "upstream_connectivity_unavailable"
|
|
467
|
+
? Math.max(
|
|
468
|
+
providerDelay ?? 0,
|
|
469
|
+
PROVIDER_CONNECTIVITY_BACKOFF_MS[
|
|
470
|
+
Math.min(
|
|
471
|
+
Math.max(Math.trunc(input.attemptNumber) - 1, 0),
|
|
472
|
+
PROVIDER_CONNECTIVITY_BACKOFF_MS.length - 1,
|
|
473
|
+
)
|
|
474
|
+
]!,
|
|
475
|
+
)
|
|
476
|
+
: PROVIDER_BACKPRESSURE_DELAY_MS;
|
|
452
477
|
return {
|
|
453
478
|
status: "recovering",
|
|
454
|
-
continueDelayMs
|
|
479
|
+
continueDelayMs,
|
|
455
480
|
};
|
|
456
481
|
}
|
|
457
482
|
|
|
483
|
+
export function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number {
|
|
484
|
+
const value = metadata.providerRecoveryCount;
|
|
485
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
function headerValue(headers: unknown, name: string): string | null {
|
|
489
|
+
if (!headers || typeof headers !== "object") return null;
|
|
490
|
+
const getter = (headers as { get?: unknown }).get;
|
|
491
|
+
if (typeof getter === "function") {
|
|
492
|
+
const value = getter.call(headers, name);
|
|
493
|
+
return typeof value === "string" ? value : null;
|
|
494
|
+
}
|
|
495
|
+
const target = name.toLowerCase();
|
|
496
|
+
const entry = Object.entries(headers as Record<string, unknown>).find(
|
|
497
|
+
([key, value]) => key.toLowerCase() === target && typeof value === "string",
|
|
498
|
+
);
|
|
499
|
+
return typeof entry?.[1] === "string" ? entry[1] : null;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/** Read a provider Retry-After hint without retaining response headers/body. */
|
|
503
|
+
export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number | null {
|
|
504
|
+
let current: unknown = error;
|
|
505
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
506
|
+
const value = current as Record<string, unknown>;
|
|
507
|
+
const body =
|
|
508
|
+
value.error && typeof value.error === "object"
|
|
509
|
+
? (value.error as Record<string, unknown>)
|
|
510
|
+
: null;
|
|
511
|
+
const directSeconds = Number(
|
|
512
|
+
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
|
|
513
|
+
);
|
|
514
|
+
const header =
|
|
515
|
+
headerValue(value.headers, "retry-after") ??
|
|
516
|
+
headerValue(value.responseHeaders, "retry-after") ??
|
|
517
|
+
headerValue(body?.headers, "retry-after");
|
|
518
|
+
const headerSeconds = header === null ? Number.NaN : Number(header);
|
|
519
|
+
const headerDate =
|
|
520
|
+
header !== null && !Number.isFinite(headerSeconds) ? Date.parse(header) : Number.NaN;
|
|
521
|
+
const seconds = Number.isFinite(directSeconds)
|
|
522
|
+
? directSeconds
|
|
523
|
+
: Number.isFinite(headerSeconds)
|
|
524
|
+
? headerSeconds
|
|
525
|
+
: Number.isFinite(headerDate)
|
|
526
|
+
? Math.max(0, (headerDate - nowMs) / 1_000)
|
|
527
|
+
: Number.NaN;
|
|
528
|
+
if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1_000);
|
|
529
|
+
current = value.cause;
|
|
530
|
+
}
|
|
531
|
+
return null;
|
|
532
|
+
}
|
|
533
|
+
|
|
458
534
|
/**
|
|
459
535
|
* Tell the model when durable MCP policy references are unavailable for this
|
|
460
536
|
* exact turn. The policy projection already bounds and validates these ids;
|
|
@@ -2378,6 +2454,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2378
2454
|
let isCodexTurn = false;
|
|
2379
2455
|
let isExternallyBilledTurn = false;
|
|
2380
2456
|
let executionGeneration = 0;
|
|
2457
|
+
let providerRecoveryCount = 0;
|
|
2381
2458
|
let modelRequestStarted = false;
|
|
2382
2459
|
// Still required by credential-loss/capacity settlements, whose own
|
|
2383
2460
|
// recovery transactions fence against worker-death redispatches.
|
|
@@ -3179,6 +3256,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3179
3256
|
const turn = claim.turn;
|
|
3180
3257
|
turnId = turn.id;
|
|
3181
3258
|
executionGeneration = turn.executionGeneration;
|
|
3259
|
+
providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
3182
3260
|
const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
|
|
3183
3261
|
const policyForAbsent =
|
|
3184
3262
|
claimedPolicy.kind === "valid"
|
|
@@ -8176,7 +8254,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8176
8254
|
}
|
|
8177
8255
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
8178
8256
|
try {
|
|
8179
|
-
const
|
|
8257
|
+
const nextProviderRecoveryCount = providerRecoveryCount + 1;
|
|
8258
|
+
const recoveryResult = providerRecoveryResult({
|
|
8259
|
+
failureCode: failure.code,
|
|
8260
|
+
attemptNumber: nextProviderRecoveryCount,
|
|
8261
|
+
retryAfterMs: providerRetryAfterMs(error),
|
|
8262
|
+
});
|
|
8180
8263
|
await flushRuntimeBatcher();
|
|
8181
8264
|
await reconcileConversationTruth({ requireDurable: true });
|
|
8182
8265
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
@@ -8185,9 +8268,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8185
8268
|
triggerEventId: triggerEventId!,
|
|
8186
8269
|
attemptId: input.attemptId,
|
|
8187
8270
|
reason: failure.code ?? "provider_unavailable",
|
|
8271
|
+
providerRecoveryCount: nextProviderRecoveryCount,
|
|
8188
8272
|
detail: {
|
|
8189
8273
|
...failure,
|
|
8190
8274
|
continueDelayMs: recoveryResult.continueDelayMs,
|
|
8275
|
+
providerRecoveryCount: nextProviderRecoveryCount,
|
|
8191
8276
|
},
|
|
8192
8277
|
});
|
|
8193
8278
|
if (recovery.action === "stale") {
|
|
@@ -8810,8 +8895,11 @@ function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean
|
|
|
8810
8895
|
|
|
8811
8896
|
export function isTransientProviderError(error: unknown): boolean {
|
|
8812
8897
|
const status =
|
|
8813
|
-
typeof error === "object" && error !== null
|
|
8814
|
-
? Number(
|
|
8898
|
+
typeof error === "object" && error !== null
|
|
8899
|
+
? Number(
|
|
8900
|
+
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
8901
|
+
(error as { statusCode?: unknown }).statusCode,
|
|
8902
|
+
)
|
|
8815
8903
|
: undefined;
|
|
8816
8904
|
// A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
|
|
8817
8905
|
// (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
|
|
@@ -8857,8 +8945,11 @@ export function agentRunFailurePayload(
|
|
|
8857
8945
|
} {
|
|
8858
8946
|
const message = error instanceof Error ? error.message : String(error);
|
|
8859
8947
|
const status =
|
|
8860
|
-
typeof error === "object" && error !== null
|
|
8861
|
-
? Number(
|
|
8948
|
+
typeof error === "object" && error !== null
|
|
8949
|
+
? Number(
|
|
8950
|
+
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
8951
|
+
(error as { statusCode?: unknown }).statusCode,
|
|
8952
|
+
)
|
|
8862
8953
|
: undefined;
|
|
8863
8954
|
const code =
|
|
8864
8955
|
typeof error === "object" && error !== null && "code" in error
|
|
@@ -9062,13 +9153,15 @@ export function classifyCodexCredentialFailure(error: unknown): CodexCredentialF
|
|
|
9062
9153
|
value.error && typeof value.error === "object"
|
|
9063
9154
|
? (value.error as Record<string, unknown>)
|
|
9064
9155
|
: null;
|
|
9065
|
-
const status = Number(value.status ?? body?.status);
|
|
9156
|
+
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
9066
9157
|
const code = String(value.code ?? body?.code ?? "").toLowerCase();
|
|
9067
9158
|
const directRetryAfter = Number(
|
|
9068
9159
|
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
|
|
9069
9160
|
);
|
|
9070
|
-
const
|
|
9071
|
-
|
|
9161
|
+
const retryAfterHeader =
|
|
9162
|
+
headerValue(value.headers, "retry-after") ??
|
|
9163
|
+
headerValue(value.responseHeaders, "retry-after") ??
|
|
9164
|
+
headerValue(body?.headers, "retry-after");
|
|
9072
9165
|
const retryAfterNumber = retryAfterHeader === null ? Number.NaN : Number(retryAfterHeader);
|
|
9073
9166
|
const retryAfterDate =
|
|
9074
9167
|
retryAfterHeader !== null && !Number.isFinite(retryAfterNumber)
|
package/src/index.ts
CHANGED
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
} from "@opengeni/observability";
|
|
23
23
|
import {
|
|
24
24
|
Connection,
|
|
25
|
+
isGrpcServiceError,
|
|
25
26
|
ScheduleAlreadyRunning,
|
|
26
27
|
ScheduleOverlapPolicy,
|
|
27
28
|
Client as TemporalClient,
|
|
@@ -127,6 +128,13 @@ export function temporalActivityLeaseSettled(
|
|
|
127
128
|
);
|
|
128
129
|
}
|
|
129
130
|
|
|
131
|
+
/** The exact Temporal workflow run is absent, so it cannot retain an activity. */
|
|
132
|
+
export function temporalWorkflowExecutionNotFound(error: unknown): boolean {
|
|
133
|
+
// gRPC status code 5 is NOT_FOUND. The typed guard prevents unrelated
|
|
134
|
+
// provider/HTTP errors carrying a numeric code from proving quiescence.
|
|
135
|
+
return isGrpcServiceError(error) && error.code === 5;
|
|
136
|
+
}
|
|
137
|
+
|
|
130
138
|
type WorkerWorkflowDefinition =
|
|
131
139
|
| { workflowBundle: WorkflowBundleOption }
|
|
132
140
|
| { workflowsPath: string };
|
|
@@ -312,10 +320,16 @@ export async function createWorkerWorkflowSignaler(
|
|
|
312
320
|
// receipt and its exact wake revision atomically.
|
|
313
321
|
},
|
|
314
322
|
inspectSessionAttemptActivity: async ({ workflowId, workflowRunId, activityId }) => {
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
323
|
+
let description;
|
|
324
|
+
try {
|
|
325
|
+
description = await connection.workflowService.describeWorkflowExecution({
|
|
326
|
+
namespace: settings.temporalNamespace,
|
|
327
|
+
execution: { workflowId, runId: workflowRunId },
|
|
328
|
+
});
|
|
329
|
+
} catch (error) {
|
|
330
|
+
if (temporalWorkflowExecutionNotFound(error)) return "settled";
|
|
331
|
+
throw error;
|
|
332
|
+
}
|
|
319
333
|
const pending = description.pendingActivities?.find(
|
|
320
334
|
(activity) => activity.activityId === activityId,
|
|
321
335
|
);
|