@opengeni/worker-bundle 0.16.11 → 0.16.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +8 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +79 -13
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
- package/src/activities/agent-turn.ts +107 -14
- package/src/index.ts +18 -4
|
@@ -37,10 +37,17 @@ export declare function legacyTurnExecutionPolicyInput(turn: Pick<SessionTurn, "
|
|
|
37
37
|
/** A retryable provider fault recovers the accepted turn itself. Goal state is
|
|
38
38
|
* irrelevant: autonomous continuation and infrastructure recovery are separate
|
|
39
39
|
* concerns. */
|
|
40
|
-
export declare function providerRecoveryResult(
|
|
40
|
+
export declare function providerRecoveryResult(input: {
|
|
41
|
+
failureCode: string | undefined;
|
|
42
|
+
attemptNumber: number;
|
|
43
|
+
retryAfterMs?: number | null;
|
|
44
|
+
}): {
|
|
41
45
|
status: "recovering";
|
|
42
46
|
continueDelayMs: number;
|
|
43
47
|
};
|
|
48
|
+
export declare function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number;
|
|
49
|
+
/** Read a provider Retry-After hint without retaining response headers/body. */
|
|
50
|
+
export declare function providerRetryAfterMs(error: unknown, nowMs?: number): number | null;
|
|
44
51
|
/**
|
|
45
52
|
* Tell the model when durable MCP policy references are unavailable for this
|
|
46
53
|
* exact turn. The policy projection already bounds and validates these ids;
|
package/dist/index.d.ts
CHANGED
|
@@ -35,6 +35,8 @@ type TemporalActivityLease = {
|
|
|
35
35
|
} | null;
|
|
36
36
|
};
|
|
37
37
|
export declare function temporalActivityLeaseSettled(pending: TemporalActivityLease | null | undefined, nowMs?: number): boolean;
|
|
38
|
+
/** The exact Temporal workflow run is absent, so it cannot retain an activity. */
|
|
39
|
+
export declare function temporalWorkflowExecutionNotFound(error: unknown): boolean;
|
|
38
40
|
type WorkerWorkflowDefinition = {
|
|
39
41
|
workflowBundle: WorkflowBundleOption;
|
|
40
42
|
} | {
|
package/dist/index.js
CHANGED
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
} from "@opengeni/observability";
|
|
24
24
|
import {
|
|
25
25
|
Connection,
|
|
26
|
+
isGrpcServiceError,
|
|
26
27
|
ScheduleAlreadyRunning,
|
|
27
28
|
ScheduleOverlapPolicy,
|
|
28
29
|
Client as TemporalClient
|
|
@@ -6443,6 +6444,7 @@ import {
|
|
|
6443
6444
|
} from "@opengeni/contracts";
|
|
6444
6445
|
import { createHash as createHash4, randomUUID as randomUUID2 } from "crypto";
|
|
6445
6446
|
var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
|
|
6447
|
+
var PROVIDER_CONNECTIVITY_BACKOFF_MS = [2e3, 5e3, 15e3, 3e4, 6e4];
|
|
6446
6448
|
function credentialSubjectIdForTurnInitiator(turn) {
|
|
6447
6449
|
return directPersonalConnectionSubjectId(turn);
|
|
6448
6450
|
}
|
|
@@ -6487,12 +6489,54 @@ function legacyTurnExecutionPolicyInput(turn) {
|
|
|
6487
6489
|
latencyModeSource: explicit ? "explicit" : "continuation"
|
|
6488
6490
|
};
|
|
6489
6491
|
}
|
|
6490
|
-
function providerRecoveryResult() {
|
|
6492
|
+
function providerRecoveryResult(input) {
|
|
6493
|
+
const providerDelay = input.retryAfterMs !== null && input.retryAfterMs !== void 0 && Number.isFinite(input.retryAfterMs) && input.retryAfterMs > 0 ? Math.ceil(input.retryAfterMs) : null;
|
|
6494
|
+
const continueDelayMs = input.failureCode === "provider_rate_limited" ? providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS : input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" ? Math.max(
|
|
6495
|
+
providerDelay ?? 0,
|
|
6496
|
+
PROVIDER_CONNECTIVITY_BACKOFF_MS[Math.min(
|
|
6497
|
+
Math.max(Math.trunc(input.attemptNumber) - 1, 0),
|
|
6498
|
+
PROVIDER_CONNECTIVITY_BACKOFF_MS.length - 1
|
|
6499
|
+
)]
|
|
6500
|
+
) : PROVIDER_BACKPRESSURE_DELAY_MS;
|
|
6491
6501
|
return {
|
|
6492
6502
|
status: "recovering",
|
|
6493
|
-
continueDelayMs
|
|
6503
|
+
continueDelayMs
|
|
6494
6504
|
};
|
|
6495
6505
|
}
|
|
6506
|
+
function providerRecoveryCountFromMetadata(metadata) {
|
|
6507
|
+
const value = metadata.providerRecoveryCount;
|
|
6508
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
6509
|
+
}
|
|
6510
|
+
function headerValue(headers, name) {
|
|
6511
|
+
if (!headers || typeof headers !== "object") return null;
|
|
6512
|
+
const getter = headers.get;
|
|
6513
|
+
if (typeof getter === "function") {
|
|
6514
|
+
const value = getter.call(headers, name);
|
|
6515
|
+
return typeof value === "string" ? value : null;
|
|
6516
|
+
}
|
|
6517
|
+
const target = name.toLowerCase();
|
|
6518
|
+
const entry = Object.entries(headers).find(
|
|
6519
|
+
([key, value]) => key.toLowerCase() === target && typeof value === "string"
|
|
6520
|
+
);
|
|
6521
|
+
return typeof entry?.[1] === "string" ? entry[1] : null;
|
|
6522
|
+
}
|
|
6523
|
+
function providerRetryAfterMs(error, nowMs = Date.now()) {
|
|
6524
|
+
let current = error;
|
|
6525
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
6526
|
+
const value = current;
|
|
6527
|
+
const body = value.error && typeof value.error === "object" ? value.error : null;
|
|
6528
|
+
const directSeconds = Number(
|
|
6529
|
+
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds
|
|
6530
|
+
);
|
|
6531
|
+
const header = headerValue(value.headers, "retry-after") ?? headerValue(value.responseHeaders, "retry-after") ?? headerValue(body?.headers, "retry-after");
|
|
6532
|
+
const headerSeconds = header === null ? Number.NaN : Number(header);
|
|
6533
|
+
const headerDate = header !== null && !Number.isFinite(headerSeconds) ? Date.parse(header) : Number.NaN;
|
|
6534
|
+
const seconds = Number.isFinite(directSeconds) ? directSeconds : Number.isFinite(headerSeconds) ? headerSeconds : Number.isFinite(headerDate) ? Math.max(0, (headerDate - nowMs) / 1e3) : Number.NaN;
|
|
6535
|
+
if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1e3);
|
|
6536
|
+
current = value.cause;
|
|
6537
|
+
}
|
|
6538
|
+
return null;
|
|
6539
|
+
}
|
|
6496
6540
|
function unavailableMcpTurnInstructions(input) {
|
|
6497
6541
|
if (input.droppedCount <= 0) {
|
|
6498
6542
|
return void 0;
|
|
@@ -7483,6 +7527,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7483
7527
|
let isCodexTurn = false;
|
|
7484
7528
|
let isExternallyBilledTurn = false;
|
|
7485
7529
|
let executionGeneration = 0;
|
|
7530
|
+
let providerRecoveryCount = 0;
|
|
7486
7531
|
let modelRequestStarted = false;
|
|
7487
7532
|
let redispatchesAtDispatch = 0;
|
|
7488
7533
|
const setLastInputTokensFenced = async (lastInputTokens) => {
|
|
@@ -8055,6 +8100,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8055
8100
|
const turn = claim.turn;
|
|
8056
8101
|
turnId = turn.id;
|
|
8057
8102
|
executionGeneration = turn.executionGeneration;
|
|
8103
|
+
providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
8058
8104
|
const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
|
|
8059
8105
|
const policyForAbsent = claimedPolicy.kind === "valid" ? claimedPolicy.policy : resolveTurnExecutionPolicyV1(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
|
|
8060
8106
|
const installedPolicy = await installOrReadTurnExecutionPolicyForAttempt(db, {
|
|
@@ -11993,7 +12039,12 @@ function createRunAgentTurnActivity(services) {
|
|
|
11993
12039
|
}
|
|
11994
12040
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
11995
12041
|
try {
|
|
11996
|
-
const
|
|
12042
|
+
const nextProviderRecoveryCount = providerRecoveryCount + 1;
|
|
12043
|
+
const recoveryResult = providerRecoveryResult({
|
|
12044
|
+
failureCode: failure.code,
|
|
12045
|
+
attemptNumber: nextProviderRecoveryCount,
|
|
12046
|
+
retryAfterMs: providerRetryAfterMs(error)
|
|
12047
|
+
});
|
|
11997
12048
|
await flushRuntimeBatcher();
|
|
11998
12049
|
await reconcileConversationTruth({ requireDurable: true });
|
|
11999
12050
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
@@ -12002,9 +12053,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
12002
12053
|
triggerEventId,
|
|
12003
12054
|
attemptId: input.attemptId,
|
|
12004
12055
|
reason: failure.code ?? "provider_unavailable",
|
|
12056
|
+
providerRecoveryCount: nextProviderRecoveryCount,
|
|
12005
12057
|
detail: {
|
|
12006
12058
|
...failure,
|
|
12007
|
-
continueDelayMs: recoveryResult.continueDelayMs
|
|
12059
|
+
continueDelayMs: recoveryResult.continueDelayMs,
|
|
12060
|
+
providerRecoveryCount: nextProviderRecoveryCount
|
|
12008
12061
|
}
|
|
12009
12062
|
});
|
|
12010
12063
|
if (recovery.action === "stale") {
|
|
@@ -12479,7 +12532,9 @@ function isExactStatuslessUpstreamConnectivityMessage(message) {
|
|
|
12479
12532
|
return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
|
|
12480
12533
|
}
|
|
12481
12534
|
function isTransientProviderError(error) {
|
|
12482
|
-
const status = typeof error === "object" && error !== null
|
|
12535
|
+
const status = typeof error === "object" && error !== null ? Number(
|
|
12536
|
+
error.status ?? error.statusCode
|
|
12537
|
+
) : void 0;
|
|
12483
12538
|
if (status !== void 0 && Number.isFinite(status)) {
|
|
12484
12539
|
return status >= 500 && status < 600;
|
|
12485
12540
|
}
|
|
@@ -12497,7 +12552,9 @@ function isTransientProviderError(error) {
|
|
|
12497
12552
|
}
|
|
12498
12553
|
function agentRunFailurePayload(error, options = {}) {
|
|
12499
12554
|
const message = error instanceof Error ? error.message : String(error);
|
|
12500
|
-
const status = typeof error === "object" && error !== null
|
|
12555
|
+
const status = typeof error === "object" && error !== null ? Number(
|
|
12556
|
+
error.status ?? error.statusCode
|
|
12557
|
+
) : void 0;
|
|
12501
12558
|
const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
|
|
12502
12559
|
if (isCodexTransportError(error) && code === "invalid_sse_terminal") {
|
|
12503
12560
|
return {
|
|
@@ -12611,13 +12668,12 @@ function classifyCodexCredentialFailure(error) {
|
|
|
12611
12668
|
for (let depth = 0; depth < 6 && cur && typeof cur === "object"; depth++) {
|
|
12612
12669
|
const value = cur;
|
|
12613
12670
|
const body = value.error && typeof value.error === "object" ? value.error : null;
|
|
12614
|
-
const status = Number(value.status ?? body?.status);
|
|
12671
|
+
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
12615
12672
|
const code = String(value.code ?? body?.code ?? "").toLowerCase();
|
|
12616
12673
|
const directRetryAfter = Number(
|
|
12617
12674
|
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds
|
|
12618
12675
|
);
|
|
12619
|
-
const
|
|
12620
|
-
const retryAfterHeader = headers?.get?.("retry-after") ?? null;
|
|
12676
|
+
const retryAfterHeader = headerValue(value.headers, "retry-after") ?? headerValue(value.responseHeaders, "retry-after") ?? headerValue(body?.headers, "retry-after");
|
|
12621
12677
|
const retryAfterNumber = retryAfterHeader === null ? Number.NaN : Number(retryAfterHeader);
|
|
12622
12678
|
const retryAfterDate = retryAfterHeader !== null && !Number.isFinite(retryAfterNumber) ? Date.parse(retryAfterHeader) : Number.NaN;
|
|
12623
12679
|
const retryAfter = Number.isFinite(directRetryAfter) ? directRetryAfter : Number.isFinite(retryAfterNumber) ? retryAfterNumber : Number.isFinite(retryAfterDate) ? Math.max(0, (retryAfterDate - Date.now()) / 1e3) : Number.NaN;
|
|
@@ -16511,6 +16567,9 @@ function temporalActivityLeaseSettled(pending, nowMs = Date.now()) {
|
|
|
16511
16567
|
const heartbeatTimeoutMs = temporalTimeMs(pending.activityOptions?.heartbeatTimeout);
|
|
16512
16568
|
return heartbeatAt !== null && heartbeatTimeoutMs !== null && nowMs >= heartbeatAt + heartbeatTimeoutMs;
|
|
16513
16569
|
}
|
|
16570
|
+
function temporalWorkflowExecutionNotFound(error) {
|
|
16571
|
+
return isGrpcServiceError(error) && error.code === 5;
|
|
16572
|
+
}
|
|
16514
16573
|
function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
|
|
16515
16574
|
const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
|
|
16516
16575
|
if (modulePath.endsWith("/src/index.ts")) {
|
|
@@ -16644,10 +16703,16 @@ async function createWorkerWorkflowSignaler(settings, db) {
|
|
|
16644
16703
|
});
|
|
16645
16704
|
},
|
|
16646
16705
|
inspectSessionAttemptActivity: async ({ workflowId, workflowRunId, activityId }) => {
|
|
16647
|
-
|
|
16648
|
-
|
|
16649
|
-
|
|
16650
|
-
|
|
16706
|
+
let description;
|
|
16707
|
+
try {
|
|
16708
|
+
description = await connection.workflowService.describeWorkflowExecution({
|
|
16709
|
+
namespace: settings.temporalNamespace,
|
|
16710
|
+
execution: { workflowId, runId: workflowRunId }
|
|
16711
|
+
});
|
|
16712
|
+
} catch (error) {
|
|
16713
|
+
if (temporalWorkflowExecutionNotFound(error)) return "settled";
|
|
16714
|
+
throw error;
|
|
16715
|
+
}
|
|
16651
16716
|
const pending = description.pendingActivities?.find(
|
|
16652
16717
|
(activity2) => activity2.activityId === activityId
|
|
16653
16718
|
);
|
|
@@ -17013,6 +17078,7 @@ export {
|
|
|
17013
17078
|
runOpenGeniWorker,
|
|
17014
17079
|
startWorker,
|
|
17015
17080
|
temporalActivityLeaseSettled,
|
|
17081
|
+
temporalWorkflowExecutionNotFound,
|
|
17016
17082
|
workerOwnsInternalSchedules
|
|
17017
17083
|
};
|
|
17018
17084
|
//# sourceMappingURL=index.js.map
|