@deksden-com/dd-flow-cli 0.9.0-beta.150 → 0.9.0-beta.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +17 -22
- package/dist/build-info.json +6 -6
- package/dist/harness-runtime/lib/agy-error-boundary.mjs +26 -5
- package/dist/harness-runtime/lib/agy-terminal-envelope.mjs +67 -0
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +10 -14
- package/dist/harness-runtime/lib/dd-agy.mjs +3 -3
- package/dist/harness-runtime/lib/dd-droid.mjs +2 -2
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +62 -10
- package/dist/harness-runtime/lib/dd-opencode.mjs +32 -14
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +1 -1
- package/dist/harness-runtime/lib/dd-zcode.mjs +37 -12
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +1 -1
- package/dist/harness-runtime/lib/delegation-instructions.mjs +13 -4
- package/dist/harness-runtime/lib/model-observations.mjs +19 -7
- package/dist/harness-runtime/lib/native-children.d.mts +12 -1
- package/dist/harness-runtime/lib/native-children.mjs +47 -1
- package/dist/harness-runtime/lib/provider-overload-policy.d.mts +9 -1
- package/dist/harness-runtime/lib/provider-overload-policy.mjs +97 -6
- package/dist/services/check-runner-evidence.js +54 -309
- package/dist/services/check-runner-proof.js +373 -0
- package/dist/services/code-checks.js +31 -12
- package/dist/services/controller-child-capacity.js +58 -0
- package/dist/services/controller-child-observer.js +9 -3
- package/dist/services/memory-init-native-capacity.js +1 -1
- package/dist/services/provider-capacity.js +15 -7
- package/dist/services/repair-intents.js +4 -1
- package/dist/services/run-controller-adapter.js +1 -1
- package/dist/services/run-controller.js +87 -18
- package/dist/services/run-recovery-guard.js +1 -1
- package/dist/services/run-recovery-runtime.js +1 -1
- package/dist/services/stage-context.js +56 -8
- package/dist/services/vnext-code-review.js +1 -1
- package/dist/services/vnext-code.js +3 -2
- package/dist/services/vnext-fanout.js +1 -1
- package/dist/services/vnext-merge.js +1 -1
- package/dist/services/work-registry.js +1 -1
- package/dist/verification-components.json +299 -0
- package/package.json +7 -6
- package/tools/verification-jobs.mjs +317 -0
|
@@ -15,7 +15,7 @@ import { commandText } from "./process-json.mjs";
|
|
|
15
15
|
import { readNativeLines, streamNativeDiagnostics, nativeObservationError } from "./native-streams.mjs";
|
|
16
16
|
import { operationProgressReporter } from "./operation-progress.mjs";
|
|
17
17
|
import { safeNativeFailure } from "./native-children.mjs";
|
|
18
|
-
import { classifyProviderError, safeProviderError } from "./provider-overload-policy.mjs";
|
|
18
|
+
import { classifyRecoveryError as classifyProviderError, safeProviderError } from "./provider-overload-policy.mjs";
|
|
19
19
|
|
|
20
20
|
import { readZcodeUsage } from "./zcode-usage.mjs";
|
|
21
21
|
const ZCODE_HARNESS_CONTRACT = "dd-zcode-harness@1";
|
|
@@ -200,6 +200,7 @@ export class AcpBridge {
|
|
|
200
200
|
clientInfo: this.options.clientInfo ?? { name: "dd-zcode", version: "0.1.0" },
|
|
201
201
|
}, this.options.initializeTimeoutMs ?? SETUP_STARTUP_MS);
|
|
202
202
|
publishSetup('native_initialized', 'native-initialize');
|
|
203
|
+
this.recoveryCapability = initialized?._meta?.zcode?.ddFlowRecovery;
|
|
203
204
|
return initialized;
|
|
204
205
|
}
|
|
205
206
|
|
|
@@ -306,7 +307,7 @@ export class AcpBridge {
|
|
|
306
307
|
code: [provider?.code, data?.code, data?.native_code].filter(value => typeof value === "string").join(" "), message: messageText };
|
|
307
308
|
const category = pending.method === "session/prompt" ? classifyProviderError(nativeFailure) : "provider_failed";
|
|
308
309
|
error.code = unknown ? nativeCode : category === "provider_failed" ? "acp_request_failed" : category;
|
|
309
|
-
error.retryable = ["provider_rate_limited", "provider_capacity_transient"].includes(error.code);
|
|
310
|
+
error.retryable = ["provider_rate_limited", "provider_capacity_transient", "provider_network_transient"].includes(error.code);
|
|
310
311
|
error.details = { acp_code: message.error.code ?? null,
|
|
311
312
|
observed_at: provider?.observed_at ?? now(), provider_session_id: pending.sessionId ?? null, source: "acp.error_response", request_id: message.id,
|
|
312
313
|
terminal_response: { method: pending.method, session_id: pending.sessionId, request_id: message.id },
|
|
@@ -581,13 +582,13 @@ async function resume(bridge, sessionId, cwd) {
|
|
|
581
582
|
|
|
582
583
|
/** Ended children are paginated by the native API (20 by default). A partial
|
|
583
584
|
* directory is never settlement evidence, including during teardown. */
|
|
584
|
-
export async function readZcodeSubagents(bridge, sessionId, timeoutMs = 30_000, retained = false, terminalCache = new Map()) {
|
|
585
|
+
export async function readZcodeSubagents(bridge, sessionId, timeoutMs = 30_000, retained = false, terminalCache = new Map(), parentSessionId = sessionId) {
|
|
585
586
|
const clock = new ObservationClock({ timeoutMs });
|
|
586
587
|
const cursors = new Set(), ended = new Map();
|
|
587
588
|
let first, cursor;
|
|
588
589
|
for (;;) {
|
|
589
590
|
if (clock.sample()) fail("ZCode child observation is unavailable", "operation_observation_lost");
|
|
590
|
-
const response = await bridge.request(retained ? "zcode/session/retainedSubagents" : "zcode/session/subagents", { sessionId, ...(cursor ? { endedCursor: cursor } : {}) }, timeoutMs);
|
|
591
|
+
const response = await bridge.request(retained ? "zcode/session/retainedSubagents" : "zcode/session/subagents", { sessionId, proofContract: "zcode/retained-child-effects@1", ...(cursor ? { endedCursor: cursor } : {}) }, timeoutMs);
|
|
591
592
|
if (retained && response?.sessionId !== sessionId) fail("Retained ZCode child directory has a foreign Session identity", "native_topology_incomplete");
|
|
592
593
|
const page = retained ? response.topology : response;
|
|
593
594
|
if (!page || !Array.isArray(page.running)) fail("ZCode child directory is unavailable", "native_topology_incomplete");
|
|
@@ -615,7 +616,7 @@ export async function readZcodeSubagents(bridge, sessionId, timeoutMs = 30_000,
|
|
|
615
616
|
const cacheKey = JSON.stringify([child.childSessionId, child.startedAt, child.endedAt]);
|
|
616
617
|
if (terminalCache.has(cacheKey)) { items.push(terminalCache.get(cacheKey)); continue; }
|
|
617
618
|
let terminal;
|
|
618
|
-
try { terminal = await bridge.request("zcode/session/retainedTerminal", { sessionId: child.childSessionId, startedAt: child.startedAt, endedAt: child.endedAt }, timeoutMs); }
|
|
619
|
+
try { terminal = await bridge.request("zcode/session/retainedTerminal", { sessionId: child.childSessionId, parentSessionId, startedAt: child.startedAt, endedAt: child.endedAt, proofContract: "zcode/retained-child-effects@1" }, timeoutMs); }
|
|
619
620
|
catch (error) { items.push({ ...child, terminal_metadata_unavailable: error.code ?? "terminal_metadata_unavailable" }); continue; }
|
|
620
621
|
if (terminal?.available !== true) { items.push({ ...child, terminal_metadata_unavailable: "terminal_metadata_unavailable" }); continue; }
|
|
621
622
|
const observedAt = Date.parse(terminal.terminal_observed_at);
|
|
@@ -623,9 +624,23 @@ export async function readZcodeSubagents(bridge, sessionId, timeoutMs = 30_000,
|
|
|
623
624
|
|| terminal.startedAt !== child.startedAt || terminal.endedAt !== child.endedAt
|
|
624
625
|
|| !Number.isFinite(observedAt) || observedAt < child.startedAt || observedAt > child.endedAt) fail("ZCode terminal failure metadata has foreign identity", "native_child_turn_conflict");
|
|
625
626
|
const providerError = safeNativeFailure(terminal.provider_error);
|
|
626
|
-
const
|
|
627
|
+
const proof = terminal.effects_proof;
|
|
628
|
+
const ref = terminal.continuation_ref;
|
|
629
|
+
const effectsBound = proof?.schema_id === "zcode/retained-child-effects@1" && proof.session_id === child.childSessionId
|
|
630
|
+
&& proof.started_at === child.startedAt && proof.ended_at === child.endedAt && proof.terminal_turn_id === terminal.terminal_turn_id
|
|
631
|
+
&& proof.settled === true && proof.native_turn_items?.observed === true && proof.native_turn_items?.pending === false;
|
|
632
|
+
const refBound = effectsBound && ref?.schema_id === "zcode/child-continuation@1" && ref.child_session_id === child.childSessionId
|
|
633
|
+
&& ref.parent_session_id === parentSessionId && ref.started_at === child.startedAt && ref.ended_at === child.endedAt && ref.terminal_turn_id === terminal.terminal_turn_id
|
|
634
|
+
&& typeof ref.agent_id === "string" && /^agent_[\w-]+$/.test(ref.agent_id) && typeof ref.parent_tool_call_id === "string" && ref.parent_tool_call_id
|
|
635
|
+
&& ref.capability === "dd-flow/managed-provider-recovery@1";
|
|
636
|
+
const item = { ...child, terminal_turn_id: terminal.terminal_turn_id, terminal_observed_at: terminal.terminal_observed_at, native_source: "zcode.retained-terminal",
|
|
637
|
+
...(effectsBound ? { native_turn_items: proof.native_turn_items, effects_proof: proof } : {}),
|
|
638
|
+
...(refBound ? { continuation_ref: ref } : {}),
|
|
639
|
+
...(!effectsBound || !refBound ? { recovery_unavailable_reason: typeof (terminal.continuation_unavailable_reason ?? proof?.reason) === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(terminal.continuation_unavailable_reason ?? proof?.reason) ? terminal.continuation_unavailable_reason ?? proof.reason : !effectsBound ? "native_effects_unproven" : "native_continuation_unproven" } : {}),
|
|
627
640
|
...(providerError ? { provider_error: providerError } : {}) };
|
|
628
|
-
|
|
641
|
+
// Effects/topology proof is admission-time authority: never reuse it
|
|
642
|
+
// across a later inspection merely because the old terminal is cached.
|
|
643
|
+
if (!proof && !ref) terminalCache.set(cacheKey, item);
|
|
629
644
|
items.push(item);
|
|
630
645
|
clock.sample(child.childSessionId);
|
|
631
646
|
}
|
|
@@ -638,10 +653,10 @@ export async function readZcodeSubagents(bridge, sessionId, timeoutMs = 30_000,
|
|
|
638
653
|
}
|
|
639
654
|
}
|
|
640
655
|
|
|
641
|
-
async function inspect(bridge, sessionId) {
|
|
656
|
+
async function inspect(bridge, sessionId, parentSessionId = sessionId) {
|
|
642
657
|
const [read, subagents] = await Promise.all([
|
|
643
658
|
bridge.request("zcode/session/read", { sessionId }),
|
|
644
|
-
readZcodeSubagents(bridge, sessionId),
|
|
659
|
+
readZcodeSubagents(bridge, sessionId, 30_000, false, new Map(), parentSessionId),
|
|
645
660
|
]);
|
|
646
661
|
let usage = null, usage_observation_error = null;
|
|
647
662
|
try { usage = await readZcodeUsage(bridge, sessionId); }
|
|
@@ -984,6 +999,8 @@ export function zcodeLifecycleQualification(sha256, bridgeCommit, harnessContrac
|
|
|
984
999
|
["636b14181d5c9a95476055079836fd6b58169f58", "dd-zcode-harness@2"],
|
|
985
1000
|
["4621fabd3f517ecdd349dfebebae9077c01d2f2b", "dd-zcode-harness@2"],
|
|
986
1001
|
["9c5ca6cc81c4eb5065e43f8b7ab6b7cc5ab7ae47", "dd-zcode-harness@2"],
|
|
1002
|
+
// CP253 installed artifact: exact same-child Read continuation and retained proof.
|
|
1003
|
+
["b18dfdbedaa9330e765315bce0f1a609792d6c91", "dd-zcode-harness@2"],
|
|
987
1004
|
]);
|
|
988
1005
|
const qualified = qualifiedBridgeCommits.get(bridgeCommit) === harnessContract;
|
|
989
1006
|
return { status: qualified ? "qualified" : "unqualified", mechanism: "cli-invocation@1", sha256,
|
|
@@ -1082,14 +1099,22 @@ export async function promptSessionWithBridge(bridge, options) {
|
|
|
1082
1099
|
const textCursor = bridge.assistantTextCursor(identity.adapterSessionId);
|
|
1083
1100
|
options.assertDispatch?.();
|
|
1084
1101
|
bridge.dispatchOperationId = options.operationId;
|
|
1085
|
-
const
|
|
1102
|
+
const managed = options.operationId ? { schema_id: "dd-flow/managed-provider-recovery@1", owner: "controller", operation_id: options.operationId, root_operation_id: options.operationId.replace(/:capacity:\d+:session\.prompt$/, "") } : null;
|
|
1103
|
+
const acknowledged = value => managed && value && Object.keys(managed).every(key => value[key] === managed[key]);
|
|
1104
|
+
if (managed && (bridge.recoveryCapability?.schema_id !== managed.schema_id || bridge.recoveryCapability?.owner !== "controller")) fail("ZCode bridge does not acknowledge controller-owned provider recovery", "provider_recovery_unavailable");
|
|
1105
|
+
const turn = await bridge.request("session/prompt", { sessionId: identity.adapterSessionId, prompt: [{ type: "text", text: prompt }], ...(managed ? { _meta: { zcode: { ddFlowRecovery: managed } } } : {}) }, timeoutMs, {
|
|
1086
1106
|
idleSessionId: identity.adapterSessionId,
|
|
1087
1107
|
idleTimeoutMs: options.livenessTimeoutMs ?? 600_000,
|
|
1088
1108
|
inactivityMode: "diagnostic",
|
|
1089
1109
|
}).catch(error => {
|
|
1090
1110
|
if (error.details?.terminal_response?.session_id === identity.adapterSessionId) error.details.provider_session_id = identity.providerSessionId;
|
|
1111
|
+
if (managed && !acknowledged(error.details?.native?._meta?.zcode?.ddFlowRecovery)) {
|
|
1112
|
+
const unknown = new Error("ZCode provider failure omitted managed recovery acknowledgement");
|
|
1113
|
+
unknown.code = "native_outcome_unknown"; unknown.details = { primary_error: errorRecord(error) }; throw unknown;
|
|
1114
|
+
}
|
|
1091
1115
|
throw error;
|
|
1092
1116
|
});
|
|
1117
|
+
if (managed && !acknowledged(turn?._meta?.zcode?.ddFlowRecovery)) fail("ZCode bridge omitted the managed recovery acknowledgement", "native_outcome_unknown");
|
|
1093
1118
|
const stopReason = typeof turn?.stopReason === "string" ? turn.stopReason : null;
|
|
1094
1119
|
const completionEvidence = turn?._meta?.zcodeCompletionEvidence;
|
|
1095
1120
|
if (completionEvidence === "inferred" || completionEvidence === "failed") {
|
|
@@ -1113,7 +1138,7 @@ export async function promptSessionWithBridge(bridge, options) {
|
|
|
1113
1138
|
throw error;
|
|
1114
1139
|
}
|
|
1115
1140
|
let evidence;
|
|
1116
|
-
try { await bridge.flush(); evidence = await inspect(bridge, identity.adapterSessionId); }
|
|
1141
|
+
try { await bridge.flush(); evidence = await inspect(bridge, identity.adapterSessionId, identity.providerSessionId); }
|
|
1117
1142
|
catch (cause) {
|
|
1118
1143
|
const error = new Error("ZCode completed the native turn, but post-turn observation failed");
|
|
1119
1144
|
error.code = "native_outcome_observation_failed";
|
|
@@ -1141,7 +1166,7 @@ export async function inspectSession(options) {
|
|
|
1141
1166
|
export async function inspectSessionWithBridge(bridge, options) {
|
|
1142
1167
|
requireJournal(options);
|
|
1143
1168
|
const identity = await controlledIdentity(bridge, options);
|
|
1144
|
-
return receipt(options, { harness: "zcode-acp", provider_session_id: identity.providerSessionId, adapter_session_id: identity.adapterSessionId, ...publicEvidence(await inspect(bridge, identity.adapterSessionId)) });
|
|
1169
|
+
return receipt(options, { harness: "zcode-acp", provider_session_id: identity.providerSessionId, adapter_session_id: identity.adapterSessionId, ...publicEvidence(await inspect(bridge, identity.adapterSessionId, identity.providerSessionId)) });
|
|
1145
1170
|
}
|
|
1146
1171
|
|
|
1147
1172
|
export async function cancelSession(options) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export type DelegationWork = { work_id: string; start_command: string; launch_policy?: string };
|
|
2
2
|
export type DelegationInput = { harness: string; stage: string; capacity: number; works?: DelegationWork[]; totalWorks?: number; remainingWorks?: number };
|
|
3
3
|
export function delegationAdapter(harness: string): string | null;
|
|
4
|
-
export function renderNativeChildFollowupInstruction(harness: string, childId: string): string | null;
|
|
4
|
+
export function renderNativeChildFollowupInstruction(harness: string, childId: string | { session_id?: unknown; parent_session_id?: unknown; terminal_turn_id?: unknown; continuation_ref?: unknown }): string | null;
|
|
5
5
|
export function delegationContract(harness: string): string;
|
|
6
6
|
export const lifecycleRetryInstruction: string;
|
|
7
7
|
export const lifecycleCommandCompletionInstruction: string;
|
|
@@ -39,12 +39,21 @@ export function delegationAdapter(harness) {
|
|
|
39
39
|
|
|
40
40
|
/** Only documented same-child APIs may continue an existing native worker. */
|
|
41
41
|
export function renderNativeChildFollowupInstruction(harness, childId) {
|
|
42
|
+
const child = childId && typeof childId === "object" ? childId : null;
|
|
43
|
+
childId = child?.session_id ?? childId;
|
|
42
44
|
if (typeof childId !== "string" || !childId) return null;
|
|
43
45
|
const adapter = delegationAdapter(harness), target = JSON.stringify(childId);
|
|
44
|
-
|
|
45
|
-
if (adapter === "
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
const message = JSON.stringify("Continue the same assigned Work after the transient provider failure; do not restart it or repeat committed lifecycle commands.");
|
|
47
|
+
if (adapter === "zcode") {
|
|
48
|
+
const ref = child?.continuation_ref;
|
|
49
|
+
if (ref?.schema_id !== "zcode/child-continuation@1" || ref.capability !== "dd-flow/managed-provider-recovery@1" || ref.child_session_id !== childId
|
|
50
|
+
|| ref.parent_session_id !== child.parent_session_id || ref.terminal_turn_id !== child.terminal_turn_id || !/^agent_[\w-]{1,200}$/.test(ref.agent_id ?? "")) return null;
|
|
51
|
+
return `Use the native SendMessage tool exactly once with to: ${JSON.stringify(ref.agent_id)}, summary: "Continue existing Work", and message: ${message}. Retain messageId and delivery only when the native result supplies them; never invent missing IDs. The adapter verifies typed native completion and successor evidence. Wait for this same agent with TaskOutput. If admission is uncertain, do not resend. Do not spawn a replacement or override model/reasoning.`;
|
|
52
|
+
}
|
|
53
|
+
if (adapter === "codex") return `Use collaboration.followup_task with target: ${target} and message: ${message}. Do not spawn a replacement.`;
|
|
54
|
+
if (adapter === "opencode") return `Use the native subagent tool with sessionID: ${target} and prompt: ${message}. Keep the same model and reasoning. Do not spawn a replacement.`;
|
|
55
|
+
if (adapter === "droid") return `Use the native Task tool with resume: ${target}, subagent_type: "dd-flow-worker", and prompt: ${message}. Omit model and complexity overrides. Do not spawn a replacement.`;
|
|
56
|
+
if (adapter === "grok") return `Use the native send_subagent_message tool with subagent_id: ${target}, delivery: "queue", and text: ${message}. Retain its accepted message_id and wait on the same subagent. If admission is uncertain, do not resend. Do not spawn a replacement.`;
|
|
48
57
|
return null;
|
|
49
58
|
}
|
|
50
59
|
|
|
@@ -29,7 +29,7 @@ export async function readModelObservations(journal) {
|
|
|
29
29
|
|
|
30
30
|
/** Only adapter-verified native facts enter this file. It records changes,
|
|
31
31
|
* not every tool or token; reading it never requires replaying native history. */
|
|
32
|
-
export async function observeModel({ journal, harness, sessionId, parentSessionId = null, requested = {}, observed = {}, source, evidence = "configured", reason = null, nativeTimestamp = null, previousModel = null }) {
|
|
32
|
+
export async function observeModel({ journal, harness, sessionId, parentSessionId = null, requested = {}, observed = {}, source, evidence = "configured", reason = null, nativeTimestamp = null, previousModel = null, nativeAgentRole = null }) {
|
|
33
33
|
if (!journal || !sessionId) return null;
|
|
34
34
|
if (!source || !["configured", "request", "response", "unavailable"].includes(evidence)) throw new Error("model observation requires a native source and evidence level");
|
|
35
35
|
if (typeof source === "string") source = { channel: source, ...(["configured", "response"].includes(evidence) ? { scope: evidence } : {}) };
|
|
@@ -38,11 +38,17 @@ export async function observeModel({ journal, harness, sessionId, parentSessionI
|
|
|
38
38
|
return withRunnerLock(file, async () => {
|
|
39
39
|
const events = await readModelObservations(journal);
|
|
40
40
|
const replay = source?.event_sha256 ? events.find(item => item.harness === harness && item.session_id === sessionId && observationScope(item.source) === observationScope(source) && item.source?.event_sha256 === source.event_sha256) : null;
|
|
41
|
-
if (replay) return replay;
|
|
42
41
|
const previous = events.findLast(item => item.harness === harness && item.session_id === sessionId && observationScope(item.source) === observationScope(source));
|
|
43
42
|
const normalizedReason = text(reason)?.slice(0, 200) ?? null;
|
|
44
|
-
|
|
45
|
-
|
|
43
|
+
const tuple = item => JSON.stringify([observedFields(item.requested), item.observed, item.parent_session_id ?? null, item.source, item.reason, item.evidence, item.native_agent_role ?? null]);
|
|
44
|
+
const candidate = { requested: observedFields(requested), observed: profile, parent_session_id: parentSessionId, source, reason: normalizedReason, evidence, native_agent_role: nativeAgentRole };
|
|
45
|
+
if (replay && tuple(replay) === tuple(candidate)) return replay;
|
|
46
|
+
const conflict = replay ? { code: 'model_observation_identity_conflict', original_id: replay.id } : null;
|
|
47
|
+
if (conflict) {
|
|
48
|
+
const recorded = events.find(item => item.evidence_conflict?.original_id === replay.id && tuple(item) === tuple(candidate));
|
|
49
|
+
if (recorded) return recorded;
|
|
50
|
+
}
|
|
51
|
+
if (!conflict && previous && tuple(previous) === tuple(candidate)) return previous;
|
|
46
52
|
const previousKnown = events.findLast(item => item.harness === harness && item.session_id === sessionId && item.evidence === evidence && item.observed?.model
|
|
47
53
|
&& (item.source?.scope ?? "session") === (source?.scope ?? "session") && (item.source?.attempt_id ?? null) === (source?.attempt_id ?? null));
|
|
48
54
|
const fromModel = text(previousModel) ?? previousKnown?.observed.model ?? null;
|
|
@@ -53,7 +59,8 @@ export async function observeModel({ journal, harness, sessionId, parentSessionI
|
|
|
53
59
|
harness, session_id: sessionId, parent_session_id: parentSessionId,
|
|
54
60
|
kind: changed ? "model_changed" : !profile.model ? "model_observation_unavailable" : "model_observed",
|
|
55
61
|
previous_model: fromModel, requested: observedFields(requested), observed: profile,
|
|
56
|
-
source, evidence, reason: normalizedReason, native_timestamp: text(nativeTimestamp), observed_at: new Date().toISOString()
|
|
62
|
+
source, evidence, reason: normalizedReason, native_timestamp: text(nativeTimestamp), observed_at: new Date().toISOString(),
|
|
63
|
+
...(nativeAgentRole ? { native_agent_role: nativeAgentRole } : {}), ...(conflict ? { evidence_conflict: conflict } : {}) };
|
|
57
64
|
value.id = createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
|
58
65
|
const handle = await open(file, "a", 0o600);
|
|
59
66
|
try {
|
|
@@ -81,7 +88,12 @@ export function modelAttribution(events, expectedSessions = []) {
|
|
|
81
88
|
: !sessions.has(`${session.harness}:${session.session_id}`));
|
|
82
89
|
const models = [...new Set(events.map(event => event.observed?.model).filter(Boolean))];
|
|
83
90
|
const providers = [...new Set(events.map(event => event.observed?.provider).filter(Boolean))];
|
|
84
|
-
const
|
|
91
|
+
const conflicts = events.filter(event => event.evidence_conflict).map(event => ({ session_id: event.session_id, ...event.evidence_conflict }));
|
|
92
|
+
for (const session of sessions.values()) {
|
|
93
|
+
const parents = [...new Set(session.observations.filter(event => !event.source?.non_asserting && Object.hasOwn(event, 'parent_session_id')).map(event => event.parent_session_id))];
|
|
94
|
+
if (parents.length > 1) conflicts.push({ session_id: session.session_id, code: 'model_observation_parent_conflict' });
|
|
95
|
+
}
|
|
96
|
+
const incomplete = conflicts.length > 0 || !events.length || missingSessions.length > 0 || [...sessions.values()].some(session => {
|
|
85
97
|
const known = session.observations.filter(event => event.observed?.model);
|
|
86
98
|
if (!known.length) return true;
|
|
87
99
|
const knownScopes = new Set(known.map(event => `${event.evidence}:${event.source?.scope ?? "session"}:${event.source?.attempt_id ?? ""}:${event.source?.turn_id ?? ""}`));
|
|
@@ -97,5 +109,5 @@ export function modelAttribution(events, expectedSessions = []) {
|
|
|
97
109
|
return { models, providers, mixed: models.length > 1 || providers.length > 1 || events.some(event => event.observed?.model && event.requested?.model && event.observed.model !== event.requested.model
|
|
98
110
|
|| event.observed?.provider && event.requested?.provider && event.observed.provider !== event.requested.provider),
|
|
99
111
|
observation_completeness: incomplete ? "incomplete" : "available_native_sources", transitions: events.filter(event => event.kind === "model_changed"),
|
|
100
|
-
sessions: [...sessions.values()], missing_sessions: missingSessions, usage_attribution: "session_totals_unattributed_to_model", note: "Native settings/events do not prove undisclosed server routing or per-model token costs." };
|
|
112
|
+
sessions: [...sessions.values()], missing_sessions: missingSessions, ...(conflicts.length ? { evidence_conflicts: conflicts } : {}), usage_attribution: "session_totals_unattributed_to_model", note: "Native settings/events do not prove undisclosed server routing or per-model token costs." };
|
|
101
113
|
}
|
|
@@ -8,6 +8,12 @@ export type NativeTreeReadiness = "ready" | "owned_children_live" | "inspection_
|
|
|
8
8
|
export function nativeTreeReadiness(receipt: Record<string, unknown>, binding: { sessionId: string; operationId: string; turnId?: string; requestProof?: unknown; allowDescendants?: boolean }): NativeTreeReadiness;
|
|
9
9
|
export function settledNativeTree(root: string, children: Record<string, unknown>[], settled: boolean, rootSettled?: boolean): Record<string, unknown>;
|
|
10
10
|
export function safeNativeFailure(value: unknown): Record<string, unknown> | undefined;
|
|
11
|
+
export type NativeContinuationReference = { schema_id: "zcode/child-continuation@1"; capability: "dd-flow/managed-provider-recovery@1"; agent_id: string; child_session_id: string; parent_session_id: string; parent_tool_call_id: string; terminal_turn_id: string; started_at: number; ended_at: number; task_id?: string };
|
|
12
|
+
export function nativeContinuationReference(value: unknown, sessionId: string, parentId: string, turnId?: string): NativeContinuationReference | null;
|
|
13
|
+
export type NativeContinuationAdmission = { schema_id: "zcode/child-admission@1"; operation_id?: string; predecessor_turn_id?: string; parent_session_id: string; child_session_id: string; agent_id: string; parent_tool_call_id: string; parent_turn_id: string } & ({ proof_kind?: "message"; message_id: string; delivery: "queued" | "steered" | "resumed_background" } | { proof_kind: "parent_tool_successor"; successor_started_at: number; successor_turn_id: string; native_evidence_sha256: string } | { proof_kind: "parent_tool_admission"; successor_started_at: number; native_evidence_sha256: string });
|
|
14
|
+
export function nativeContinuationAdmission(value: unknown, sessionId: string, parentId: string): NativeContinuationAdmission | null;
|
|
15
|
+
export function nativeContinuationSuccessor(child: NativeChild, predecessorTurnId?: string, predecessorEndedAt?: number): boolean;
|
|
16
|
+
export function nativeContinuationAdmitted(child: NativeChild, predecessorTurnId?: string, predecessorEndedAt?: number): boolean;
|
|
11
17
|
export interface NativeChild {
|
|
12
18
|
session_id: string;
|
|
13
19
|
parent_session_id: string;
|
|
@@ -17,7 +23,12 @@ export interface NativeChild {
|
|
|
17
23
|
terminal_turn_id?: string;
|
|
18
24
|
terminal_observed_at?: string;
|
|
19
25
|
provider_error?: unknown;
|
|
20
|
-
|
|
26
|
+
recovery_unavailable_reason?: string;
|
|
27
|
+
continuation_ref?: NativeContinuationReference;
|
|
28
|
+
continuation_admission?: NativeContinuationAdmission;
|
|
29
|
+
resumed_from_message_id?: string;
|
|
30
|
+
current_turn_id?: string;
|
|
31
|
+
native_source?: "opencode.v2.session.execution" | "agy.result" | "acp.error_response" | "droid.agent_turn_outcome" | "grok.subagent.event" | "zcode.retained-terminal";
|
|
21
32
|
native_turn_items?: { observed?: boolean; possible_effects?: boolean; pending?: boolean };
|
|
22
33
|
}
|
|
23
34
|
export function normalizeNativeChildren(receipt: Record<string, unknown>, rootSessionId: string): NativeChild[];
|
|
@@ -125,6 +125,9 @@ export function safeNativeFailure(value, depth = 0) {
|
|
|
125
125
|
// invalid value must reach the policy as invalid, not disappear into default backoff.
|
|
126
126
|
const timing = safeProviderError(value);
|
|
127
127
|
if (Object.hasOwn(timing, "retryAfter")) result.retryAfter = timing.retryAfter;
|
|
128
|
+
if (Object.hasOwn(timing, "retry_after")) result.retry_after = timing.retry_after;
|
|
129
|
+
if (Object.hasOwn(timing, "retryAfterMs")) result.retryAfterMs = timing.retryAfterMs;
|
|
130
|
+
if (timing.headers) result.headers = timing.headers;
|
|
128
131
|
if (typeof value.codexErrorInfo === "string" && /^[\w-]{1,128}$/.test(value.codexErrorInfo)) result.codexErrorInfo = value.codexErrorInfo;
|
|
129
132
|
else if (value.codexErrorInfo && typeof value.codexErrorInfo.code === "string" && /^[\w-]{1,128}$/.test(value.codexErrorInfo.code)) result.codexErrorInfo = { code: value.codexErrorInfo.code };
|
|
130
133
|
const attribution = depth === 0 && value.attribution && safeNativeFailure(value.attribution, 1);
|
|
@@ -134,6 +137,40 @@ export function safeNativeFailure(value, depth = 0) {
|
|
|
134
137
|
|
|
135
138
|
|
|
136
139
|
/** Normalize provider-observed topology only; never parse assistant prose. */
|
|
140
|
+
export function nativeContinuationReference(value, sessionId, parentId, turnId) {
|
|
141
|
+
if (value?.schema_id !== "zcode/child-continuation@1" || value.capability !== "dd-flow/managed-provider-recovery@1"
|
|
142
|
+
|| value.child_session_id !== sessionId || value.parent_session_id !== parentId || value.terminal_turn_id !== turnId
|
|
143
|
+
|| !/^agent_[\w-]{1,200}$/.test(value.agent_id ?? "") || ![value.parent_tool_call_id, value.terminal_turn_id].every(item => typeof item === "string" && item.length > 0 && item.length <= 256)
|
|
144
|
+
|| !Number.isSafeInteger(value.started_at) || !Number.isSafeInteger(value.ended_at) || value.started_at <= 0 || value.ended_at < value.started_at) return null;
|
|
145
|
+
return { schema_id: value.schema_id, capability: value.capability, agent_id: value.agent_id, child_session_id: sessionId, parent_session_id: parentId,
|
|
146
|
+
parent_tool_call_id: value.parent_tool_call_id, terminal_turn_id: turnId, started_at: value.started_at, ended_at: value.ended_at,
|
|
147
|
+
...(typeof value.task_id === "string" && value.task_id.length <= 256 ? { task_id: value.task_id } : {}) };
|
|
148
|
+
}
|
|
149
|
+
export function nativeContinuationAdmission(value, sessionId, parentId) {
|
|
150
|
+
const keys = ["parent_tool_call_id", "parent_turn_id", "agent_id"];
|
|
151
|
+
if (value?.schema_id !== "zcode/child-admission@1" || value.child_session_id !== sessionId || value.parent_session_id !== parentId
|
|
152
|
+
|| !keys.every(key => typeof value[key] === "string" && value[key].length > 0 && value[key].length <= 512)) return null;
|
|
153
|
+
const successorProof = value.proof_kind === "parent_tool_successor";
|
|
154
|
+
const parentAdmission = successorProof || value.proof_kind === "parent_tool_admission";
|
|
155
|
+
if (parentAdmission ? !Number.isSafeInteger(value.successor_started_at) || value.successor_started_at <= 0 || successorProof && (typeof value.successor_turn_id !== "string" || !value.successor_turn_id || value.successor_turn_id.length > 512) || !/^[a-f0-9]{64}$/.test(value.native_evidence_sha256 ?? "")
|
|
156
|
+
: value.proof_kind !== undefined && value.proof_kind !== "message" || typeof value.message_id !== "string" || !value.message_id || value.message_id.length > 512 || !["queued", "steered", "resumed_background"].includes(value.delivery)) return null;
|
|
157
|
+
return Object.fromEntries(["schema_id", "child_session_id", "parent_session_id", ...keys,
|
|
158
|
+
...(parentAdmission ? ["proof_kind", "successor_started_at", "native_evidence_sha256", ...(successorProof ? ["successor_turn_id"] : [])] : ["message_id", "delivery"]),
|
|
159
|
+
...["operation_id", "predecessor_turn_id"].filter(key => typeof value[key] === "string" && value[key].length > 0 && value[key].length <= 512)].map(key => [key, value[key]]));
|
|
160
|
+
}
|
|
161
|
+
export function nativeContinuationSuccessor(child, predecessorTurnId, predecessorEndedAt) {
|
|
162
|
+
const admission = child?.continuation_admission, turn = child?.current_turn_id ?? child?.terminal_turn_id;
|
|
163
|
+
if (!admission?.operation_id || admission.predecessor_turn_id !== predecessorTurnId || typeof turn !== "string" || turn === predecessorTurnId) return false;
|
|
164
|
+
if (admission.proof_kind === "parent_tool_admission") return false;
|
|
165
|
+
return admission.proof_kind === "parent_tool_successor"
|
|
166
|
+
? Number.isSafeInteger(predecessorEndedAt) && admission.successor_started_at > predecessorEndedAt && admission.successor_turn_id === turn
|
|
167
|
+
: child.resumed_from_message_id === admission.message_id;
|
|
168
|
+
}
|
|
169
|
+
export function nativeContinuationAdmitted(child, predecessorTurnId, predecessorEndedAt) {
|
|
170
|
+
const admission = child?.continuation_admission;
|
|
171
|
+
return Boolean(admission?.proof_kind === "parent_tool_admission" && (child.status === "running" || child.status === "failed" && child.terminal_turn_id === predecessorTurnId) && admission.operation_id
|
|
172
|
+
&& admission.predecessor_turn_id === predecessorTurnId && Number.isSafeInteger(predecessorEndedAt) && admission.successor_started_at > predecessorEndedAt);
|
|
173
|
+
}
|
|
137
174
|
export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
138
175
|
const candidates = [];
|
|
139
176
|
const object = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
@@ -173,7 +210,12 @@ export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
|
173
210
|
...(text(value, ["terminal_turn_id"]) ? { terminal_turn_id: text(value, ["terminal_turn_id"]) } : {}),
|
|
174
211
|
...(text(value, ["terminal_observed_at"]) ? { terminal_observed_at: text(value, ["terminal_observed_at"]) } : {}),
|
|
175
212
|
...(value.provider_error !== undefined ? { provider_error: value.provider_error } : {}),
|
|
176
|
-
...(
|
|
213
|
+
...(typeof value.recovery_unavailable_reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(value.recovery_unavailable_reason) ? { recovery_unavailable_reason: value.recovery_unavailable_reason } : {}),
|
|
214
|
+
...(nativeContinuationReference(value.continuation_ref, id, parent, value.terminal_turn_id) ? { continuation_ref: nativeContinuationReference(value.continuation_ref, id, parent, value.terminal_turn_id) } : {}),
|
|
215
|
+
...(nativeContinuationAdmission(value.continuation_admission, id, parent) ? { continuation_admission: nativeContinuationAdmission(value.continuation_admission, id, parent) } : {}),
|
|
216
|
+
...(text(value, ["resumed_from_message_id"]) ? { resumed_from_message_id: text(value, ["resumed_from_message_id"]) } : {}),
|
|
217
|
+
...(text(value, ["current_turn_id"]) ? { current_turn_id: text(value, ["current_turn_id"]) } : {}),
|
|
218
|
+
...(["opencode.v2.session.execution", "agy.result", "acp.error_response", "droid.agent_turn_outcome", "grok.subagent.event", "zcode.retained-terminal"].includes(value.native_source) ? { native_source: value.native_source } : {}),
|
|
177
219
|
...(value.native_turn_items && typeof value.native_turn_items === "object" ? { native_turn_items: value.native_turn_items } : {}) });
|
|
178
220
|
};
|
|
179
221
|
for (const value of list(receipt.descendants)) add(value, "adapter.descendants");
|
|
@@ -190,6 +232,10 @@ export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
|
190
232
|
const terminal = value => ["completed", "failed", "cancelled", "settled_by_root"].includes(value);
|
|
191
233
|
for (const child of candidates) {
|
|
192
234
|
const previous = unique.get(child.session_id);
|
|
235
|
+
const successor = (current, old) => current.parent_session_id === old.parent_session_id
|
|
236
|
+
&& (nativeContinuationSuccessor(current, old.terminal_turn_id, old.continuation_ref?.ended_at) || nativeContinuationAdmitted(current, old.terminal_turn_id, old.continuation_ref?.ended_at));
|
|
237
|
+
if (previous && successor(child, previous)) { unique.set(child.session_id, child); continue; }
|
|
238
|
+
if (previous && successor(previous, child)) continue;
|
|
193
239
|
if (previous && (terminal(previous.status) && child.native_turn_items?.pending === true
|
|
194
240
|
|| terminal(child.status) && previous.native_turn_items?.pending === true))
|
|
195
241
|
throw nativeFailure("native_child_observation_conflict", "Current child outcome contradicts pending native items", 1, { session_id: child.session_id, observations: [previous, child] });
|
|
@@ -1,11 +1,19 @@
|
|
|
1
1
|
export const CAPACITY_POLICY: "provider-overload-episode@1";
|
|
2
|
+
export const RECOVERY_POLICY: "provider-recovery-episode@1";
|
|
3
|
+
export function classifyRecoveryError(native: unknown): string;
|
|
4
|
+
export function recoveryRetryAfter(native: unknown, now?: number): number;
|
|
5
|
+
export const normalizeRecoveryFailure: typeof normalizeProviderFailure;
|
|
6
|
+
export const terminalProviderRecovery: typeof terminalProviderOverload;
|
|
7
|
+
export const recoveryBackoff: typeof capacityBackoff;
|
|
8
|
+
export const recoveryRefusal: typeof refusalObservation;
|
|
9
|
+
export const recoveryExhausted: typeof overloadExhausted;
|
|
2
10
|
export const MAX_OVERLOAD_RETRIES: 3;
|
|
3
11
|
export const MAX_RETRY_AFTER_MS: 30000;
|
|
4
12
|
export function classifyProviderError(native: unknown): string;
|
|
5
13
|
export function safeProviderError(native: unknown): Record<string, unknown>;
|
|
6
14
|
export type RequestProof = { kind: "native_turn"; session_id: string; turn_id: string } | { kind: "stream_request"; daemon_instance: string; connection_id: string; operation_id: string; turn_generation: number; session_id: string; outbound_position: number; terminal_position: number };
|
|
7
15
|
export function streamRequestKey(proof: unknown): string | null;
|
|
8
|
-
export type CapacityRefusal = { policy: string; ordinal: number; session_id: string; turn_id: string; request_key?: string; request_proof?: RequestProof; first_observed_at: string; native_terminal_at?: string; native_terminal_basis?: string };
|
|
16
|
+
export type CapacityRefusal = { policy: string; ordinal: number; session_id: string; turn_id: string; category?: string; request_key?: string; request_proof?: RequestProof; first_observed_at: string; native_terminal_at?: string; native_terminal_basis?: string };
|
|
9
17
|
export function parseCapacityOperation(id: string): { root: string; ordinal: number };
|
|
10
18
|
export function normalizeProviderFailure(error: unknown, sessionId?: string, turnId?: string): null | { code: string; session_id: string; turn_id: string; request_key: string; request_proof: RequestProof; native_error: Record<string, unknown>; observed_at: unknown; terminal_at: unknown; native_turn_items: unknown; primary_error: { details?: Record<string, unknown> } };
|
|
11
19
|
export function terminalProviderOverload(error: unknown, sessionId?: string, turnId?: string): boolean;
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
export const CAPACITY_POLICY = "provider-overload-episode@1";
|
|
3
3
|
export const MAX_OVERLOAD_RETRIES = 3;
|
|
4
4
|
export const MAX_RETRY_AFTER_MS = 30_000;
|
|
5
|
+
export const RECOVERY_POLICY = "provider-recovery-episode@1";
|
|
5
6
|
|
|
6
7
|
function policyError(code, message, details = {}) { return Object.assign(new Error(message), { code, details }); }
|
|
7
8
|
|
|
@@ -25,10 +26,28 @@ export function safeProviderError(native) {
|
|
|
25
26
|
if (!native || typeof native !== "object") return result;
|
|
26
27
|
const status = native.status ?? native.statusCode ?? native.status_code ?? native.http_status;
|
|
27
28
|
if (Number.isInteger(Number(status))) result.status = Number(status);
|
|
28
|
-
const
|
|
29
|
-
const
|
|
29
|
+
const retryHeaders = Object.entries(native.headers ?? {}).filter(([key]) => key.toLowerCase() === "retry-after");
|
|
30
|
+
const header = retryHeaders[0]?.[1];
|
|
31
|
+
const retry = native.retryAfter ?? native.retry_after ?? header;
|
|
30
32
|
if (retry !== undefined && retry !== null) result.retryAfter = typeof retry === "string" && retry.length <= 128 || typeof retry === "number" && Number.isFinite(retry) ? retry : "invalid-retry-after";
|
|
33
|
+
if (native.retry_after !== undefined && native.retry_after !== null) result.retry_after = typeof native.retry_after === "string" && native.retry_after.length <= 128 || typeof native.retry_after === "number" && Number.isFinite(native.retry_after) ? native.retry_after : "invalid-retry-after";
|
|
34
|
+
if (header !== undefined && header !== null && (native.retryAfter !== undefined || native.retry_after !== undefined || retryHeaders.length > 1)) result.headers = Object.fromEntries(retryHeaders.map(([key, value]) => [key, typeof value === "string" && value.length <= 128 || typeof value === "number" && Number.isFinite(value) ? value : "invalid-retry-after"]));
|
|
31
35
|
if (typeof native.willRetry === "boolean") result.willRetry = native.willRetry;
|
|
36
|
+
for (const key of ["type", "source", "reason", "errorPhase", "exceptionKind", "transport", "retryable", "retryAfterMs"]) {
|
|
37
|
+
const value = native[key];
|
|
38
|
+
if (typeof value === "boolean" || typeof value === "number" && Number.isFinite(value) || typeof value === "string" && value.length <= 128) result[key] = value;
|
|
39
|
+
}
|
|
40
|
+
if (!result.type && /^(model_request_failed|ECONNRESET|EPIPE|ETIMEDOUT|EAI_AGAIN|UND_ERR_SOCKET|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT)$/.test(native.code ?? "")) result.type = native.code;
|
|
41
|
+
if (Object.hasOwn(native, "retryAfterMs") && native.retryAfterMs !== null) result.retryAfterMs = typeof native.retryAfterMs === "number" && Number.isFinite(native.retryAfterMs) || typeof native.retryAfterMs === "string" && native.retryAfterMs.length <= 128 ? native.retryAfterMs : "invalid-retry-after";
|
|
42
|
+
if (native.attribution && typeof native.attribution === "object") {
|
|
43
|
+
const attribution = {};
|
|
44
|
+
for (const key of ["source", "reason", "errorPhase", "exceptionKind", "transport", "retryable", "retryAfterMs"]) {
|
|
45
|
+
const value = native.attribution[key];
|
|
46
|
+
if (typeof value === "boolean" || typeof value === "number" && Number.isFinite(value) || typeof value === "string" && value.length <= 128) attribution[key] = value;
|
|
47
|
+
}
|
|
48
|
+
if (Object.hasOwn(native.attribution, "retryAfterMs") && native.attribution.retryAfterMs !== null) attribution.retryAfterMs = typeof native.attribution.retryAfterMs === "number" && Number.isFinite(native.attribution.retryAfterMs) || typeof native.attribution.retryAfterMs === "string" && native.attribution.retryAfterMs.length <= 128 ? native.attribution.retryAfterMs : "invalid-retry-after";
|
|
49
|
+
if (Object.keys(attribution).length) result.attribution = attribution;
|
|
50
|
+
}
|
|
32
51
|
for (const key of ["reset_at", "resets_at", "resetAt", "resetsAt", "reset_time"]) {
|
|
33
52
|
const value = native[key];
|
|
34
53
|
if (typeof value === "string" && value.length <= 128 || typeof value === "number" && Number.isFinite(value)) result[key] = value;
|
|
@@ -36,6 +55,70 @@ export function safeProviderError(native) {
|
|
|
36
55
|
return result;
|
|
37
56
|
}
|
|
38
57
|
|
|
58
|
+
/** New native-recovery ABI; the overload-only exports below retain their scope. */
|
|
59
|
+
export function classifyRecoveryError(native) {
|
|
60
|
+
const legacy = classifyProviderError(native);
|
|
61
|
+
if (["provider_quota_exhausted", "provider_auth_failed"].includes(legacy)) return legacy;
|
|
62
|
+
if (!native || typeof native !== "object") return legacy;
|
|
63
|
+
const facts = [native, native.attribution].filter(value => value && typeof value === "object");
|
|
64
|
+
const text = [native.code, native.type, native.message, ...facts.flatMap(value => [value.reason, value.exceptionKind])].filter(value => typeof value === "string").join(" ");
|
|
65
|
+
if (facts.some(value => value.retryable === false) || /cancel|CERT_|certificate|permission|configuration/i.test(text)) return "provider_failed";
|
|
66
|
+
if (legacy === "provider_capacity_transient") return legacy;
|
|
67
|
+
if (["provider_rate_limited", "provider_limit_unknown"].includes(legacy) && facts.some(value => value.retryAfterMs !== undefined)) return "provider_rate_limited";
|
|
68
|
+
if (legacy === "provider_rate_limited") return legacy;
|
|
69
|
+
const structuredNetwork = [native.type, native.code].includes("model_request_failed") && facts.some(value => value.source === "network" && value.reason === "network_error" && value.errorPhase === "stream" && value.exceptionKind === "transport");
|
|
70
|
+
const networkSource = facts.some(value => ["network", "provider", "transport"].includes(value.source));
|
|
71
|
+
const code = [native.code, native.type, native.transport, ...facts.map(value => value.transport)].filter(value => typeof value === "string");
|
|
72
|
+
const knownCode = code.some(value => /^(ECONNRESET|EPIPE|ETIMEDOUT|EAI_AGAIN|UND_ERR_SOCKET|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT)$/.test(value));
|
|
73
|
+
const status = Number(native.status ?? native.statusCode ?? native.status_code ?? native.http_status);
|
|
74
|
+
return structuredNetwork || networkSource && (knownCode || [408, 502, 504].includes(status)) ? "provider_network_transient" : legacy;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** All delay representations must agree; no units guessing and no moving deadline. */
|
|
78
|
+
export function recoveryRetryAfter(native, now = Date.now()) {
|
|
79
|
+
if (!Number.isFinite(now)) throw policyError("capacity_retry_metadata_invalid", "Invalid recovery observation time");
|
|
80
|
+
if (!native || typeof native !== "object") return 0;
|
|
81
|
+
const values = [];
|
|
82
|
+
for (const source of [native, native.attribution].filter(value => value && typeof value === "object")) {
|
|
83
|
+
for (const key of ["retryAfterMs", "retryAfter", "retry_after"]) if (source[key] !== undefined && source[key] !== null) values.push([source[key], key === "retryAfterMs"]);
|
|
84
|
+
for (const [key, value] of Object.entries(source.headers ?? {})) if (key.toLowerCase() === "retry-after") values.push([value, false]);
|
|
85
|
+
}
|
|
86
|
+
const parsed = values.map(([value, milliseconds]) => {
|
|
87
|
+
const numeric = typeof value === "number" || typeof value === "string" && /^\d+(?:\.\d+)?$/.test(value.trim());
|
|
88
|
+
const duration = numeric ? Number(value) * (milliseconds ? 1 : 1000) : !milliseconds && typeof value === "string" && /^\w{3}, \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(value.trim()) ? Math.max(0, Date.parse(value) - now) : NaN;
|
|
89
|
+
if (!numeric && Number.isFinite(duration) && new Date(Date.parse(value)).toUTCString() !== value.trim()) throw policyError("capacity_retry_metadata_invalid", "Invalid recovery HTTP date");
|
|
90
|
+
if (!Number.isFinite(duration) || duration < 0 || !Number.isFinite(now + duration) || now + duration > 8.64e15) throw policyError("capacity_retry_metadata_invalid", "Invalid recovery Retry-After metadata");
|
|
91
|
+
return duration;
|
|
92
|
+
});
|
|
93
|
+
if (parsed.some(value => value !== parsed[0])) throw policyError("capacity_retry_metadata_invalid", "Conflicting recovery Retry-After metadata");
|
|
94
|
+
return parsed[0] ?? 0;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function normalizeRecoveryFailure(error, expectedSessionId, expectedTurnId) {
|
|
98
|
+
const normalized = normalizeNativeFailure(error, expectedSessionId, expectedTurnId, true);
|
|
99
|
+
if (!normalized) return null;
|
|
100
|
+
const category = classifyRecoveryError(normalized.native_error);
|
|
101
|
+
if (!["provider_capacity_transient", "provider_rate_limited", "provider_network_transient"].includes(category)) return null;
|
|
102
|
+
const nativeCode = typeof normalized.native_error.codexErrorInfo === "string" ? normalized.native_error.codexErrorInfo : normalized.native_error.codexErrorInfo?.code;
|
|
103
|
+
return { ...normalized, code: nativeCode ?? category, category };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function terminalProviderRecovery(error, sessionId, turnId) { return normalizeRecoveryFailure(error, sessionId, turnId) !== null; }
|
|
107
|
+
export function recoveryBackoff(error, ordinal, now = Date.now(), random = Math.random) {
|
|
108
|
+
const native = normalizeRecoveryFailure(error)?.native_error ?? error?.details?.provider_error ?? error;
|
|
109
|
+
try { return capacityBackoff({ retryAfter: recoveryRetryAfter(native, now) / 1000 }, ordinal, now, random); }
|
|
110
|
+
catch (failure) { if (failure.details?.policy === CAPACITY_POLICY) failure.details = { ...failure.details, policy: RECOVERY_POLICY }; throw failure; }
|
|
111
|
+
}
|
|
112
|
+
export function recoveryRefusal(error, ordinal, firstObservedAt) {
|
|
113
|
+
const failure = normalizeRecoveryFailure(error);
|
|
114
|
+
return { ...observedRefusal(failure, ordinal, firstObservedAt, RECOVERY_POLICY), category: failure.category };
|
|
115
|
+
}
|
|
116
|
+
export function recoveryExhausted(refusals, maxRetries = MAX_OVERLOAD_RETRIES) {
|
|
117
|
+
if (refusals.some(value => value.policy !== RECOVERY_POLICY)) throw policyError("capacity_policy_mismatch", "Retained episode uses another recovery policy");
|
|
118
|
+
const exhausted = overloadExhausted(refusals.map(value => ({ ...value, policy: CAPACITY_POLICY })), maxRetries);
|
|
119
|
+
return exhausted ? { ...exhausted, policy: RECOVERY_POLICY, reason: "provider_retries_exhausted", refusals } : null;
|
|
120
|
+
}
|
|
121
|
+
|
|
39
122
|
export function parseCapacityOperation(id) {
|
|
40
123
|
if (typeof id !== "string" || !id) throw policyError("capacity_continuation_conflict", "Missing capacity operation identity");
|
|
41
124
|
const match = /^(.*):capacity:(\d+):session\.prompt$/.exec(id);
|
|
@@ -55,17 +138,22 @@ export function streamRequestKey(proof) {
|
|
|
55
138
|
}
|
|
56
139
|
|
|
57
140
|
export function normalizeProviderFailure(error, expectedSessionId, expectedTurnId) {
|
|
141
|
+
return normalizeNativeFailure(error, expectedSessionId, expectedTurnId, false);
|
|
142
|
+
}
|
|
143
|
+
function normalizeNativeFailure(error, expectedSessionId, expectedTurnId, recovery) {
|
|
58
144
|
let current = error;
|
|
59
145
|
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth++) {
|
|
60
146
|
// The same continuation authority applies to a proven v2 execution
|
|
61
147
|
// refusal. Never classify transport loss or an active native retry here.
|
|
62
|
-
if (current.code === "provider_capacity_transient" || current.code === "provider_rate_limited") {
|
|
148
|
+
if (current.code === "provider_capacity_transient" || current.code === "provider_rate_limited" || recovery && current.code === "provider_network_transient") {
|
|
63
149
|
const details = current.details ?? {};
|
|
64
150
|
const streamKey = details.native_source === "agy.result" && details.turn_id === undefined && details.native_turn_id === undefined ? streamRequestKey(details.request_proof) : null;
|
|
65
151
|
const turnKey = streamKey ?? details.turn_id;
|
|
66
|
-
|
|
152
|
+
const sources = ["opencode.v2.session.execution", "agy.result", "acp.error_response", "droid.agent_turn_outcome", "grok.subagent.event", ...(recovery ? ["zcode.retained-terminal"] : [])];
|
|
153
|
+
const categories = ["provider_capacity_transient", "provider_rate_limited", ...(recovery ? ["provider_network_transient"] : [])];
|
|
154
|
+
if (!sources.includes(details.native_source) || details.terminal_status !== "failed"
|
|
67
155
|
|| details.will_retry === true || details.native_retry_active === true || details.provider_error?.willRetry === true
|
|
68
|
-
|| details.provider_error && !
|
|
156
|
+
|| details.provider_error && !categories.includes((recovery ? classifyRecoveryError : classifyProviderError)(details.provider_error))
|
|
69
157
|
|| typeof details.provider_session_id !== "string" || !details.provider_session_id
|
|
70
158
|
|| typeof turnKey !== "string" || !turnKey || (streamKey ? details.request_proof.session_id !== details.provider_session_id || details.native_turn_id !== undefined : details.native_turn_id !== details.turn_id)
|
|
71
159
|
|| expectedSessionId !== undefined && details.provider_session_id !== expectedSessionId
|
|
@@ -123,13 +211,16 @@ export function capacityBackoff(error, ordinal, now = Date.now(), random = Math.
|
|
|
123
211
|
|
|
124
212
|
export function refusalObservation(error, ordinal, firstObservedAt) {
|
|
125
213
|
const failure = normalizeProviderFailure(error);
|
|
214
|
+
return observedRefusal(failure, ordinal, firstObservedAt, CAPACITY_POLICY);
|
|
215
|
+
}
|
|
216
|
+
function observedRefusal(failure, ordinal, firstObservedAt, policy) {
|
|
126
217
|
const observed = Date.parse(firstObservedAt);
|
|
127
218
|
if (!failure || !Number.isSafeInteger(ordinal) || ordinal < 0 || !Number.isFinite(observed))
|
|
128
219
|
throw policyError("capacity_continuation_conflict", "Refusal lacks an immutable native identity/time");
|
|
129
220
|
if (failure.observed_at !== null && (typeof failure.observed_at !== "string" || !Number.isFinite(Date.parse(failure.observed_at)) || Date.parse(failure.observed_at) !== observed))
|
|
130
221
|
throw policyError("capacity_continuation_conflict", "Native refusal observation time is invalid or changed");
|
|
131
222
|
const native = failure.terminal_at && Date.parse(failure.terminal_at);
|
|
132
|
-
return { policy
|
|
223
|
+
return { policy, ordinal, session_id: failure.session_id, turn_id: failure.turn_id,
|
|
133
224
|
request_key: failure.request_key, request_proof: failure.request_proof,
|
|
134
225
|
first_observed_at: firstObservedAt, ...(Number.isFinite(native) && typeof failure.terminal_basis === "string" && failure.terminal_basis ? { native_terminal_at: failure.terminal_at, native_terminal_basis: failure.terminal_basis } : {}) };
|
|
135
226
|
}
|