@deksden-com/dd-flow-cli 0.9.0-beta.150 → 0.9.0-beta.151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +17 -22
- package/dist/build-info.json +6 -6
- package/dist/harness-runtime/lib/agy-error-boundary.mjs +26 -5
- package/dist/harness-runtime/lib/agy-terminal-envelope.mjs +67 -0
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +10 -14
- package/dist/harness-runtime/lib/dd-agy.mjs +3 -3
- package/dist/harness-runtime/lib/dd-droid.mjs +2 -2
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +8 -3
- package/dist/harness-runtime/lib/dd-opencode.mjs +2 -2
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +1 -1
- package/dist/harness-runtime/lib/dd-zcode.mjs +37 -12
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +1 -1
- package/dist/harness-runtime/lib/delegation-instructions.mjs +13 -4
- package/dist/harness-runtime/lib/model-observations.mjs +19 -7
- package/dist/harness-runtime/lib/native-children.d.mts +12 -1
- package/dist/harness-runtime/lib/native-children.mjs +47 -1
- package/dist/harness-runtime/lib/provider-overload-policy.d.mts +9 -1
- package/dist/harness-runtime/lib/provider-overload-policy.mjs +97 -6
- package/dist/services/check-runner-evidence.js +17 -302
- package/dist/services/check-runner-proof.js +305 -0
- package/dist/services/code-checks.js +4 -1
- package/dist/services/controller-child-capacity.js +58 -0
- package/dist/services/controller-child-observer.js +9 -3
- package/dist/services/memory-init-native-capacity.js +1 -1
- package/dist/services/provider-capacity.js +15 -7
- package/dist/services/run-controller-adapter.js +1 -1
- package/dist/services/run-controller.js +87 -18
- package/dist/services/run-recovery-guard.js +1 -1
- package/dist/services/run-recovery-runtime.js +1 -1
- package/dist/services/stage-context.js +56 -8
- package/dist/services/vnext-fanout.js +1 -1
- package/dist/verification-components.json +299 -0
- package/package.json +7 -6
|
@@ -29,7 +29,7 @@ export async function readModelObservations(journal) {
|
|
|
29
29
|
|
|
30
30
|
/** Only adapter-verified native facts enter this file. It records changes,
|
|
31
31
|
* not every tool or token; reading it never requires replaying native history. */
|
|
32
|
-
export async function observeModel({ journal, harness, sessionId, parentSessionId = null, requested = {}, observed = {}, source, evidence = "configured", reason = null, nativeTimestamp = null, previousModel = null }) {
|
|
32
|
+
export async function observeModel({ journal, harness, sessionId, parentSessionId = null, requested = {}, observed = {}, source, evidence = "configured", reason = null, nativeTimestamp = null, previousModel = null, nativeAgentRole = null }) {
|
|
33
33
|
if (!journal || !sessionId) return null;
|
|
34
34
|
if (!source || !["configured", "request", "response", "unavailable"].includes(evidence)) throw new Error("model observation requires a native source and evidence level");
|
|
35
35
|
if (typeof source === "string") source = { channel: source, ...(["configured", "response"].includes(evidence) ? { scope: evidence } : {}) };
|
|
@@ -38,11 +38,17 @@ export async function observeModel({ journal, harness, sessionId, parentSessionI
|
|
|
38
38
|
return withRunnerLock(file, async () => {
|
|
39
39
|
const events = await readModelObservations(journal);
|
|
40
40
|
const replay = source?.event_sha256 ? events.find(item => item.harness === harness && item.session_id === sessionId && observationScope(item.source) === observationScope(source) && item.source?.event_sha256 === source.event_sha256) : null;
|
|
41
|
-
if (replay) return replay;
|
|
42
41
|
const previous = events.findLast(item => item.harness === harness && item.session_id === sessionId && observationScope(item.source) === observationScope(source));
|
|
43
42
|
const normalizedReason = text(reason)?.slice(0, 200) ?? null;
|
|
44
|
-
|
|
45
|
-
|
|
43
|
+
const tuple = item => JSON.stringify([observedFields(item.requested), item.observed, item.parent_session_id ?? null, item.source, item.reason, item.evidence, item.native_agent_role ?? null]);
|
|
44
|
+
const candidate = { requested: observedFields(requested), observed: profile, parent_session_id: parentSessionId, source, reason: normalizedReason, evidence, native_agent_role: nativeAgentRole };
|
|
45
|
+
if (replay && tuple(replay) === tuple(candidate)) return replay;
|
|
46
|
+
const conflict = replay ? { code: 'model_observation_identity_conflict', original_id: replay.id } : null;
|
|
47
|
+
if (conflict) {
|
|
48
|
+
const recorded = events.find(item => item.evidence_conflict?.original_id === replay.id && tuple(item) === tuple(candidate));
|
|
49
|
+
if (recorded) return recorded;
|
|
50
|
+
}
|
|
51
|
+
if (!conflict && previous && tuple(previous) === tuple(candidate)) return previous;
|
|
46
52
|
const previousKnown = events.findLast(item => item.harness === harness && item.session_id === sessionId && item.evidence === evidence && item.observed?.model
|
|
47
53
|
&& (item.source?.scope ?? "session") === (source?.scope ?? "session") && (item.source?.attempt_id ?? null) === (source?.attempt_id ?? null));
|
|
48
54
|
const fromModel = text(previousModel) ?? previousKnown?.observed.model ?? null;
|
|
@@ -53,7 +59,8 @@ export async function observeModel({ journal, harness, sessionId, parentSessionI
|
|
|
53
59
|
harness, session_id: sessionId, parent_session_id: parentSessionId,
|
|
54
60
|
kind: changed ? "model_changed" : !profile.model ? "model_observation_unavailable" : "model_observed",
|
|
55
61
|
previous_model: fromModel, requested: observedFields(requested), observed: profile,
|
|
56
|
-
source, evidence, reason: normalizedReason, native_timestamp: text(nativeTimestamp), observed_at: new Date().toISOString()
|
|
62
|
+
source, evidence, reason: normalizedReason, native_timestamp: text(nativeTimestamp), observed_at: new Date().toISOString(),
|
|
63
|
+
...(nativeAgentRole ? { native_agent_role: nativeAgentRole } : {}), ...(conflict ? { evidence_conflict: conflict } : {}) };
|
|
57
64
|
value.id = createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
|
58
65
|
const handle = await open(file, "a", 0o600);
|
|
59
66
|
try {
|
|
@@ -81,7 +88,12 @@ export function modelAttribution(events, expectedSessions = []) {
|
|
|
81
88
|
: !sessions.has(`${session.harness}:${session.session_id}`));
|
|
82
89
|
const models = [...new Set(events.map(event => event.observed?.model).filter(Boolean))];
|
|
83
90
|
const providers = [...new Set(events.map(event => event.observed?.provider).filter(Boolean))];
|
|
84
|
-
const
|
|
91
|
+
const conflicts = events.filter(event => event.evidence_conflict).map(event => ({ session_id: event.session_id, ...event.evidence_conflict }));
|
|
92
|
+
for (const session of sessions.values()) {
|
|
93
|
+
const parents = [...new Set(session.observations.filter(event => !event.source?.non_asserting && Object.hasOwn(event, 'parent_session_id')).map(event => event.parent_session_id))];
|
|
94
|
+
if (parents.length > 1) conflicts.push({ session_id: session.session_id, code: 'model_observation_parent_conflict' });
|
|
95
|
+
}
|
|
96
|
+
const incomplete = conflicts.length > 0 || !events.length || missingSessions.length > 0 || [...sessions.values()].some(session => {
|
|
85
97
|
const known = session.observations.filter(event => event.observed?.model);
|
|
86
98
|
if (!known.length) return true;
|
|
87
99
|
const knownScopes = new Set(known.map(event => `${event.evidence}:${event.source?.scope ?? "session"}:${event.source?.attempt_id ?? ""}:${event.source?.turn_id ?? ""}`));
|
|
@@ -97,5 +109,5 @@ export function modelAttribution(events, expectedSessions = []) {
|
|
|
97
109
|
return { models, providers, mixed: models.length > 1 || providers.length > 1 || events.some(event => event.observed?.model && event.requested?.model && event.observed.model !== event.requested.model
|
|
98
110
|
|| event.observed?.provider && event.requested?.provider && event.observed.provider !== event.requested.provider),
|
|
99
111
|
observation_completeness: incomplete ? "incomplete" : "available_native_sources", transitions: events.filter(event => event.kind === "model_changed"),
|
|
100
|
-
sessions: [...sessions.values()], missing_sessions: missingSessions, usage_attribution: "session_totals_unattributed_to_model", note: "Native settings/events do not prove undisclosed server routing or per-model token costs." };
|
|
112
|
+
sessions: [...sessions.values()], missing_sessions: missingSessions, ...(conflicts.length ? { evidence_conflicts: conflicts } : {}), usage_attribution: "session_totals_unattributed_to_model", note: "Native settings/events do not prove undisclosed server routing or per-model token costs." };
|
|
101
113
|
}
|
|
@@ -8,6 +8,12 @@ export type NativeTreeReadiness = "ready" | "owned_children_live" | "inspection_
|
|
|
8
8
|
export function nativeTreeReadiness(receipt: Record<string, unknown>, binding: { sessionId: string; operationId: string; turnId?: string; requestProof?: unknown; allowDescendants?: boolean }): NativeTreeReadiness;
|
|
9
9
|
export function settledNativeTree(root: string, children: Record<string, unknown>[], settled: boolean, rootSettled?: boolean): Record<string, unknown>;
|
|
10
10
|
export function safeNativeFailure(value: unknown): Record<string, unknown> | undefined;
|
|
11
|
+
export type NativeContinuationReference = { schema_id: "zcode/child-continuation@1"; capability: "dd-flow/managed-provider-recovery@1"; agent_id: string; child_session_id: string; parent_session_id: string; parent_tool_call_id: string; terminal_turn_id: string; started_at: number; ended_at: number; task_id?: string };
|
|
12
|
+
export function nativeContinuationReference(value: unknown, sessionId: string, parentId: string, turnId?: string): NativeContinuationReference | null;
|
|
13
|
+
export type NativeContinuationAdmission = { schema_id: "zcode/child-admission@1"; operation_id?: string; predecessor_turn_id?: string; parent_session_id: string; child_session_id: string; agent_id: string; parent_tool_call_id: string; parent_turn_id: string } & ({ proof_kind?: "message"; message_id: string; delivery: "queued" | "steered" | "resumed_background" } | { proof_kind: "parent_tool_successor"; successor_started_at: number; successor_turn_id: string; native_evidence_sha256: string } | { proof_kind: "parent_tool_admission"; successor_started_at: number; native_evidence_sha256: string });
|
|
14
|
+
export function nativeContinuationAdmission(value: unknown, sessionId: string, parentId: string): NativeContinuationAdmission | null;
|
|
15
|
+
export function nativeContinuationSuccessor(child: NativeChild, predecessorTurnId?: string, predecessorEndedAt?: number): boolean;
|
|
16
|
+
export function nativeContinuationAdmitted(child: NativeChild, predecessorTurnId?: string, predecessorEndedAt?: number): boolean;
|
|
11
17
|
export interface NativeChild {
|
|
12
18
|
session_id: string;
|
|
13
19
|
parent_session_id: string;
|
|
@@ -17,7 +23,12 @@ export interface NativeChild {
|
|
|
17
23
|
terminal_turn_id?: string;
|
|
18
24
|
terminal_observed_at?: string;
|
|
19
25
|
provider_error?: unknown;
|
|
20
|
-
|
|
26
|
+
recovery_unavailable_reason?: string;
|
|
27
|
+
continuation_ref?: NativeContinuationReference;
|
|
28
|
+
continuation_admission?: NativeContinuationAdmission;
|
|
29
|
+
resumed_from_message_id?: string;
|
|
30
|
+
current_turn_id?: string;
|
|
31
|
+
native_source?: "opencode.v2.session.execution" | "agy.result" | "acp.error_response" | "droid.agent_turn_outcome" | "grok.subagent.event" | "zcode.retained-terminal";
|
|
21
32
|
native_turn_items?: { observed?: boolean; possible_effects?: boolean; pending?: boolean };
|
|
22
33
|
}
|
|
23
34
|
export function normalizeNativeChildren(receipt: Record<string, unknown>, rootSessionId: string): NativeChild[];
|
|
@@ -125,6 +125,9 @@ export function safeNativeFailure(value, depth = 0) {
|
|
|
125
125
|
// invalid value must reach the policy as invalid, not disappear into default backoff.
|
|
126
126
|
const timing = safeProviderError(value);
|
|
127
127
|
if (Object.hasOwn(timing, "retryAfter")) result.retryAfter = timing.retryAfter;
|
|
128
|
+
if (Object.hasOwn(timing, "retry_after")) result.retry_after = timing.retry_after;
|
|
129
|
+
if (Object.hasOwn(timing, "retryAfterMs")) result.retryAfterMs = timing.retryAfterMs;
|
|
130
|
+
if (timing.headers) result.headers = timing.headers;
|
|
128
131
|
if (typeof value.codexErrorInfo === "string" && /^[\w-]{1,128}$/.test(value.codexErrorInfo)) result.codexErrorInfo = value.codexErrorInfo;
|
|
129
132
|
else if (value.codexErrorInfo && typeof value.codexErrorInfo.code === "string" && /^[\w-]{1,128}$/.test(value.codexErrorInfo.code)) result.codexErrorInfo = { code: value.codexErrorInfo.code };
|
|
130
133
|
const attribution = depth === 0 && value.attribution && safeNativeFailure(value.attribution, 1);
|
|
@@ -134,6 +137,40 @@ export function safeNativeFailure(value, depth = 0) {
|
|
|
134
137
|
|
|
135
138
|
|
|
136
139
|
/** Normalize provider-observed topology only; never parse assistant prose. */
|
|
140
|
+
export function nativeContinuationReference(value, sessionId, parentId, turnId) {
|
|
141
|
+
if (value?.schema_id !== "zcode/child-continuation@1" || value.capability !== "dd-flow/managed-provider-recovery@1"
|
|
142
|
+
|| value.child_session_id !== sessionId || value.parent_session_id !== parentId || value.terminal_turn_id !== turnId
|
|
143
|
+
|| !/^agent_[\w-]{1,200}$/.test(value.agent_id ?? "") || ![value.parent_tool_call_id, value.terminal_turn_id].every(item => typeof item === "string" && item.length > 0 && item.length <= 256)
|
|
144
|
+
|| !Number.isSafeInteger(value.started_at) || !Number.isSafeInteger(value.ended_at) || value.started_at <= 0 || value.ended_at < value.started_at) return null;
|
|
145
|
+
return { schema_id: value.schema_id, capability: value.capability, agent_id: value.agent_id, child_session_id: sessionId, parent_session_id: parentId,
|
|
146
|
+
parent_tool_call_id: value.parent_tool_call_id, terminal_turn_id: turnId, started_at: value.started_at, ended_at: value.ended_at,
|
|
147
|
+
...(typeof value.task_id === "string" && value.task_id.length <= 256 ? { task_id: value.task_id } : {}) };
|
|
148
|
+
}
|
|
149
|
+
export function nativeContinuationAdmission(value, sessionId, parentId) {
|
|
150
|
+
const keys = ["parent_tool_call_id", "parent_turn_id", "agent_id"];
|
|
151
|
+
if (value?.schema_id !== "zcode/child-admission@1" || value.child_session_id !== sessionId || value.parent_session_id !== parentId
|
|
152
|
+
|| !keys.every(key => typeof value[key] === "string" && value[key].length > 0 && value[key].length <= 512)) return null;
|
|
153
|
+
const successorProof = value.proof_kind === "parent_tool_successor";
|
|
154
|
+
const parentAdmission = successorProof || value.proof_kind === "parent_tool_admission";
|
|
155
|
+
if (parentAdmission ? !Number.isSafeInteger(value.successor_started_at) || value.successor_started_at <= 0 || successorProof && (typeof value.successor_turn_id !== "string" || !value.successor_turn_id || value.successor_turn_id.length > 512) || !/^[a-f0-9]{64}$/.test(value.native_evidence_sha256 ?? "")
|
|
156
|
+
: value.proof_kind !== undefined && value.proof_kind !== "message" || typeof value.message_id !== "string" || !value.message_id || value.message_id.length > 512 || !["queued", "steered", "resumed_background"].includes(value.delivery)) return null;
|
|
157
|
+
return Object.fromEntries(["schema_id", "child_session_id", "parent_session_id", ...keys,
|
|
158
|
+
...(parentAdmission ? ["proof_kind", "successor_started_at", "native_evidence_sha256", ...(successorProof ? ["successor_turn_id"] : [])] : ["message_id", "delivery"]),
|
|
159
|
+
...["operation_id", "predecessor_turn_id"].filter(key => typeof value[key] === "string" && value[key].length > 0 && value[key].length <= 512)].map(key => [key, value[key]]));
|
|
160
|
+
}
|
|
161
|
+
export function nativeContinuationSuccessor(child, predecessorTurnId, predecessorEndedAt) {
|
|
162
|
+
const admission = child?.continuation_admission, turn = child?.current_turn_id ?? child?.terminal_turn_id;
|
|
163
|
+
if (!admission?.operation_id || admission.predecessor_turn_id !== predecessorTurnId || typeof turn !== "string" || turn === predecessorTurnId) return false;
|
|
164
|
+
if (admission.proof_kind === "parent_tool_admission") return false;
|
|
165
|
+
return admission.proof_kind === "parent_tool_successor"
|
|
166
|
+
? Number.isSafeInteger(predecessorEndedAt) && admission.successor_started_at > predecessorEndedAt && admission.successor_turn_id === turn
|
|
167
|
+
: child.resumed_from_message_id === admission.message_id;
|
|
168
|
+
}
|
|
169
|
+
export function nativeContinuationAdmitted(child, predecessorTurnId, predecessorEndedAt) {
|
|
170
|
+
const admission = child?.continuation_admission;
|
|
171
|
+
return Boolean(admission?.proof_kind === "parent_tool_admission" && (child.status === "running" || child.status === "failed" && child.terminal_turn_id === predecessorTurnId) && admission.operation_id
|
|
172
|
+
&& admission.predecessor_turn_id === predecessorTurnId && Number.isSafeInteger(predecessorEndedAt) && admission.successor_started_at > predecessorEndedAt);
|
|
173
|
+
}
|
|
137
174
|
export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
138
175
|
const candidates = [];
|
|
139
176
|
const object = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
@@ -173,7 +210,12 @@ export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
|
173
210
|
...(text(value, ["terminal_turn_id"]) ? { terminal_turn_id: text(value, ["terminal_turn_id"]) } : {}),
|
|
174
211
|
...(text(value, ["terminal_observed_at"]) ? { terminal_observed_at: text(value, ["terminal_observed_at"]) } : {}),
|
|
175
212
|
...(value.provider_error !== undefined ? { provider_error: value.provider_error } : {}),
|
|
176
|
-
...(
|
|
213
|
+
...(typeof value.recovery_unavailable_reason === "string" && /^[a-z][a-z0-9_]{0,127}$/.test(value.recovery_unavailable_reason) ? { recovery_unavailable_reason: value.recovery_unavailable_reason } : {}),
|
|
214
|
+
...(nativeContinuationReference(value.continuation_ref, id, parent, value.terminal_turn_id) ? { continuation_ref: nativeContinuationReference(value.continuation_ref, id, parent, value.terminal_turn_id) } : {}),
|
|
215
|
+
...(nativeContinuationAdmission(value.continuation_admission, id, parent) ? { continuation_admission: nativeContinuationAdmission(value.continuation_admission, id, parent) } : {}),
|
|
216
|
+
...(text(value, ["resumed_from_message_id"]) ? { resumed_from_message_id: text(value, ["resumed_from_message_id"]) } : {}),
|
|
217
|
+
...(text(value, ["current_turn_id"]) ? { current_turn_id: text(value, ["current_turn_id"]) } : {}),
|
|
218
|
+
...(["opencode.v2.session.execution", "agy.result", "acp.error_response", "droid.agent_turn_outcome", "grok.subagent.event", "zcode.retained-terminal"].includes(value.native_source) ? { native_source: value.native_source } : {}),
|
|
177
219
|
...(value.native_turn_items && typeof value.native_turn_items === "object" ? { native_turn_items: value.native_turn_items } : {}) });
|
|
178
220
|
};
|
|
179
221
|
for (const value of list(receipt.descendants)) add(value, "adapter.descendants");
|
|
@@ -190,6 +232,10 @@ export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
|
190
232
|
const terminal = value => ["completed", "failed", "cancelled", "settled_by_root"].includes(value);
|
|
191
233
|
for (const child of candidates) {
|
|
192
234
|
const previous = unique.get(child.session_id);
|
|
235
|
+
const successor = (current, old) => current.parent_session_id === old.parent_session_id
|
|
236
|
+
&& (nativeContinuationSuccessor(current, old.terminal_turn_id, old.continuation_ref?.ended_at) || nativeContinuationAdmitted(current, old.terminal_turn_id, old.continuation_ref?.ended_at));
|
|
237
|
+
if (previous && successor(child, previous)) { unique.set(child.session_id, child); continue; }
|
|
238
|
+
if (previous && successor(previous, child)) continue;
|
|
193
239
|
if (previous && (terminal(previous.status) && child.native_turn_items?.pending === true
|
|
194
240
|
|| terminal(child.status) && previous.native_turn_items?.pending === true))
|
|
195
241
|
throw nativeFailure("native_child_observation_conflict", "Current child outcome contradicts pending native items", 1, { session_id: child.session_id, observations: [previous, child] });
|
|
@@ -1,11 +1,19 @@
|
|
|
1
1
|
export const CAPACITY_POLICY: "provider-overload-episode@1";
|
|
2
|
+
export const RECOVERY_POLICY: "provider-recovery-episode@1";
|
|
3
|
+
export function classifyRecoveryError(native: unknown): string;
|
|
4
|
+
export function recoveryRetryAfter(native: unknown, now?: number): number;
|
|
5
|
+
export const normalizeRecoveryFailure: typeof normalizeProviderFailure;
|
|
6
|
+
export const terminalProviderRecovery: typeof terminalProviderOverload;
|
|
7
|
+
export const recoveryBackoff: typeof capacityBackoff;
|
|
8
|
+
export const recoveryRefusal: typeof refusalObservation;
|
|
9
|
+
export const recoveryExhausted: typeof overloadExhausted;
|
|
2
10
|
export const MAX_OVERLOAD_RETRIES: 3;
|
|
3
11
|
export const MAX_RETRY_AFTER_MS: 30000;
|
|
4
12
|
export function classifyProviderError(native: unknown): string;
|
|
5
13
|
export function safeProviderError(native: unknown): Record<string, unknown>;
|
|
6
14
|
export type RequestProof = { kind: "native_turn"; session_id: string; turn_id: string } | { kind: "stream_request"; daemon_instance: string; connection_id: string; operation_id: string; turn_generation: number; session_id: string; outbound_position: number; terminal_position: number };
|
|
7
15
|
export function streamRequestKey(proof: unknown): string | null;
|
|
8
|
-
export type CapacityRefusal = { policy: string; ordinal: number; session_id: string; turn_id: string; request_key?: string; request_proof?: RequestProof; first_observed_at: string; native_terminal_at?: string; native_terminal_basis?: string };
|
|
16
|
+
export type CapacityRefusal = { policy: string; ordinal: number; session_id: string; turn_id: string; category?: string; request_key?: string; request_proof?: RequestProof; first_observed_at: string; native_terminal_at?: string; native_terminal_basis?: string };
|
|
9
17
|
export function parseCapacityOperation(id: string): { root: string; ordinal: number };
|
|
10
18
|
export function normalizeProviderFailure(error: unknown, sessionId?: string, turnId?: string): null | { code: string; session_id: string; turn_id: string; request_key: string; request_proof: RequestProof; native_error: Record<string, unknown>; observed_at: unknown; terminal_at: unknown; native_turn_items: unknown; primary_error: { details?: Record<string, unknown> } };
|
|
11
19
|
export function terminalProviderOverload(error: unknown, sessionId?: string, turnId?: string): boolean;
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
export const CAPACITY_POLICY = "provider-overload-episode@1";
|
|
3
3
|
export const MAX_OVERLOAD_RETRIES = 3;
|
|
4
4
|
export const MAX_RETRY_AFTER_MS = 30_000;
|
|
5
|
+
export const RECOVERY_POLICY = "provider-recovery-episode@1";
|
|
5
6
|
|
|
6
7
|
function policyError(code, message, details = {}) { return Object.assign(new Error(message), { code, details }); }
|
|
7
8
|
|
|
@@ -25,10 +26,28 @@ export function safeProviderError(native) {
|
|
|
25
26
|
if (!native || typeof native !== "object") return result;
|
|
26
27
|
const status = native.status ?? native.statusCode ?? native.status_code ?? native.http_status;
|
|
27
28
|
if (Number.isInteger(Number(status))) result.status = Number(status);
|
|
28
|
-
const
|
|
29
|
-
const
|
|
29
|
+
const retryHeaders = Object.entries(native.headers ?? {}).filter(([key]) => key.toLowerCase() === "retry-after");
|
|
30
|
+
const header = retryHeaders[0]?.[1];
|
|
31
|
+
const retry = native.retryAfter ?? native.retry_after ?? header;
|
|
30
32
|
if (retry !== undefined && retry !== null) result.retryAfter = typeof retry === "string" && retry.length <= 128 || typeof retry === "number" && Number.isFinite(retry) ? retry : "invalid-retry-after";
|
|
33
|
+
if (native.retry_after !== undefined && native.retry_after !== null) result.retry_after = typeof native.retry_after === "string" && native.retry_after.length <= 128 || typeof native.retry_after === "number" && Number.isFinite(native.retry_after) ? native.retry_after : "invalid-retry-after";
|
|
34
|
+
if (header !== undefined && header !== null && (native.retryAfter !== undefined || native.retry_after !== undefined || retryHeaders.length > 1)) result.headers = Object.fromEntries(retryHeaders.map(([key, value]) => [key, typeof value === "string" && value.length <= 128 || typeof value === "number" && Number.isFinite(value) ? value : "invalid-retry-after"]));
|
|
31
35
|
if (typeof native.willRetry === "boolean") result.willRetry = native.willRetry;
|
|
36
|
+
for (const key of ["type", "source", "reason", "errorPhase", "exceptionKind", "transport", "retryable", "retryAfterMs"]) {
|
|
37
|
+
const value = native[key];
|
|
38
|
+
if (typeof value === "boolean" || typeof value === "number" && Number.isFinite(value) || typeof value === "string" && value.length <= 128) result[key] = value;
|
|
39
|
+
}
|
|
40
|
+
if (!result.type && /^(model_request_failed|ECONNRESET|EPIPE|ETIMEDOUT|EAI_AGAIN|UND_ERR_SOCKET|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT)$/.test(native.code ?? "")) result.type = native.code;
|
|
41
|
+
if (Object.hasOwn(native, "retryAfterMs") && native.retryAfterMs !== null) result.retryAfterMs = typeof native.retryAfterMs === "number" && Number.isFinite(native.retryAfterMs) || typeof native.retryAfterMs === "string" && native.retryAfterMs.length <= 128 ? native.retryAfterMs : "invalid-retry-after";
|
|
42
|
+
if (native.attribution && typeof native.attribution === "object") {
|
|
43
|
+
const attribution = {};
|
|
44
|
+
for (const key of ["source", "reason", "errorPhase", "exceptionKind", "transport", "retryable", "retryAfterMs"]) {
|
|
45
|
+
const value = native.attribution[key];
|
|
46
|
+
if (typeof value === "boolean" || typeof value === "number" && Number.isFinite(value) || typeof value === "string" && value.length <= 128) attribution[key] = value;
|
|
47
|
+
}
|
|
48
|
+
if (Object.hasOwn(native.attribution, "retryAfterMs") && native.attribution.retryAfterMs !== null) attribution.retryAfterMs = typeof native.attribution.retryAfterMs === "number" && Number.isFinite(native.attribution.retryAfterMs) || typeof native.attribution.retryAfterMs === "string" && native.attribution.retryAfterMs.length <= 128 ? native.attribution.retryAfterMs : "invalid-retry-after";
|
|
49
|
+
if (Object.keys(attribution).length) result.attribution = attribution;
|
|
50
|
+
}
|
|
32
51
|
for (const key of ["reset_at", "resets_at", "resetAt", "resetsAt", "reset_time"]) {
|
|
33
52
|
const value = native[key];
|
|
34
53
|
if (typeof value === "string" && value.length <= 128 || typeof value === "number" && Number.isFinite(value)) result[key] = value;
|
|
@@ -36,6 +55,70 @@ export function safeProviderError(native) {
|
|
|
36
55
|
return result;
|
|
37
56
|
}
|
|
38
57
|
|
|
58
|
+
/** New native-recovery ABI; the overload-only exports below retain their scope. */
|
|
59
|
+
export function classifyRecoveryError(native) {
|
|
60
|
+
const legacy = classifyProviderError(native);
|
|
61
|
+
if (["provider_quota_exhausted", "provider_auth_failed"].includes(legacy)) return legacy;
|
|
62
|
+
if (!native || typeof native !== "object") return legacy;
|
|
63
|
+
const facts = [native, native.attribution].filter(value => value && typeof value === "object");
|
|
64
|
+
const text = [native.code, native.type, native.message, ...facts.flatMap(value => [value.reason, value.exceptionKind])].filter(value => typeof value === "string").join(" ");
|
|
65
|
+
if (facts.some(value => value.retryable === false) || /cancel|CERT_|certificate|permission|configuration/i.test(text)) return "provider_failed";
|
|
66
|
+
if (legacy === "provider_capacity_transient") return legacy;
|
|
67
|
+
if (["provider_rate_limited", "provider_limit_unknown"].includes(legacy) && facts.some(value => value.retryAfterMs !== undefined)) return "provider_rate_limited";
|
|
68
|
+
if (legacy === "provider_rate_limited") return legacy;
|
|
69
|
+
const structuredNetwork = [native.type, native.code].includes("model_request_failed") && facts.some(value => value.source === "network" && value.reason === "network_error" && value.errorPhase === "stream" && value.exceptionKind === "transport");
|
|
70
|
+
const networkSource = facts.some(value => ["network", "provider", "transport"].includes(value.source));
|
|
71
|
+
const code = [native.code, native.type, native.transport, ...facts.map(value => value.transport)].filter(value => typeof value === "string");
|
|
72
|
+
const knownCode = code.some(value => /^(ECONNRESET|EPIPE|ETIMEDOUT|EAI_AGAIN|UND_ERR_SOCKET|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT)$/.test(value));
|
|
73
|
+
const status = Number(native.status ?? native.statusCode ?? native.status_code ?? native.http_status);
|
|
74
|
+
return structuredNetwork || networkSource && (knownCode || [408, 502, 504].includes(status)) ? "provider_network_transient" : legacy;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** All delay representations must agree; no units guessing and no moving deadline. */
|
|
78
|
+
export function recoveryRetryAfter(native, now = Date.now()) {
|
|
79
|
+
if (!Number.isFinite(now)) throw policyError("capacity_retry_metadata_invalid", "Invalid recovery observation time");
|
|
80
|
+
if (!native || typeof native !== "object") return 0;
|
|
81
|
+
const values = [];
|
|
82
|
+
for (const source of [native, native.attribution].filter(value => value && typeof value === "object")) {
|
|
83
|
+
for (const key of ["retryAfterMs", "retryAfter", "retry_after"]) if (source[key] !== undefined && source[key] !== null) values.push([source[key], key === "retryAfterMs"]);
|
|
84
|
+
for (const [key, value] of Object.entries(source.headers ?? {})) if (key.toLowerCase() === "retry-after") values.push([value, false]);
|
|
85
|
+
}
|
|
86
|
+
const parsed = values.map(([value, milliseconds]) => {
|
|
87
|
+
const numeric = typeof value === "number" || typeof value === "string" && /^\d+(?:\.\d+)?$/.test(value.trim());
|
|
88
|
+
const duration = numeric ? Number(value) * (milliseconds ? 1 : 1000) : !milliseconds && typeof value === "string" && /^\w{3}, \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(value.trim()) ? Math.max(0, Date.parse(value) - now) : NaN;
|
|
89
|
+
if (!numeric && Number.isFinite(duration) && new Date(Date.parse(value)).toUTCString() !== value.trim()) throw policyError("capacity_retry_metadata_invalid", "Invalid recovery HTTP date");
|
|
90
|
+
if (!Number.isFinite(duration) || duration < 0 || !Number.isFinite(now + duration) || now + duration > 8.64e15) throw policyError("capacity_retry_metadata_invalid", "Invalid recovery Retry-After metadata");
|
|
91
|
+
return duration;
|
|
92
|
+
});
|
|
93
|
+
if (parsed.some(value => value !== parsed[0])) throw policyError("capacity_retry_metadata_invalid", "Conflicting recovery Retry-After metadata");
|
|
94
|
+
return parsed[0] ?? 0;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function normalizeRecoveryFailure(error, expectedSessionId, expectedTurnId) {
|
|
98
|
+
const normalized = normalizeNativeFailure(error, expectedSessionId, expectedTurnId, true);
|
|
99
|
+
if (!normalized) return null;
|
|
100
|
+
const category = classifyRecoveryError(normalized.native_error);
|
|
101
|
+
if (!["provider_capacity_transient", "provider_rate_limited", "provider_network_transient"].includes(category)) return null;
|
|
102
|
+
const nativeCode = typeof normalized.native_error.codexErrorInfo === "string" ? normalized.native_error.codexErrorInfo : normalized.native_error.codexErrorInfo?.code;
|
|
103
|
+
return { ...normalized, code: nativeCode ?? category, category };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function terminalProviderRecovery(error, sessionId, turnId) { return normalizeRecoveryFailure(error, sessionId, turnId) !== null; }
|
|
107
|
+
export function recoveryBackoff(error, ordinal, now = Date.now(), random = Math.random) {
|
|
108
|
+
const native = normalizeRecoveryFailure(error)?.native_error ?? error?.details?.provider_error ?? error;
|
|
109
|
+
try { return capacityBackoff({ retryAfter: recoveryRetryAfter(native, now) / 1000 }, ordinal, now, random); }
|
|
110
|
+
catch (failure) { if (failure.details?.policy === CAPACITY_POLICY) failure.details = { ...failure.details, policy: RECOVERY_POLICY }; throw failure; }
|
|
111
|
+
}
|
|
112
|
+
export function recoveryRefusal(error, ordinal, firstObservedAt) {
|
|
113
|
+
const failure = normalizeRecoveryFailure(error);
|
|
114
|
+
return { ...observedRefusal(failure, ordinal, firstObservedAt, RECOVERY_POLICY), category: failure.category };
|
|
115
|
+
}
|
|
116
|
+
export function recoveryExhausted(refusals, maxRetries = MAX_OVERLOAD_RETRIES) {
|
|
117
|
+
if (refusals.some(value => value.policy !== RECOVERY_POLICY)) throw policyError("capacity_policy_mismatch", "Retained episode uses another recovery policy");
|
|
118
|
+
const exhausted = overloadExhausted(refusals.map(value => ({ ...value, policy: CAPACITY_POLICY })), maxRetries);
|
|
119
|
+
return exhausted ? { ...exhausted, policy: RECOVERY_POLICY, reason: "provider_retries_exhausted", refusals } : null;
|
|
120
|
+
}
|
|
121
|
+
|
|
39
122
|
export function parseCapacityOperation(id) {
|
|
40
123
|
if (typeof id !== "string" || !id) throw policyError("capacity_continuation_conflict", "Missing capacity operation identity");
|
|
41
124
|
const match = /^(.*):capacity:(\d+):session\.prompt$/.exec(id);
|
|
@@ -55,17 +138,22 @@ export function streamRequestKey(proof) {
|
|
|
55
138
|
}
|
|
56
139
|
|
|
57
140
|
export function normalizeProviderFailure(error, expectedSessionId, expectedTurnId) {
|
|
141
|
+
return normalizeNativeFailure(error, expectedSessionId, expectedTurnId, false);
|
|
142
|
+
}
|
|
143
|
+
function normalizeNativeFailure(error, expectedSessionId, expectedTurnId, recovery) {
|
|
58
144
|
let current = error;
|
|
59
145
|
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth++) {
|
|
60
146
|
// The same continuation authority applies to a proven v2 execution
|
|
61
147
|
// refusal. Never classify transport loss or an active native retry here.
|
|
62
|
-
if (current.code === "provider_capacity_transient" || current.code === "provider_rate_limited") {
|
|
148
|
+
if (current.code === "provider_capacity_transient" || current.code === "provider_rate_limited" || recovery && current.code === "provider_network_transient") {
|
|
63
149
|
const details = current.details ?? {};
|
|
64
150
|
const streamKey = details.native_source === "agy.result" && details.turn_id === undefined && details.native_turn_id === undefined ? streamRequestKey(details.request_proof) : null;
|
|
65
151
|
const turnKey = streamKey ?? details.turn_id;
|
|
66
|
-
|
|
152
|
+
const sources = ["opencode.v2.session.execution", "agy.result", "acp.error_response", "droid.agent_turn_outcome", "grok.subagent.event", ...(recovery ? ["zcode.retained-terminal"] : [])];
|
|
153
|
+
const categories = ["provider_capacity_transient", "provider_rate_limited", ...(recovery ? ["provider_network_transient"] : [])];
|
|
154
|
+
if (!sources.includes(details.native_source) || details.terminal_status !== "failed"
|
|
67
155
|
|| details.will_retry === true || details.native_retry_active === true || details.provider_error?.willRetry === true
|
|
68
|
-
|| details.provider_error && !
|
|
156
|
+
|| details.provider_error && !categories.includes((recovery ? classifyRecoveryError : classifyProviderError)(details.provider_error))
|
|
69
157
|
|| typeof details.provider_session_id !== "string" || !details.provider_session_id
|
|
70
158
|
|| typeof turnKey !== "string" || !turnKey || (streamKey ? details.request_proof.session_id !== details.provider_session_id || details.native_turn_id !== undefined : details.native_turn_id !== details.turn_id)
|
|
71
159
|
|| expectedSessionId !== undefined && details.provider_session_id !== expectedSessionId
|
|
@@ -123,13 +211,16 @@ export function capacityBackoff(error, ordinal, now = Date.now(), random = Math.
|
|
|
123
211
|
|
|
124
212
|
export function refusalObservation(error, ordinal, firstObservedAt) {
|
|
125
213
|
const failure = normalizeProviderFailure(error);
|
|
214
|
+
return observedRefusal(failure, ordinal, firstObservedAt, CAPACITY_POLICY);
|
|
215
|
+
}
|
|
216
|
+
function observedRefusal(failure, ordinal, firstObservedAt, policy) {
|
|
126
217
|
const observed = Date.parse(firstObservedAt);
|
|
127
218
|
if (!failure || !Number.isSafeInteger(ordinal) || ordinal < 0 || !Number.isFinite(observed))
|
|
128
219
|
throw policyError("capacity_continuation_conflict", "Refusal lacks an immutable native identity/time");
|
|
129
220
|
if (failure.observed_at !== null && (typeof failure.observed_at !== "string" || !Number.isFinite(Date.parse(failure.observed_at)) || Date.parse(failure.observed_at) !== observed))
|
|
130
221
|
throw policyError("capacity_continuation_conflict", "Native refusal observation time is invalid or changed");
|
|
131
222
|
const native = failure.terminal_at && Date.parse(failure.terminal_at);
|
|
132
|
-
return { policy
|
|
223
|
+
return { policy, ordinal, session_id: failure.session_id, turn_id: failure.turn_id,
|
|
133
224
|
request_key: failure.request_key, request_proof: failure.request_proof,
|
|
134
225
|
first_observed_at: firstObservedAt, ...(Number.isFinite(native) && typeof failure.terminal_basis === "string" && failure.terminal_basis ? { native_terminal_at: failure.terminal_at, native_terminal_basis: failure.terminal_basis } : {}) };
|
|
135
226
|
}
|