@wrongstack/core 0.310.1 → 0.313.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chronicle/index.js +84 -6391
- package/dist/chronicle/project-server.js +84 -4868
- package/dist/chunk-2BUUIMQB.js +225 -0
- package/dist/chunk-2FK4WTFF.js +207 -0
- package/dist/chunk-2FQL4PDC.js +292 -0
- package/dist/chunk-2KKODIQU.js +2256 -0
- package/dist/chunk-2YU4WVBW.js +134 -0
- package/dist/chunk-3E244UDA.js +356 -0
- package/dist/chunk-3XPVB7BA.js +135 -0
- package/dist/chunk-4LJJYBAU.js +388 -0
- package/dist/chunk-4VWO3ZXL.js +342 -0
- package/dist/chunk-4XV6IR37.js +187 -0
- package/dist/chunk-4ZHTKJHM.js +183 -0
- package/dist/chunk-52RTVH4M.js +361 -0
- package/dist/chunk-57GEWK5X.js +26 -0
- package/dist/chunk-5DSMEVA4.js +931 -0
- package/dist/chunk-5RP3QTRL.js +1223 -0
- package/dist/chunk-5S437PRC.js +50 -0
- package/dist/chunk-5XDUGTKN.js +511 -0
- package/dist/chunk-62V2AFVG.js +422 -0
- package/dist/chunk-65RDIOR2.js +2470 -0
- package/dist/chunk-6AKOTGLM.js +597 -0
- package/dist/chunk-7ZHZE5EO.js +28 -0
- package/dist/chunk-A455ATZB.js +140 -0
- package/dist/chunk-AD7ZFZCS.js +46 -0
- package/dist/chunk-AI4LLQ2U.js +856 -0
- package/dist/chunk-ASX6AW36.js +789 -0
- package/dist/chunk-AT4V4XGI.js +3522 -0
- package/dist/chunk-AWNQ6ED6.js +311 -0
- package/dist/chunk-B4AMLZPB.js +9 -0
- package/dist/chunk-BAKVJLEV.js +417 -0
- package/dist/chunk-BJHVULHO.js +55 -0
- package/dist/chunk-CB2HFKTP.js +9 -0
- package/dist/chunk-CE4ZOCOI.js +1640 -0
- package/dist/chunk-CQQL4ZOQ.js +151 -0
- package/dist/chunk-DEIQMBD2.js +65 -0
- package/dist/chunk-E6I5RXVK.js +58 -0
- package/dist/chunk-EFEHDQGY.js +1589 -0
- package/dist/chunk-EODDVJBM.js +82 -0
- package/dist/chunk-EPC3GZD4.js +308 -0
- package/dist/chunk-FAYRX2AD.js +13 -0
- package/dist/chunk-FFSX2QDQ.js +362 -0
- package/dist/chunk-FHAMQS7Q.js +5856 -0
- package/dist/chunk-FIV4IJL7.js +2263 -0
- package/dist/chunk-FNXVT7HF.js +52 -0
- package/dist/chunk-FSWCWSTS.js +395 -0
- package/dist/chunk-FW5LDDMJ.js +16464 -0
- package/dist/chunk-FXQLIX3K.js +211 -0
- package/dist/chunk-G5AAKEEZ.js +228 -0
- package/dist/chunk-G5S3S5ZN.js +38 -0
- package/dist/chunk-GAZEWQEZ.js +231 -0
- package/dist/chunk-GFJOMSHG.js +125 -0
- package/dist/chunk-GFXZTHNI.js +158 -0
- package/dist/chunk-GM54GJBU.js +960 -0
- package/dist/chunk-GPQ3NJPR.js +900 -0
- package/dist/chunk-GTCU6DZ3.js +1 -0
- package/dist/chunk-H5E77KUJ.js +15 -0
- package/dist/chunk-H72H4XDU.js +1023 -0
- package/dist/chunk-HA4ZYA6T.js +178 -0
- package/dist/chunk-I45KAW2N.js +5020 -0
- package/dist/chunk-IQ35HMQL.js +1396 -0
- package/dist/chunk-IYFBFX34.js +86 -0
- package/dist/chunk-JJPMWPB6.js +109 -0
- package/dist/chunk-JTHKOYWU.js +663 -0
- package/dist/chunk-JVGPR3TS.js +365 -0
- package/dist/chunk-JVKB2TBL.js +48 -0
- package/dist/chunk-JX7VBZDK.js +31 -0
- package/dist/chunk-K3G6JBUC.js +5232 -0
- package/dist/chunk-K4YUNKU7.js +589 -0
- package/dist/chunk-KBZGE6LN.js +710 -0
- package/dist/chunk-KDQJCT2J.js +45 -0
- package/dist/chunk-KI4L2DUL.js +560 -0
- package/dist/chunk-KJ2ZTQSL.js +47 -0
- package/dist/chunk-KSJ5UTP7.js +154 -0
- package/dist/chunk-LHOA5DWA.js +28 -0
- package/dist/chunk-LX5QWLBL.js +27 -0
- package/dist/chunk-LZDDLNCZ.js +279 -0
- package/dist/chunk-M2WF6AXG.js +115 -0
- package/dist/chunk-M7BS2K6U.js +1 -0
- package/dist/chunk-NDBIPFNY.js +102 -0
- package/dist/chunk-NEMIWEXC.js +1199 -0
- package/dist/chunk-NFAYHPRH.js +323 -0
- package/dist/chunk-O37BQ65S.js +21 -0
- package/dist/chunk-O74MJHXU.js +187 -0
- package/dist/chunk-OJZEX73F.js +112 -0
- package/dist/chunk-PAAWLLZX.js +74 -0
- package/dist/chunk-PFXFQQRT.js +339 -0
- package/dist/chunk-PFZO3C6H.js +14 -0
- package/dist/chunk-PHH7LD45.js +113 -0
- package/dist/chunk-QRTG4XTI.js +188 -0
- package/dist/chunk-QVOVMXJR.js +1351 -0
- package/dist/chunk-RCQYT2I4.js +757 -0
- package/dist/chunk-RD6DEE36.js +1178 -0
- package/dist/chunk-RJIZEJ42.js +30 -0
- package/dist/chunk-RPOA7IAE.js +150 -0
- package/dist/chunk-SN2DQRL6.js +1172 -0
- package/dist/chunk-SQJVFEBV.js +41 -0
- package/dist/chunk-SQP64RCS.js +5391 -0
- package/dist/chunk-SVT4RNTR.js +49 -0
- package/dist/chunk-SWUFN7XM.js +43 -0
- package/dist/chunk-SXMVYC5H.js +174 -0
- package/dist/chunk-SZ6AYUIZ.js +23 -0
- package/dist/chunk-T3KGZBU5.js +3171 -0
- package/dist/chunk-TBRL2RCC.js +9 -0
- package/dist/chunk-TRT2HRWZ.js +18 -0
- package/dist/chunk-TYW2GI5I.js +1077 -0
- package/dist/chunk-UIWONWVG.js +4959 -0
- package/dist/chunk-VB3H5WWB.js +151 -0
- package/dist/chunk-W4A6QA3D.js +258 -0
- package/dist/chunk-WBPVKEEE.js +227 -0
- package/dist/chunk-WEBMSQP2.js +123 -0
- package/dist/chunk-WOEZV3Q4.js +333 -0
- package/dist/chunk-WQQON3SV.js +473 -0
- package/dist/chunk-WQTOVMGV.js +119 -0
- package/dist/chunk-WZO23AWO.js +2779 -0
- package/dist/chunk-XBDP4QPW.js +7 -0
- package/dist/chunk-XEBHSJLT.js +103 -0
- package/dist/chunk-XQ2ECNK3.js +1 -0
- package/dist/chunk-XQ2XGIBF.js +2086 -0
- package/dist/chunk-Y2GE5TSI.js +355 -0
- package/dist/chunk-YBUJCZ57.js +351 -0
- package/dist/chunk-YNCVSFGF.js +149 -0
- package/dist/chunk-YUPMXOGP.js +282 -0
- package/dist/chunk-YY6APYUM.js +49 -0
- package/dist/chunk-Z2IJUO45.js +691 -0
- package/dist/chunk-ZEHUCJHY.js +129 -0
- package/dist/chunk-ZYU2YOJ6.js +1496 -0
- package/dist/chunk-ZZ4V3WIZ.js +163 -0
- package/dist/coordination/agents/index.js +145 -6260
- package/dist/coordination/index.js +365 -34323
- package/dist/coordination/mailbox-project-server.js +48 -3220
- package/dist/coordination/mailbox-types.d.ts +13 -1
- package/dist/coordination/provider-status-tracker.d.ts +4 -0
- package/dist/coordination/sqlite-mailbox.d.ts +2 -1
- package/dist/core/agent.d.ts +14 -1
- package/dist/core/index.js +267 -12479
- package/dist/core/model-ref.js +5 -23
- package/dist/core/provider-runner.d.ts +36 -1
- package/dist/design/index.js +50 -1087
- package/dist/execution/index.d.ts +1 -1
- package/dist/execution/index.js +1599 -21887
- package/dist/execution/prompt-enhancer.js +16 -535
- package/dist/execution/retry-policy.d.ts +31 -1
- package/dist/extension/index.js +7 -311
- package/dist/goal/index.js +33 -3435
- package/dist/hooks/index.js +14 -1138
- package/dist/hq/index.js +112 -5139
- package/dist/hq/protocol.js +21 -668
- package/dist/index.js +1410 -92655
- package/dist/infrastructure/index.js +64 -4811
- package/dist/kernel/events/provider-events.d.ts +15 -0
- package/dist/kernel/events/wrongtrace-events.d.ts +26 -0
- package/dist/kernel/events.d.ts +16 -7
- package/dist/kernel/index.js +14 -992
- package/dist/models/index.js +40 -1117
- package/dist/notifications/index.js +3 -108
- package/dist/observability/index.js +17 -279
- package/dist/plugin/index.js +584 -14835
- package/dist/prompts/index.js +14 -948
- package/dist/registry/index.js +9 -822
- package/dist/registry-R23UBE7H.js +14 -0
- package/dist/replay/index.js +6 -84
- package/dist/review-finding-store-DKAV3AS6.js +21 -0
- package/dist/review-report-store-OWY3EC3N.js +19 -0
- package/dist/security/index.js +114 -2202
- package/dist/session-catalog/index.js +36 -2103
- package/dist/session-catalog/project-server.js +42 -1435
- package/dist/skills/index.js +35 -1578
- package/dist/storage/index.js +182 -15600
- package/dist/tasking/index.js +10 -676
- package/dist/tools/index.js +52 -9589
- package/dist/types/config/autonomy.d.ts +5 -0
- package/dist/types/config/tools.d.ts +2 -2
- package/dist/types/config/ui.d.ts +1 -1
- package/dist/types/errors.d.ts +18 -0
- package/dist/types/index.js +121 -1341
- package/dist/types/provider-runner.d.ts +8 -0
- package/dist/types/session-markers.js +8 -144
- package/dist/utils/error.js +3 -4
- package/dist/utils/expect-defined.js +3 -9
- package/dist/utils/heap-watchdog.js +6 -706
- package/dist/utils/index.js +269 -5685
- package/dist/utils/sage-output-block.js +5 -42
- package/dist/wiring/proxy-rewrite.d.ts +92 -4
- package/dist/wiring/proxy-rewrite.js +20 -61
- package/dist/worktree/index.js +6 -887
- package/package.json +4 -4
|
@@ -0,0 +1,1023 @@
|
|
|
1
|
+
import {
|
|
2
|
+
deactivateProxyOnConnectionFailure,
|
|
3
|
+
waitForProxyRoutingSettle
|
|
4
|
+
} from "./chunk-4ZHTKJHM.js";
|
|
5
|
+
import {
|
|
6
|
+
scrubErrorText
|
|
7
|
+
} from "./chunk-PAAWLLZX.js";
|
|
8
|
+
import {
|
|
9
|
+
runWithNetworkTelemetry
|
|
10
|
+
} from "./chunk-HA4ZYA6T.js";
|
|
11
|
+
import {
|
|
12
|
+
FallbackProfileManager,
|
|
13
|
+
evaluateModelCalendar,
|
|
14
|
+
logicalCalendarTarget
|
|
15
|
+
} from "./chunk-WQQON3SV.js";
|
|
16
|
+
import {
|
|
17
|
+
createChroniclePromptManifest
|
|
18
|
+
} from "./chunk-2YU4WVBW.js";
|
|
19
|
+
import {
|
|
20
|
+
ProviderError,
|
|
21
|
+
isFallbackWorthy
|
|
22
|
+
} from "./chunk-NFAYHPRH.js";
|
|
23
|
+
import {
|
|
24
|
+
bindRequestProvider,
|
|
25
|
+
completePartialObject,
|
|
26
|
+
resolveEventSessionId
|
|
27
|
+
} from "./chunk-EFEHDQGY.js";
|
|
28
|
+
import {
|
|
29
|
+
isTextBlock,
|
|
30
|
+
isToolUseBlock
|
|
31
|
+
} from "./chunk-O37BQ65S.js";
|
|
32
|
+
import {
|
|
33
|
+
toWrongStackError
|
|
34
|
+
} from "./chunk-WOEZV3Q4.js";
|
|
35
|
+
|
|
36
|
+
// src/core/streaming-response-builder.ts
|
|
37
|
+
import { randomUUID } from "node:crypto";
|
|
38
|
+
var STREAM_DRAIN_TIMEOUT_MS = 500;
|
|
39
|
+
function buildResponse(state) {
|
|
40
|
+
const content = [];
|
|
41
|
+
for (const b of state.blockOrder) {
|
|
42
|
+
if (b.kind === "text") {
|
|
43
|
+
const txt = state.textBuffers[b.idx] ?? "";
|
|
44
|
+
if (txt) content.push({ type: "text", text: txt });
|
|
45
|
+
} else if (b.kind === "thinking") {
|
|
46
|
+
const t = state.thinking[b.idx];
|
|
47
|
+
if (!t) continue;
|
|
48
|
+
if (!t.textBuf && !t.signature) continue;
|
|
49
|
+
const block = { type: "thinking", thinking: t.textBuf };
|
|
50
|
+
if (t.signature) block.signature = t.signature;
|
|
51
|
+
if (t.providerMeta && Object.keys(t.providerMeta).length > 0) {
|
|
52
|
+
block.providerMeta = t.providerMeta;
|
|
53
|
+
}
|
|
54
|
+
content.push(block);
|
|
55
|
+
} else {
|
|
56
|
+
const tb = state.tools.get(b.id);
|
|
57
|
+
if (tb) {
|
|
58
|
+
const block = {
|
|
59
|
+
type: "tool_use",
|
|
60
|
+
id: b.id,
|
|
61
|
+
name: tb.name,
|
|
62
|
+
input: tb.input ?? {}
|
|
63
|
+
};
|
|
64
|
+
if (tb.providerMeta && Object.keys(tb.providerMeta).length > 0) {
|
|
65
|
+
block.providerMeta = tb.providerMeta;
|
|
66
|
+
}
|
|
67
|
+
content.push(block);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (content.length === 0) content.push({ type: "text", text: "" });
|
|
72
|
+
return { content, stopReason: state.stopReason, usage: state.usage, model: state.model };
|
|
73
|
+
}
|
|
74
|
+
function createStreamingState(model) {
|
|
75
|
+
return {
|
|
76
|
+
model,
|
|
77
|
+
stopReason: "end_turn",
|
|
78
|
+
usage: { input: 0, output: 0 },
|
|
79
|
+
textBuffers: [],
|
|
80
|
+
currentTextIndex: -1,
|
|
81
|
+
tools: /* @__PURE__ */ new Map(),
|
|
82
|
+
thinking: [],
|
|
83
|
+
currentThinkingIndex: -1,
|
|
84
|
+
blockOrder: []
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
function handleMessageStart(state, model) {
|
|
88
|
+
state.model = model;
|
|
89
|
+
}
|
|
90
|
+
function handleContentBlockStart(state, ev) {
|
|
91
|
+
const kind = ev.kind ?? "text";
|
|
92
|
+
if (kind === "text") {
|
|
93
|
+
state.currentTextIndex = state.textBuffers.length;
|
|
94
|
+
state.textBuffers.push("");
|
|
95
|
+
state.blockOrder.push({ kind: "text", idx: state.currentTextIndex });
|
|
96
|
+
} else if (kind === "tool_use") {
|
|
97
|
+
const id = ev.id ?? randomUUID();
|
|
98
|
+
state.tools.set(id, { name: ev.name ?? "unknown", partial: "" });
|
|
99
|
+
state.blockOrder.push({ kind: "tool", id });
|
|
100
|
+
state.currentTextIndex = -1;
|
|
101
|
+
} else if (kind === "thinking") {
|
|
102
|
+
state.currentThinkingIndex = state.thinking.length;
|
|
103
|
+
state.thinking.push({
|
|
104
|
+
textBuf: "",
|
|
105
|
+
...ev.providerMeta ? { providerMeta: ev.providerMeta } : {}
|
|
106
|
+
});
|
|
107
|
+
state.blockOrder.push({ kind: "thinking", idx: state.currentThinkingIndex });
|
|
108
|
+
state.currentTextIndex = -1;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
function handleContentBlockStop(state, ev) {
|
|
112
|
+
void state;
|
|
113
|
+
void ev;
|
|
114
|
+
}
|
|
115
|
+
function handleTextDelta(state, text) {
|
|
116
|
+
if (state.currentTextIndex === -1) {
|
|
117
|
+
state.currentTextIndex = state.textBuffers.length;
|
|
118
|
+
state.textBuffers.push("");
|
|
119
|
+
state.blockOrder.push({ kind: "text", idx: state.currentTextIndex });
|
|
120
|
+
}
|
|
121
|
+
state.textBuffers[state.currentTextIndex] = (state.textBuffers[state.currentTextIndex] ?? "") + text;
|
|
122
|
+
}
|
|
123
|
+
function handleToolUseStart(state, ev) {
|
|
124
|
+
state.currentTextIndex = -1;
|
|
125
|
+
state.tools.set(ev.id, { name: ev.name, partial: "" });
|
|
126
|
+
state.blockOrder.push({ kind: "tool", id: ev.id });
|
|
127
|
+
}
|
|
128
|
+
function handleToolUseInputDelta(state, ev) {
|
|
129
|
+
const t = state.tools.get(ev.id);
|
|
130
|
+
if (t) t.partial += ev.partial;
|
|
131
|
+
}
|
|
132
|
+
function safeJsonOrRaw(s) {
|
|
133
|
+
if (!s) return {};
|
|
134
|
+
try {
|
|
135
|
+
return JSON.parse(s);
|
|
136
|
+
} catch {
|
|
137
|
+
const repaired = completePartialObject(s);
|
|
138
|
+
try {
|
|
139
|
+
return JSON.parse(repaired);
|
|
140
|
+
} catch {
|
|
141
|
+
return { _raw: repaired };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
function handleToolUseStop(state, ev) {
|
|
146
|
+
const t = state.tools.get(ev.id);
|
|
147
|
+
if (t) {
|
|
148
|
+
t.input = ev.input !== void 0 ? ev.input : safeJsonOrRaw(t.partial);
|
|
149
|
+
if (ev.providerMeta) t.providerMeta = ev.providerMeta;
|
|
150
|
+
}
|
|
151
|
+
state.currentTextIndex = -1;
|
|
152
|
+
}
|
|
153
|
+
function handleThinkingStart(state, ev) {
|
|
154
|
+
state.currentThinkingIndex = state.thinking.length;
|
|
155
|
+
state.thinking.push({
|
|
156
|
+
textBuf: "",
|
|
157
|
+
...ev.providerMeta ? { providerMeta: ev.providerMeta } : {}
|
|
158
|
+
});
|
|
159
|
+
state.blockOrder.push({ kind: "thinking", idx: state.currentThinkingIndex });
|
|
160
|
+
state.currentTextIndex = -1;
|
|
161
|
+
}
|
|
162
|
+
function handleThinkingDelta(state, text) {
|
|
163
|
+
if (state.currentThinkingIndex === -1) {
|
|
164
|
+
handleThinkingStart(state, {});
|
|
165
|
+
}
|
|
166
|
+
const t = state.thinking[state.currentThinkingIndex];
|
|
167
|
+
if (t) t.textBuf += text;
|
|
168
|
+
}
|
|
169
|
+
function handleThinkingSignature(state, signature) {
|
|
170
|
+
if (state.currentThinkingIndex === -1) {
|
|
171
|
+
handleThinkingStart(state, {});
|
|
172
|
+
}
|
|
173
|
+
const t = state.thinking[state.currentThinkingIndex];
|
|
174
|
+
if (t) t.signature = signature;
|
|
175
|
+
}
|
|
176
|
+
function handleThinkingStop(state) {
|
|
177
|
+
state.currentThinkingIndex = -1;
|
|
178
|
+
}
|
|
179
|
+
function handleMessageStop(state, ev) {
|
|
180
|
+
state.stopReason = ev.stopReason ?? "end_turn";
|
|
181
|
+
state.usage = ev.usage ?? { input: 0, output: 0 };
|
|
182
|
+
}
|
|
183
|
+
async function streamProviderToResponse(provider, req, signal, ctx, events, logger) {
|
|
184
|
+
const state = createStreamingState(req.model);
|
|
185
|
+
const sessionId = resolveEventSessionId(ctx);
|
|
186
|
+
logger.debug("Stream started", { providerId: provider.id, model: req.model });
|
|
187
|
+
const TEXT_BATCH_SIZE = 4;
|
|
188
|
+
let pendingText = "";
|
|
189
|
+
let pendingCount = 0;
|
|
190
|
+
const flushText = () => {
|
|
191
|
+
if (pendingCount > 0) {
|
|
192
|
+
events.emit("provider.text_delta", { sessionId, ctx, text: pendingText });
|
|
193
|
+
pendingText = "";
|
|
194
|
+
pendingCount = 0;
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
const iter = provider.stream(req, { signal })[Symbol.asyncIterator]();
|
|
198
|
+
try {
|
|
199
|
+
for (; ; ) {
|
|
200
|
+
const next = await iter.next();
|
|
201
|
+
if (next.done) break;
|
|
202
|
+
const ev = next.value;
|
|
203
|
+
try {
|
|
204
|
+
switch (ev.type) {
|
|
205
|
+
case "message_start":
|
|
206
|
+
handleMessageStart(state, ev.model);
|
|
207
|
+
break;
|
|
208
|
+
case "content_block_start":
|
|
209
|
+
handleContentBlockStart(state, ev);
|
|
210
|
+
break;
|
|
211
|
+
case "content_block_stop":
|
|
212
|
+
handleContentBlockStop(state, ev);
|
|
213
|
+
break;
|
|
214
|
+
case "text_delta":
|
|
215
|
+
handleTextDelta(state, ev.text);
|
|
216
|
+
pendingText += ev.text;
|
|
217
|
+
pendingCount++;
|
|
218
|
+
if (pendingCount >= TEXT_BATCH_SIZE) flushText();
|
|
219
|
+
break;
|
|
220
|
+
case "tool_use_start": {
|
|
221
|
+
flushText();
|
|
222
|
+
const idVal = ev.id;
|
|
223
|
+
const nameVal = ev.name;
|
|
224
|
+
handleToolUseStart(state, { id: idVal, name: nameVal });
|
|
225
|
+
const emittedPayload = {
|
|
226
|
+
sessionId,
|
|
227
|
+
ctx,
|
|
228
|
+
id: idVal ?? "unknown",
|
|
229
|
+
name: nameVal ?? "unknown"
|
|
230
|
+
};
|
|
231
|
+
events.emit("provider.tool_use_start", emittedPayload);
|
|
232
|
+
break;
|
|
233
|
+
}
|
|
234
|
+
case "tool_use_input_delta":
|
|
235
|
+
handleToolUseInputDelta(state, ev);
|
|
236
|
+
break;
|
|
237
|
+
case "tool_use_stop": {
|
|
238
|
+
flushText();
|
|
239
|
+
const stoppedName = state.tools.get(ev.id)?.name ?? "unknown";
|
|
240
|
+
handleToolUseStop(state, ev);
|
|
241
|
+
events.emit("provider.tool_use_stop", { sessionId, ctx, id: ev.id, name: stoppedName });
|
|
242
|
+
break;
|
|
243
|
+
}
|
|
244
|
+
case "thinking_start":
|
|
245
|
+
handleThinkingStart(state, ev);
|
|
246
|
+
break;
|
|
247
|
+
case "thinking_delta":
|
|
248
|
+
flushText();
|
|
249
|
+
handleThinkingDelta(state, ev.text);
|
|
250
|
+
events.emit("provider.thinking_delta", { sessionId, ctx, text: ev.text });
|
|
251
|
+
break;
|
|
252
|
+
case "thinking_signature":
|
|
253
|
+
handleThinkingSignature(state, ev.signature);
|
|
254
|
+
break;
|
|
255
|
+
case "thinking_stop":
|
|
256
|
+
handleThinkingStop(state);
|
|
257
|
+
break;
|
|
258
|
+
case "message_stop":
|
|
259
|
+
handleMessageStop(state, ev);
|
|
260
|
+
break;
|
|
261
|
+
default: {
|
|
262
|
+
const unknownEv = ev;
|
|
263
|
+
logger.warn(`Stream received unknown event type: "${String(unknownEv.type)}"`, {
|
|
264
|
+
providerId: provider.id,
|
|
265
|
+
model: req.model,
|
|
266
|
+
eventType: String(unknownEv.type)
|
|
267
|
+
});
|
|
268
|
+
break;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
} catch (handlerErr) {
|
|
272
|
+
const errMsg = handlerErr instanceof Error ? handlerErr.message : String(handlerErr);
|
|
273
|
+
const evAny = ev;
|
|
274
|
+
logger.warn(`Stream handler error for event type "${String(evAny.type)}": ${errMsg}`, {
|
|
275
|
+
providerId: provider.id,
|
|
276
|
+
model: req.model,
|
|
277
|
+
eventType: String(evAny.type),
|
|
278
|
+
errorMessage: errMsg
|
|
279
|
+
});
|
|
280
|
+
flushText();
|
|
281
|
+
events.emit("provider.stream_error", {
|
|
282
|
+
sessionId,
|
|
283
|
+
ctx,
|
|
284
|
+
eventType: String(evAny.type),
|
|
285
|
+
msg: errMsg
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
} catch (err) {
|
|
290
|
+
if (signal.aborted) {
|
|
291
|
+
state.stopReason = "end_turn";
|
|
292
|
+
flushText();
|
|
293
|
+
logger.debug("Stream aborted \u2014 returning partial state", {
|
|
294
|
+
providerId: provider.id,
|
|
295
|
+
model: req.model,
|
|
296
|
+
textBlockCount: state.textBuffers.length,
|
|
297
|
+
toolBlockCount: state.tools.size,
|
|
298
|
+
thinkingBlockCount: state.thinking.length
|
|
299
|
+
});
|
|
300
|
+
return buildResponse(state);
|
|
301
|
+
}
|
|
302
|
+
throw err;
|
|
303
|
+
} finally {
|
|
304
|
+
try {
|
|
305
|
+
let drainTimer = null;
|
|
306
|
+
try {
|
|
307
|
+
const drainPromise = Promise.resolve(iter.return?.()).catch(() => {
|
|
308
|
+
});
|
|
309
|
+
await Promise.race([
|
|
310
|
+
drainPromise,
|
|
311
|
+
new Promise((resolve) => {
|
|
312
|
+
drainTimer = setTimeout(resolve, STREAM_DRAIN_TIMEOUT_MS);
|
|
313
|
+
})
|
|
314
|
+
]);
|
|
315
|
+
} finally {
|
|
316
|
+
if (drainTimer) clearTimeout(drainTimer);
|
|
317
|
+
}
|
|
318
|
+
} catch {
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
flushText();
|
|
322
|
+
logger.debug("Stream completed", {
|
|
323
|
+
providerId: provider.id,
|
|
324
|
+
model: req.model,
|
|
325
|
+
stopReason: state.stopReason,
|
|
326
|
+
textBlockCount: state.textBuffers.length,
|
|
327
|
+
toolBlockCount: state.tools.size,
|
|
328
|
+
thinkingBlockCount: state.thinking.length,
|
|
329
|
+
usageInput: state.usage.input,
|
|
330
|
+
usageOutput: state.usage.output
|
|
331
|
+
});
|
|
332
|
+
return buildResponse(state);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
// src/core/provider-runner.ts
|
|
336
|
+
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
337
|
+
function scrubProviderBody(body) {
|
|
338
|
+
if (!body) return void 0;
|
|
339
|
+
return {
|
|
340
|
+
...body,
|
|
341
|
+
...body.type !== void 0 ? { type: scrubErrorText(body.type) } : {},
|
|
342
|
+
...body.message !== void 0 ? { message: scrubErrorText(body.message) } : {},
|
|
343
|
+
...body.raw !== void 0 ? { raw: scrubErrorText(body.raw) } : {},
|
|
344
|
+
...body.requestId !== void 0 ? { requestId: scrubErrorText(body.requestId) } : {}
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
function providerLogCtx(p, r) {
|
|
348
|
+
return {
|
|
349
|
+
providerId: p.id,
|
|
350
|
+
model: r.model,
|
|
351
|
+
streaming: p.capabilities.streaming,
|
|
352
|
+
msgCount: r.messages.length,
|
|
353
|
+
toolCount: r.tools?.length ?? 0
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
var PROVIDER_FAILURE_TRACKED = /* @__PURE__ */ Symbol.for("wrongstack.providerFailureTracked");
|
|
357
|
+
function isProviderFailureTracked(err) {
|
|
358
|
+
return !!err && typeof err === "object" && err[PROVIDER_FAILURE_TRACKED] === true;
|
|
359
|
+
}
|
|
360
|
+
function markProviderFailureTracked(err) {
|
|
361
|
+
if (!err || typeof err !== "object") return;
|
|
362
|
+
try {
|
|
363
|
+
Object.defineProperty(err, PROVIDER_FAILURE_TRACKED, {
|
|
364
|
+
value: true,
|
|
365
|
+
enumerable: false,
|
|
366
|
+
configurable: true,
|
|
367
|
+
writable: true
|
|
368
|
+
});
|
|
369
|
+
} catch {
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
async function runProviderWithRetry(opts) {
|
|
373
|
+
const { provider, request, signal, ctx, events, retry, logger, tracer, statusTracker } = opts;
|
|
374
|
+
const logicalRequestId = randomUUID2();
|
|
375
|
+
const promptManifest = createChroniclePromptManifest(request);
|
|
376
|
+
ctx.activeLogicalRequestId = logicalRequestId;
|
|
377
|
+
ctx.activePromptManifestId = promptManifest.manifestId;
|
|
378
|
+
let attempt = 0;
|
|
379
|
+
for (; ; ) {
|
|
380
|
+
const currentProvider = ctx.provider && ctx.provider.id === provider.id ? ctx.provider : provider;
|
|
381
|
+
if (statusTracker && !statusTracker.isAvailable(currentProvider.id, request.model)) {
|
|
382
|
+
const status = statusTracker.getStatus(currentProvider.id, request.model);
|
|
383
|
+
const until = status?.stateExpiresAt;
|
|
384
|
+
const detail = status?.lastErrorMessage ?? "rate limit or repeated failures";
|
|
385
|
+
logger.warn(
|
|
386
|
+
`provider-gate: refusing "${currentProvider.id}/${request.model}" \u2014 quarantined${until ? ` for another ${Math.max(0, Math.round((until - Date.now()) / 1e3))}s` : ""} (${detail})`,
|
|
387
|
+
providerLogCtx(currentProvider, request)
|
|
388
|
+
);
|
|
389
|
+
const gateErr = new ProviderError(
|
|
390
|
+
`"${currentProvider.id}/${request.model}" is in the provider waiting room \u2014 skipped without calling the API (${detail})`,
|
|
391
|
+
429,
|
|
392
|
+
true,
|
|
393
|
+
currentProvider.id,
|
|
394
|
+
{
|
|
395
|
+
kind: "rate_limit",
|
|
396
|
+
body: {
|
|
397
|
+
type: "rate_limit_error",
|
|
398
|
+
message: detail,
|
|
399
|
+
...until ? { retryAfterMs: Math.max(0, until - Date.now()) } : {}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
);
|
|
403
|
+
markProviderFailureTracked(gateErr);
|
|
404
|
+
throw gateErr;
|
|
405
|
+
}
|
|
406
|
+
const attemptId = randomUUID2();
|
|
407
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
408
|
+
const startedNs = process.hrtime.bigint();
|
|
409
|
+
const correlation = {
|
|
410
|
+
sessionId: resolveEventSessionId(ctx),
|
|
411
|
+
...ctx.traceId ? { traceId: ctx.traceId } : {},
|
|
412
|
+
...ctx.agentId ? { agentId: ctx.agentId } : {},
|
|
413
|
+
logicalRequestId,
|
|
414
|
+
promptManifestId: promptManifest.manifestId,
|
|
415
|
+
attemptId,
|
|
416
|
+
attempt,
|
|
417
|
+
providerId: currentProvider.id,
|
|
418
|
+
model: request.model,
|
|
419
|
+
taskId: ctx.currentKanbanTaskId,
|
|
420
|
+
boardId: ctx.currentKanbanBoardId
|
|
421
|
+
};
|
|
422
|
+
events.emit("provider.attempt.started", {
|
|
423
|
+
...correlation,
|
|
424
|
+
streaming: currentProvider.capabilities.streaming,
|
|
425
|
+
messageCount: request.messages.length,
|
|
426
|
+
toolCount: request.tools?.length ?? 0,
|
|
427
|
+
promptManifest,
|
|
428
|
+
startedAt
|
|
429
|
+
});
|
|
430
|
+
const span = tracer?.startSpan("provider.complete", {
|
|
431
|
+
"provider.id": currentProvider.id,
|
|
432
|
+
"provider.model": request.model,
|
|
433
|
+
"provider.streaming": currentProvider.capabilities.streaming,
|
|
434
|
+
"provider.attempt": attempt
|
|
435
|
+
});
|
|
436
|
+
logger.debug(
|
|
437
|
+
`Provider attempt ${attempt + 1} starting`,
|
|
438
|
+
providerLogCtx(currentProvider, request)
|
|
439
|
+
);
|
|
440
|
+
try {
|
|
441
|
+
const res = await runWithNetworkTelemetry(
|
|
442
|
+
{
|
|
443
|
+
events,
|
|
444
|
+
sessionId: resolveEventSessionId(ctx),
|
|
445
|
+
...ctx.traceId ? { traceId: ctx.traceId } : {},
|
|
446
|
+
...ctx.agentId ? { agentId: ctx.agentId } : {},
|
|
447
|
+
attemptId,
|
|
448
|
+
initiator: "provider",
|
|
449
|
+
operationName: `${currentProvider.id}.complete`
|
|
450
|
+
},
|
|
451
|
+
() => currentProvider.capabilities.streaming ? streamProviderToResponse(currentProvider, request, signal, ctx, events, logger) : currentProvider.complete(request, { signal })
|
|
452
|
+
);
|
|
453
|
+
statusTracker?.recordSuccess(currentProvider.id, request.model, {
|
|
454
|
+
sessionId: resolveEventSessionId(ctx),
|
|
455
|
+
...ctx.agentId ? { agentId: ctx.agentId } : {}
|
|
456
|
+
});
|
|
457
|
+
span?.setAttribute("provider.stopReason", res.stopReason);
|
|
458
|
+
span?.setAttribute("provider.usage_in", res.usage.input);
|
|
459
|
+
span?.setAttribute("provider.usage_out", res.usage.output);
|
|
460
|
+
span?.end();
|
|
461
|
+
events.emit("provider.attempt.completed", {
|
|
462
|
+
...correlation,
|
|
463
|
+
startedAt,
|
|
464
|
+
endedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
465
|
+
durationMs: elapsedMs(startedNs),
|
|
466
|
+
stopReason: res.stopReason,
|
|
467
|
+
usage: res.usage
|
|
468
|
+
});
|
|
469
|
+
logger.debug("Provider call succeeded", {
|
|
470
|
+
...providerLogCtx(currentProvider, request),
|
|
471
|
+
stopReason: res.stopReason,
|
|
472
|
+
usageInput: res.usage.input,
|
|
473
|
+
usageOutput: res.usage.output,
|
|
474
|
+
cacheRead: res.usage.cacheRead,
|
|
475
|
+
cacheWrite: res.usage.cacheWrite,
|
|
476
|
+
attempts: attempt + 1
|
|
477
|
+
});
|
|
478
|
+
return res;
|
|
479
|
+
} catch (err) {
|
|
480
|
+
if (err instanceof Error) span?.recordError(err);
|
|
481
|
+
span?.end();
|
|
482
|
+
if (signal.aborted) throw err;
|
|
483
|
+
if (deactivateProxyOnConnectionFailure(err)) {
|
|
484
|
+
await waitForProxyRoutingSettle(void 0, signal);
|
|
485
|
+
if (signal.aborted) throw err;
|
|
486
|
+
logger.info(
|
|
487
|
+
"WrongProxy connection failed \u2014 deactivated proxy and switched to direct provider connection"
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
const isProviderErr = err instanceof ProviderError || ProviderError.isProviderError(err);
|
|
491
|
+
const errAsErr = err instanceof Error ? err : new Error(String(err));
|
|
492
|
+
const canRetry = retry.shouldRetry(isProviderErr ? err : errAsErr, attempt);
|
|
493
|
+
const providerErrorBody = isProviderErr ? scrubProviderBody(err.body) : void 0;
|
|
494
|
+
const description = scrubErrorText(
|
|
495
|
+
isProviderErr ? err.describe() : errAsErr.message
|
|
496
|
+
);
|
|
497
|
+
const delay = canRetry ? Math.round(retry.delayMs(attempt, isProviderErr ? err : errAsErr)) : void 0;
|
|
498
|
+
events.emit("provider.attempt.failed", {
|
|
499
|
+
...correlation,
|
|
500
|
+
startedAt,
|
|
501
|
+
endedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
502
|
+
durationMs: elapsedMs(startedNs),
|
|
503
|
+
status: isProviderErr ? err.status : 0,
|
|
504
|
+
failureKind: isProviderErr ? err.kind : "unknown",
|
|
505
|
+
description,
|
|
506
|
+
retryable: canRetry,
|
|
507
|
+
retryScheduled: canRetry,
|
|
508
|
+
...delay !== void 0 ? { retryDelayMs: delay } : {},
|
|
509
|
+
...providerErrorBody?.requestId ? { providerRequestId: providerErrorBody.requestId } : {},
|
|
510
|
+
...providerErrorBody ? { errorBody: providerErrorBody } : {}
|
|
511
|
+
});
|
|
512
|
+
if (!canRetry) {
|
|
513
|
+
if (statusTracker && isProviderErr && !isProviderFailureTracked(err)) {
|
|
514
|
+
const perr = err;
|
|
515
|
+
markProviderFailureTracked(err);
|
|
516
|
+
statusTracker.recordFailure(
|
|
517
|
+
currentProvider.id,
|
|
518
|
+
request.model,
|
|
519
|
+
perr.kind,
|
|
520
|
+
perr.status,
|
|
521
|
+
description,
|
|
522
|
+
{
|
|
523
|
+
sessionId: resolveEventSessionId(ctx),
|
|
524
|
+
...ctx.agentId ? { agentId: ctx.agentId } : {},
|
|
525
|
+
...perr.body?.retryAfterMs ? { retryAfterMs: perr.body.retryAfterMs } : {}
|
|
526
|
+
}
|
|
527
|
+
);
|
|
528
|
+
}
|
|
529
|
+
events.emit("provider.error", {
|
|
530
|
+
sessionId: resolveEventSessionId(ctx),
|
|
531
|
+
providerId: isProviderErr ? err.providerId : provider.id,
|
|
532
|
+
status: isProviderErr ? err.status : 0,
|
|
533
|
+
description,
|
|
534
|
+
retryable: false,
|
|
535
|
+
...providerErrorBody ? { errorBody: providerErrorBody } : {}
|
|
536
|
+
});
|
|
537
|
+
logger.error(`Provider call failed after ${attempt + 1} attempt(s) \u2014 ${description}`, {
|
|
538
|
+
...providerLogCtx(provider, request),
|
|
539
|
+
attempts: attempt + 1,
|
|
540
|
+
errorDescription: description,
|
|
541
|
+
status: isProviderErr ? err.status : void 0,
|
|
542
|
+
errorBody: providerErrorBody,
|
|
543
|
+
errorName: err instanceof Error ? err.name : void 0,
|
|
544
|
+
errorStack: err instanceof Error ? err.stack?.split("\n").slice(0, 3).join("\n") : void 0
|
|
545
|
+
});
|
|
546
|
+
throw isProviderErr ? err : toWrongStackError(err);
|
|
547
|
+
}
|
|
548
|
+
const attemptNum = attempt + 1;
|
|
549
|
+
const maxAttempts = retry.maxAttempts(isProviderErr ? err : errAsErr);
|
|
550
|
+
logger.warn(`Provider retry ${attemptNum}/${maxAttempts} in ${delay}ms \u2014 ${description}`, {
|
|
551
|
+
...providerLogCtx(provider, request),
|
|
552
|
+
attempt: attemptNum,
|
|
553
|
+
maxAttempts,
|
|
554
|
+
delayMs: delay,
|
|
555
|
+
errorDescription: description,
|
|
556
|
+
status: isProviderErr ? err.status : void 0,
|
|
557
|
+
errorBody: providerErrorBody
|
|
558
|
+
});
|
|
559
|
+
events.emit("provider.retry", {
|
|
560
|
+
sessionId: resolveEventSessionId(ctx),
|
|
561
|
+
providerId: isProviderErr ? err.providerId : provider.id,
|
|
562
|
+
attempt: attemptNum,
|
|
563
|
+
delayMs: delay,
|
|
564
|
+
status: isProviderErr ? err.status : 0,
|
|
565
|
+
description,
|
|
566
|
+
...providerErrorBody ? { errorBody: providerErrorBody } : {}
|
|
567
|
+
});
|
|
568
|
+
await new Promise((resolve, reject) => {
|
|
569
|
+
let settled = false;
|
|
570
|
+
const cleanup = () => {
|
|
571
|
+
clearTimeout(t);
|
|
572
|
+
signal.removeEventListener("abort", onAbort);
|
|
573
|
+
};
|
|
574
|
+
const onAbort = () => {
|
|
575
|
+
if (settled) return;
|
|
576
|
+
settled = true;
|
|
577
|
+
cleanup();
|
|
578
|
+
reject(new Error("aborted"));
|
|
579
|
+
};
|
|
580
|
+
const t = setTimeout(() => {
|
|
581
|
+
if (settled) return;
|
|
582
|
+
settled = true;
|
|
583
|
+
cleanup();
|
|
584
|
+
resolve();
|
|
585
|
+
}, delay);
|
|
586
|
+
if (signal.aborted) {
|
|
587
|
+
onAbort();
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
590
|
+
signal.addEventListener("abort", onAbort);
|
|
591
|
+
});
|
|
592
|
+
attempt++;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
function elapsedMs(startedNs) {
|
|
597
|
+
return Number(process.hrtime.bigint() - startedNs) / 1e6;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
// src/core/fallback-model.ts
|
|
601
|
+
import { randomUUID as randomUUID3 } from "node:crypto";
|
|
602
|
+
function fallbackProfileChain(config, profileName) {
|
|
603
|
+
if (!profileName) return [];
|
|
604
|
+
const mgr = new FallbackProfileManager(config);
|
|
605
|
+
return mgr.resolve(profileName).map((e) => `${e.providerId}/${e.model}`);
|
|
606
|
+
}
|
|
607
|
+
function shouldFallback(err) {
|
|
608
|
+
if (err instanceof ProviderError || ProviderError.isProviderError(err)) {
|
|
609
|
+
const kind = err.kind;
|
|
610
|
+
return isFallbackWorthy(kind) ? err.status : null;
|
|
611
|
+
}
|
|
612
|
+
if (err instanceof Error) {
|
|
613
|
+
const msg = err.message.toLowerCase();
|
|
614
|
+
if (msg.includes("econnrefused") || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("fetch failed") || msg.includes("failed to fetch") || msg.includes("network") || msg.includes("timeout") || msg.includes("503") || msg.includes("502") || msg.includes("504") || msg.includes("overloaded") || msg.includes("rate limit") || msg.includes("quota")) {
|
|
615
|
+
return 503;
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
return null;
|
|
619
|
+
}
|
|
620
|
+
function isUsableModelResponse(response) {
|
|
621
|
+
if (!response?.content) return void 0;
|
|
622
|
+
return response.content.some(
|
|
623
|
+
(block) => isToolUseBlock(block) || isTextBlock(block) && block.text.trim().length > 0
|
|
624
|
+
);
|
|
625
|
+
}
|
|
626
|
+
function ensureUsableModelResponse(response, providerId, model) {
|
|
627
|
+
const usable = isUsableModelResponse(response);
|
|
628
|
+
if (usable !== false) return response;
|
|
629
|
+
throw new ProviderError(
|
|
630
|
+
`Empty response from ${providerId}/${model}; trying the next configured model`,
|
|
631
|
+
503,
|
|
632
|
+
true,
|
|
633
|
+
providerId,
|
|
634
|
+
{ kind: "overloaded" }
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
function smartDefaultFallbackChain(config) {
|
|
638
|
+
const mgr = new FallbackProfileManager(config);
|
|
639
|
+
return mgr.resolveEffective({ fallbackAuto: true }).map((e) => `${e.providerId}/${e.model}`);
|
|
640
|
+
}
|
|
641
|
+
function effectiveFallbackChain(config) {
|
|
642
|
+
const mgr = new FallbackProfileManager(config);
|
|
643
|
+
return mgr.resolveEffective({
|
|
644
|
+
fallbackModels: config.fallbackModels,
|
|
645
|
+
fallbackProfile: config.fallbackProfile,
|
|
646
|
+
fallbackAuto: config.fallbackAuto
|
|
647
|
+
}).map((e) => `${e.providerId}/${e.model}`);
|
|
648
|
+
}
|
|
649
|
+
function runtimeFallbackChain(config) {
|
|
650
|
+
const mgr = new FallbackProfileManager(config);
|
|
651
|
+
const current = primaryTarget(config);
|
|
652
|
+
return mgr.resolveCandidates(current, {}).map((e) => `${e.providerId}/${e.model}`);
|
|
653
|
+
}
|
|
654
|
+
var DEFAULT_PRIMARY_COOLDOWN_MS = 6e4;
|
|
655
|
+
var DEFAULT_PRIMARY_COOLDOWN_MAX_MS = 10 * 6e4;
|
|
656
|
+
var DEFAULT_PRIMARY_RECOVERY_SUCCESSES = 2;
|
|
657
|
+
function sameTarget(a, b) {
|
|
658
|
+
return !!a && a.providerId === b.providerId && a.model === b.model;
|
|
659
|
+
}
|
|
660
|
+
function fallbackCandidates(config, current, opts = {}) {
|
|
661
|
+
const mgr = opts.sharedManager ?? new FallbackProfileManager(config);
|
|
662
|
+
return mgr.resolveCandidates(current, {
|
|
663
|
+
fallbackModels: opts.fallbackModels,
|
|
664
|
+
fallbackProfile: opts.fallbackProfile,
|
|
665
|
+
primary: opts.primary ?? primaryTarget(config),
|
|
666
|
+
closedWorld: opts.closedWorld
|
|
667
|
+
});
|
|
668
|
+
}
|
|
669
|
+
var primaryTarget = (cfg) => ({ providerId: cfg.provider, model: cfg.model });
|
|
670
|
+
function maxContextOf(provider) {
|
|
671
|
+
const max = provider.capabilities.maxContext;
|
|
672
|
+
return typeof max === "number" && Number.isFinite(max) ? max : 0;
|
|
673
|
+
}
|
|
674
|
+
function contextWindowWarning(currentProvider, nextProvider, currentTokens) {
|
|
675
|
+
const fromMaxContext = maxContextOf(currentProvider);
|
|
676
|
+
const toMaxContext = maxContextOf(nextProvider);
|
|
677
|
+
if (fromMaxContext <= 0 || toMaxContext <= 0 || toMaxContext >= fromMaxContext) return void 0;
|
|
678
|
+
return {
|
|
679
|
+
fromMaxContext,
|
|
680
|
+
toMaxContext,
|
|
681
|
+
...typeof currentTokens === "number" && currentTokens > 0 ? { currentTokens } : {}
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
function createFallbackModelExtension(deps) {
|
|
685
|
+
let dirty = false;
|
|
686
|
+
let primaryFailureStreak = 0;
|
|
687
|
+
let primaryRecoveryHits = 0;
|
|
688
|
+
let stickyTurnsElapsed = 0;
|
|
689
|
+
let lastWorkingFallback;
|
|
690
|
+
let blockedPrimary;
|
|
691
|
+
let primaryBlockedUntil = 0;
|
|
692
|
+
const now = () => deps.now?.() ?? Date.now();
|
|
693
|
+
const liveStickiness = () => deps.getConfig().fallbackStickiness;
|
|
694
|
+
const cooldownBase = () => Math.max(
|
|
695
|
+
0,
|
|
696
|
+
deps.primaryCooldownMs ?? liveStickiness()?.primaryProbeInterval ?? DEFAULT_PRIMARY_COOLDOWN_MS
|
|
697
|
+
);
|
|
698
|
+
const cooldownMax = () => Math.max(cooldownBase(), deps.primaryCooldownMaxMs ?? DEFAULT_PRIMARY_COOLDOWN_MAX_MS);
|
|
699
|
+
const selectedPrimary = (cfg) => deps.getPrimaryTarget?.() ?? primaryTarget(cfg);
|
|
700
|
+
const primaryInCooldown = (cfg) => sameTarget(blockedPrimary, selectedPrimary(cfg)) && now() < primaryBlockedUntil;
|
|
701
|
+
const markPrimaryFailure = (cfg) => {
|
|
702
|
+
const primary = selectedPrimary(cfg);
|
|
703
|
+
primaryFailureStreak = sameTarget(blockedPrimary, primary) ? primaryFailureStreak + 1 : 1;
|
|
704
|
+
primaryRecoveryHits = 0;
|
|
705
|
+
stickyTurnsElapsed = 0;
|
|
706
|
+
blockedPrimary = primary;
|
|
707
|
+
const base = cooldownBase();
|
|
708
|
+
if (base <= 0) {
|
|
709
|
+
primaryBlockedUntil = 0;
|
|
710
|
+
return;
|
|
711
|
+
}
|
|
712
|
+
const multiplier = 2 ** Math.max(0, primaryFailureStreak - 1);
|
|
713
|
+
primaryBlockedUntil = now() + Math.min(cooldownMax(), base * multiplier);
|
|
714
|
+
};
|
|
715
|
+
const recoveryTarget = () => Math.max(1, deps.primaryRecoverySuccesses ?? DEFAULT_PRIMARY_RECOVERY_SUCCESSES);
|
|
716
|
+
const stickyTarget = () => Math.max(0, deps.stickyFallbackTurns ?? liveStickiness()?.stickyFallbackTurns ?? 0);
|
|
717
|
+
const inStickyWindow = () => stickyTurnsElapsed < stickyTarget();
|
|
718
|
+
const onPrimarySuccess = (cfg) => {
|
|
719
|
+
if (!sameTarget(blockedPrimary, selectedPrimary(cfg))) return;
|
|
720
|
+
primaryRecoveryHits += 1;
|
|
721
|
+
if (primaryRecoveryHits >= recoveryTarget()) {
|
|
722
|
+
primaryFailureStreak = 0;
|
|
723
|
+
primaryRecoveryHits = 0;
|
|
724
|
+
stickyTurnsElapsed = 0;
|
|
725
|
+
lastWorkingFallback = void 0;
|
|
726
|
+
blockedPrimary = void 0;
|
|
727
|
+
primaryBlockedUntil = 0;
|
|
728
|
+
} else {
|
|
729
|
+
primaryBlockedUntil = 0;
|
|
730
|
+
}
|
|
731
|
+
};
|
|
732
|
+
return {
|
|
733
|
+
name: "fallback-model",
|
|
734
|
+
beforeRun: async (ctx) => {
|
|
735
|
+
if (!dirty) return;
|
|
736
|
+
const cfg = deps.getConfig();
|
|
737
|
+
const primary = selectedPrimary(cfg);
|
|
738
|
+
stickyTurnsElapsed += 1;
|
|
739
|
+
if (inStickyWindow()) return;
|
|
740
|
+
if (primaryInCooldown(cfg)) return;
|
|
741
|
+
if (!evaluateModelCalendar(cfg.modelAvailabilitySchedule, primary.providerId, primary.model).allowed || deps.statusTracker && !deps.statusTracker.isAvailable(primary.providerId, primary.model))
|
|
742
|
+
return;
|
|
743
|
+
try {
|
|
744
|
+
const primaryProvider = await deps.buildProvider(primary.providerId, primary.model);
|
|
745
|
+
const currentTokens = ctx.lastRequestTokens;
|
|
746
|
+
const maxContext = maxContextOf(primaryProvider);
|
|
747
|
+
if (maxContext > 0 && typeof currentTokens === "number" && currentTokens > maxContext) {
|
|
748
|
+
deps.events.emit("provider.primary_probe_context_blocked", {
|
|
749
|
+
sessionId: resolveEventSessionId(ctx),
|
|
750
|
+
providerId: primary.providerId,
|
|
751
|
+
model: primary.model,
|
|
752
|
+
currentTokens,
|
|
753
|
+
maxContext,
|
|
754
|
+
timestamp: now()
|
|
755
|
+
});
|
|
756
|
+
deps.logger?.warn(
|
|
757
|
+
`fallback-model: deferring primary probe "${primary.providerId}/${primary.model}" \u2014 context (${currentTokens}) exceeds target window (${maxContext})`
|
|
758
|
+
);
|
|
759
|
+
markPrimaryFailure(cfg);
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
ctx.provider = primaryProvider;
|
|
763
|
+
ctx.model = primary.model;
|
|
764
|
+
await deps.onModelSwitch?.(primary.providerId, primary.model);
|
|
765
|
+
primaryBlockedUntil = 0;
|
|
766
|
+
} catch (err) {
|
|
767
|
+
deps.logger?.warn(
|
|
768
|
+
`fallback-model: could not restore primary "${primary.providerId}/${primary.model}": ${err instanceof Error ? err.message : String(err)}`
|
|
769
|
+
);
|
|
770
|
+
markPrimaryFailure(cfg);
|
|
771
|
+
return;
|
|
772
|
+
}
|
|
773
|
+
dirty = false;
|
|
774
|
+
},
|
|
775
|
+
wrapProviderRunner: async (ctx, request, inner) => {
|
|
776
|
+
const tracker = deps.statusTracker;
|
|
777
|
+
const calendar = evaluateModelCalendar(
|
|
778
|
+
deps.getConfig().modelAvailabilitySchedule,
|
|
779
|
+
ctx.provider.id,
|
|
780
|
+
ctx.model
|
|
781
|
+
);
|
|
782
|
+
const trackerBlocked = tracker ? !tracker.isAvailable(ctx.provider.id, ctx.model) : false;
|
|
783
|
+
if (trackerBlocked || !calendar.allowed) {
|
|
784
|
+
deps.logger?.warn(
|
|
785
|
+
`provider-status: "${ctx.provider.id}/${ctx.model}" is blocked \u2014 trying fallback chain`
|
|
786
|
+
);
|
|
787
|
+
const status = tracker?.getStatus(ctx.provider.id, ctx.model);
|
|
788
|
+
const logical = tracker?.logicalIdentity(ctx.provider.id, ctx.model) ?? logicalCalendarTarget(ctx.provider.id, ctx.model);
|
|
789
|
+
deps.events.emit("provider.active_blocked", {
|
|
790
|
+
providerId: logical.providerId,
|
|
791
|
+
model: logical.model,
|
|
792
|
+
state: "blocked",
|
|
793
|
+
fallbackProviderId: "",
|
|
794
|
+
fallbackModel: "",
|
|
795
|
+
lastError: calendar.rule?.label ?? (calendar.rule ? "Blocked by model availability calendar" : void 0) ?? status?.lastErrorMessage ?? "Rate limit or repeated failures",
|
|
796
|
+
sessionId: resolveEventSessionId(ctx),
|
|
797
|
+
timestamp: Date.now()
|
|
798
|
+
});
|
|
799
|
+
const skipErr = new ProviderError(
|
|
800
|
+
`Skipping unavailable "${ctx.provider.id}/${ctx.model}" \u2014 try fallback`,
|
|
801
|
+
429,
|
|
802
|
+
true,
|
|
803
|
+
ctx.provider.id,
|
|
804
|
+
{ kind: "rate_limit" }
|
|
805
|
+
);
|
|
806
|
+
return runFallbackChain(ctx, request, inner, skipErr, true);
|
|
807
|
+
}
|
|
808
|
+
try {
|
|
809
|
+
const response = ensureUsableModelResponse(
|
|
810
|
+
await inner(ctx, request),
|
|
811
|
+
ctx.provider.id,
|
|
812
|
+
ctx.model
|
|
813
|
+
);
|
|
814
|
+
tracker?.recordSuccess(ctx.provider.id, ctx.model, {
|
|
815
|
+
sessionId: resolveEventSessionId(ctx),
|
|
816
|
+
agentId: ctx.agentId
|
|
817
|
+
});
|
|
818
|
+
const cfg = deps.getConfig();
|
|
819
|
+
const primary = selectedPrimary(cfg);
|
|
820
|
+
if (ctx.provider.id === primary.providerId && ctx.model === primary.model) {
|
|
821
|
+
onPrimarySuccess(cfg);
|
|
822
|
+
}
|
|
823
|
+
return response;
|
|
824
|
+
} catch (firstErr) {
|
|
825
|
+
return runFallbackChain(ctx, request, inner, firstErr);
|
|
826
|
+
}
|
|
827
|
+
async function runFallbackChain(ctx_, request_, inner_, firstErr_, alreadyTracked = false) {
|
|
828
|
+
if (ctx_.signal?.aborted) throw firstErr_;
|
|
829
|
+
let lastErr = firstErr_;
|
|
830
|
+
const cfg = deps.getConfig();
|
|
831
|
+
const current = { providerId: ctx_.provider.id, model: ctx_.model };
|
|
832
|
+
const firstErrIsProvider = firstErr_ instanceof ProviderError || ProviderError.isProviderError(firstErr_);
|
|
833
|
+
if (!alreadyTracked && firstErrIsProvider && tracker && !isProviderFailureTracked(firstErr_)) {
|
|
834
|
+
tracker.recordFailure(
|
|
835
|
+
ctx_.provider.id,
|
|
836
|
+
ctx_.model,
|
|
837
|
+
firstErr_.kind,
|
|
838
|
+
firstErr_.status,
|
|
839
|
+
firstErr_.describe(),
|
|
840
|
+
{
|
|
841
|
+
sessionId: resolveEventSessionId(ctx_),
|
|
842
|
+
agentId: ctx_.agentId,
|
|
843
|
+
retryAfterMs: firstErr_.body?.retryAfterMs
|
|
844
|
+
}
|
|
845
|
+
);
|
|
846
|
+
}
|
|
847
|
+
const chain = fallbackCandidates(cfg, current, {
|
|
848
|
+
fallbackModels: deps.getFallbackModels?.(),
|
|
849
|
+
fallbackProfile: deps.getFallbackProfile?.(),
|
|
850
|
+
sharedManager: deps.fallbackProfileManager,
|
|
851
|
+
primary: selectedPrimary(cfg),
|
|
852
|
+
closedWorld: deps.isClosedWorld?.() ?? false
|
|
853
|
+
});
|
|
854
|
+
let usableChain = tracker ? chain.filter((e) => tracker.isAvailable(e.providerId, e.model)) : chain;
|
|
855
|
+
usableChain = usableChain.filter(
|
|
856
|
+
(e) => evaluateModelCalendar(cfg.modelAvailabilitySchedule, e.providerId, e.model).allowed
|
|
857
|
+
);
|
|
858
|
+
if (lastWorkingFallback && usableChain.length > 1 && // Don't front-load if the last-working is the current model
|
|
859
|
+
// (we're already on it) or if it's now blocked.
|
|
860
|
+
!(lastWorkingFallback.providerId === current.providerId && lastWorkingFallback.model === current.model) && !(tracker && !tracker.isAvailable(lastWorkingFallback.providerId, lastWorkingFallback.model))) {
|
|
861
|
+
const lwfKey = `${lastWorkingFallback.providerId}/${lastWorkingFallback.model}`;
|
|
862
|
+
const lwfEntry = usableChain.find((e) => `${e.providerId}/${e.model}` === lwfKey);
|
|
863
|
+
if (lwfEntry) {
|
|
864
|
+
usableChain = [
|
|
865
|
+
lwfEntry,
|
|
866
|
+
...usableChain.filter((e) => `${e.providerId}/${e.model}` !== lwfKey)
|
|
867
|
+
];
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
if (!alreadyTracked && shouldFallback(firstErr_) !== null && sameTarget(selectedPrimary(cfg), current)) {
|
|
871
|
+
markPrimaryFailure(cfg);
|
|
872
|
+
}
|
|
873
|
+
const status = shouldFallback(firstErr_);
|
|
874
|
+
if (status === null) throw firstErr_;
|
|
875
|
+
let gateRequestId;
|
|
876
|
+
const configuredGateSeconds = cfg.fallbackGateSeconds ?? deps.fallbackGateSeconds;
|
|
877
|
+
if (configuredGateSeconds !== 0 && deps.fallbackGate && usableChain.length > 0) {
|
|
878
|
+
gateRequestId = randomUUID3();
|
|
879
|
+
const autoSwitchSeconds = Math.max(1, configuredGateSeconds ?? 7);
|
|
880
|
+
const gateCandidates = usableChain.map((e) => ({
|
|
881
|
+
providerId: e.providerId,
|
|
882
|
+
model: e.model
|
|
883
|
+
}));
|
|
884
|
+
try {
|
|
885
|
+
const choice = await deps.fallbackGate({
|
|
886
|
+
events: deps.events,
|
|
887
|
+
sessionId: resolveEventSessionId(ctx_),
|
|
888
|
+
from: {
|
|
889
|
+
providerId: ctx_.provider.id,
|
|
890
|
+
model: ctx_.model
|
|
891
|
+
},
|
|
892
|
+
status,
|
|
893
|
+
candidates: gateCandidates,
|
|
894
|
+
autoSwitchSeconds,
|
|
895
|
+
requestId: gateRequestId
|
|
896
|
+
});
|
|
897
|
+
if (choice) {
|
|
898
|
+
const chosen = usableChain.find(
|
|
899
|
+
(e) => e.providerId === choice.providerId && e.model === choice.model
|
|
900
|
+
);
|
|
901
|
+
if (chosen) {
|
|
902
|
+
usableChain = [
|
|
903
|
+
chosen,
|
|
904
|
+
...usableChain.filter(
|
|
905
|
+
(e) => !(e.providerId === choice.providerId && e.model === choice.model)
|
|
906
|
+
)
|
|
907
|
+
];
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
} catch (gateErr) {
|
|
911
|
+
deps.logger?.warn(
|
|
912
|
+
`fallback-model: gate error \u2014 proceeding with default chain: ${gateErr instanceof Error ? gateErr.message : String(gateErr)}`
|
|
913
|
+
);
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
if (ctx_.signal?.aborted) throw firstErr_;
|
|
917
|
+
for (const entry of usableChain) {
|
|
918
|
+
if (ctx_.signal?.aborted) throw lastErr;
|
|
919
|
+
if (!evaluateModelCalendar(cfg.modelAvailabilitySchedule, entry.providerId, entry.model).allowed)
|
|
920
|
+
continue;
|
|
921
|
+
if (tracker && !tracker.isAvailable(entry.providerId, entry.model)) {
|
|
922
|
+
deps.logger?.warn(
|
|
923
|
+
`provider-status: "${entry.providerId}/${entry.model}" entered the waiting room since the chain was computed \u2014 skipping`
|
|
924
|
+
);
|
|
925
|
+
continue;
|
|
926
|
+
}
|
|
927
|
+
const targetProviderId = entry.providerId;
|
|
928
|
+
const targetModel = entry.model;
|
|
929
|
+
if (targetProviderId === ctx_.provider.id && targetModel === ctx_.model) continue;
|
|
930
|
+
if (primaryInCooldown(cfg) && sameTarget(selectedPrimary(cfg), {
|
|
931
|
+
providerId: targetProviderId,
|
|
932
|
+
model: targetModel
|
|
933
|
+
})) {
|
|
934
|
+
continue;
|
|
935
|
+
}
|
|
936
|
+
const from = { providerId: ctx_.provider.id, model: ctx_.model };
|
|
937
|
+
const logicalFrom = tracker?.logicalIdentity(from.providerId, from.model) ?? from;
|
|
938
|
+
let nextProvider;
|
|
939
|
+
try {
|
|
940
|
+
nextProvider = await deps.buildProvider(targetProviderId, targetModel);
|
|
941
|
+
} catch (err) {
|
|
942
|
+
deps.logger?.warn(
|
|
943
|
+
`fallback-model: skipping "${targetProviderId}/${targetModel}" \u2014 cannot build provider "${targetProviderId}": ${err instanceof Error ? err.message : String(err)}`
|
|
944
|
+
);
|
|
945
|
+
continue;
|
|
946
|
+
}
|
|
947
|
+
const entryMaxContext = maxContextOf(nextProvider);
|
|
948
|
+
const currentTokens = ctx_.lastRequestTokens;
|
|
949
|
+
if (entryMaxContext > 0 && typeof currentTokens === "number" && currentTokens > 0 && currentTokens > entryMaxContext) {
|
|
950
|
+
deps.logger?.warn(
|
|
951
|
+
`fallback-model: skipping "${targetProviderId}/${targetModel}" \u2014 context window (${entryMaxContext}) is smaller than current request tokens (${currentTokens})`
|
|
952
|
+
);
|
|
953
|
+
continue;
|
|
954
|
+
}
|
|
955
|
+
const providerSwitched = nextProvider.id !== from.providerId;
|
|
956
|
+
const warning = contextWindowWarning(ctx_.provider, nextProvider, ctx_.lastRequestTokens);
|
|
957
|
+
ctx_.provider = nextProvider;
|
|
958
|
+
ctx_.model = targetModel;
|
|
959
|
+
request_.model = targetModel;
|
|
960
|
+
bindRequestProvider(request_, nextProvider);
|
|
961
|
+
dirty = true;
|
|
962
|
+
await deps.onModelSwitch?.(targetProviderId, targetModel);
|
|
963
|
+
deps.events.emit("provider.fallback", {
|
|
964
|
+
sessionId: resolveEventSessionId(ctx_),
|
|
965
|
+
from: logicalFrom,
|
|
966
|
+
to: tracker?.logicalIdentity(nextProvider.id, targetModel) ?? {
|
|
967
|
+
providerId: nextProvider.id,
|
|
968
|
+
model: targetModel
|
|
969
|
+
},
|
|
970
|
+
status,
|
|
971
|
+
providerSwitched,
|
|
972
|
+
// Correlate this completion with the gate that paused for the
|
|
973
|
+
// user's pick — clients clear the fallback modal only when the
|
|
974
|
+
// requestId matches the pending request.
|
|
975
|
+
...gateRequestId ? { requestId: gateRequestId } : {},
|
|
976
|
+
...warning ? { contextWindowWarning: warning } : {}
|
|
977
|
+
});
|
|
978
|
+
if (ctx_.signal?.aborted) throw lastErr;
|
|
979
|
+
try {
|
|
980
|
+
const response = ensureUsableModelResponse(
|
|
981
|
+
await inner_(ctx_, request_),
|
|
982
|
+
ctx_.provider.id,
|
|
983
|
+
ctx_.model
|
|
984
|
+
);
|
|
985
|
+
tracker?.recordSuccess(nextProvider.id, targetModel, {
|
|
986
|
+
sessionId: resolveEventSessionId(ctx_),
|
|
987
|
+
agentId: ctx_.agentId
|
|
988
|
+
});
|
|
989
|
+
lastWorkingFallback = { providerId: nextProvider.id, model: targetModel };
|
|
990
|
+
return response;
|
|
991
|
+
} catch (err) {
|
|
992
|
+
if ((err instanceof ProviderError || ProviderError.isProviderError(err)) && tracker && !isProviderFailureTracked(err)) {
|
|
993
|
+
tracker.recordFailure(
|
|
994
|
+
nextProvider.id,
|
|
995
|
+
targetModel,
|
|
996
|
+
err.kind,
|
|
997
|
+
err.status,
|
|
998
|
+
err.describe(),
|
|
999
|
+
{
|
|
1000
|
+
sessionId: resolveEventSessionId(ctx_),
|
|
1001
|
+
agentId: ctx_.agentId,
|
|
1002
|
+
retryAfterMs: err.body?.retryAfterMs
|
|
1003
|
+
}
|
|
1004
|
+
);
|
|
1005
|
+
}
|
|
1006
|
+
lastErr = err;
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
throw lastErr;
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
};
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
export {
|
|
1016
|
+
runProviderWithRetry,
|
|
1017
|
+
fallbackProfileChain,
|
|
1018
|
+
smartDefaultFallbackChain,
|
|
1019
|
+
effectiveFallbackChain,
|
|
1020
|
+
runtimeFallbackChain,
|
|
1021
|
+
createFallbackModelExtension
|
|
1022
|
+
};
|
|
1023
|
+
//# sourceMappingURL=chunk-H72H4XDU.js.map
|