@te-river/opencode-team-mode 1.7.10 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/README.zh-CN.md +5 -3
- package/dist/host/v2-events.d.ts +2 -0
- package/dist/host/v2-events.d.ts.map +1 -1
- package/dist/host/v2-events.js +25 -1
- package/dist/host/v2-events.js.map +1 -1
- package/dist/host/v2-retry.d.ts +115 -18
- package/dist/host/v2-retry.d.ts.map +1 -1
- package/dist/host/v2-retry.js +376 -57
- package/dist/host/v2-retry.js.map +1 -1
- package/dist/host/v2-scope.d.ts +2 -0
- package/dist/host/v2-scope.d.ts.map +1 -1
- package/dist/host/v2-scope.js +4 -0
- package/dist/host/v2-scope.js.map +1 -1
- package/dist/host/v2-types.d.ts +23 -0
- package/dist/host/v2-types.d.ts.map +1 -1
- package/dist/host/v2.d.ts.map +1 -1
- package/dist/host/v2.js +41 -4
- package/dist/host/v2.js.map +1 -1
- package/dist/tm/config-layers.d.ts.map +1 -1
- package/dist/tm/config-layers.js +2 -0
- package/dist/tm/config-layers.js.map +1 -1
- package/dist/tm/config.d.ts +2 -0
- package/dist/tm/config.d.ts.map +1 -1
- package/dist/tm/config.js.map +1 -1
- package/dist/tm/index.d.ts +8 -0
- package/dist/tm/index.d.ts.map +1 -1
- package/dist/tm/index.js +1 -0
- package/dist/tm/index.js.map +1 -1
- package/docs/installation-v2.md +11 -6
- package/docs/v2-port-notes.md +46 -6
- package/package.json +1 -1
- package/scripts/gen-v2-config.mjs +1 -1
package/dist/host/v2-retry.js
CHANGED
|
@@ -3,15 +3,20 @@
|
|
|
3
3
|
* throttles us".
|
|
4
4
|
*
|
|
5
5
|
* THE HONEST BOUNDARY, stated first because everything below is shaped by it:
|
|
6
|
-
* a plugin has NO seam that
|
|
7
|
-
* request the model decides to send, and we must not pretend to.
|
|
8
|
-
* does
|
|
6
|
+
* a plugin has NO seam that INTERCEPTS the model's own retry — we cannot delay a
|
|
7
|
+
* request the model decides to send, and we must not pretend to. But the method
|
|
8
|
+
* surface does let a plugin RE-DRIVE execution: `ctx.session.synthetic` is callable
|
|
9
|
+
* on 2.1.2 (zero-token probe 2026-10-21, docs/v2-port-notes.md) and the host
|
|
10
|
+
* documents it as "Durably admit synthetic session input and schedule execution
|
|
11
|
+
* unless resume is false". So this layer does four things, and says so:
|
|
9
12
|
*
|
|
10
13
|
* 1. RECOGNISE a provider throttle from the event feed (`session.error`), by
|
|
11
14
|
* matching explicit signatures only — never by guessing. A text that
|
|
12
15
|
* not carry a known signature classifies as `unknown` and changes nothing.
|
|
13
|
-
* 2. COMPUTE an exact wait (
|
|
14
|
-
*
|
|
16
|
+
* 2. COMPUTE an exact wait (a per-request ladder: retries #1..#5 wait
|
|
17
|
+
* 5s/10s/20s/40s/60s; the chain never accumulates across requests, and the
|
|
18
|
+
* host's own `session.retry.scheduled` stands our directive down — the desktop
|
|
19
|
+
* is already rendering that countdown; jitter applies only when retryJitter > 0) — but ONLY for the
|
|
15
20
|
* kinds a wait can fix. A HARD quota (allocated-exceeded / increase-your-limit /
|
|
16
21
|
* token-limit / 余额不足) names no wait: the directive says stop and raise-or-reset,
|
|
17
22
|
* and the breaker trips on the SECOND consecutive hard-quota error instead of
|
|
@@ -24,6 +29,13 @@
|
|
|
24
29
|
* trips and new `subagent` dispatches are DENIED for `cooldownMs`, on the
|
|
25
30
|
* same `permission.evaluate` seam the permission guards use. Both only ever
|
|
26
31
|
* make a rule stricter, so they compose without fighting.
|
|
32
|
+
* 4. RE-DRIVE: when a rung's wait is over and nobody moved — the host scheduled no
|
|
33
|
+
* retry of its own and the session assembled no request — the layer admits a
|
|
34
|
+
* synthetic continuation for that session (`session.synthetic`, resume:true).
|
|
35
|
+
* One re-drive per rung, capped by `maxRetries` per chain, NEVER for a session
|
|
36
|
+
* the scope has not seen, and cancelled the moment the session or the host
|
|
37
|
+
* continues first. A hard quota is re-driven only when the provider stated a
|
|
38
|
+
* retry time — otherwise waiting is the one thing that cannot help.
|
|
27
39
|
*
|
|
28
40
|
* Discipline (the same as every other v2 layer): Team-scoped, never throws (a
|
|
29
41
|
* failure allows the call through and is counted), never rewrites args, and
|
|
@@ -71,11 +83,15 @@ function parseRetryAfter(s) {
|
|
|
71
83
|
return Number.isFinite(n) && n >= 0 ? n * 1000 : 0;
|
|
72
84
|
}
|
|
73
85
|
/**
|
|
74
|
-
* The default ladder is FIXED
|
|
75
|
-
* wait
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
86
|
+
* The default ladder is FIXED, deterministic AND PER REQUEST: retries #1..#5 of the
|
|
87
|
+
* SAME failed request wait 5s / 10s / 20s / 40s / 60s (base 5000 doubling, jitter 0),
|
|
88
|
+
* then that request's budget is spent (`maxRetries` 5) and the chain stops. A chain
|
|
89
|
+
* never accumulates across requests (idle / gap / post-exhaustion all restart it), and
|
|
90
|
+
* the host's own `session.retry.scheduled` stands our ladder directive down because the
|
|
91
|
+
* desktop is already showing that countdown. Jitter stays a knob — a user who would
|
|
92
|
+
* rather parallel sessions not re-fire in lockstep can set `retryJitter` back to a
|
|
93
|
+
* fraction — but the shipped default promises the fixed ladder, and a promise with
|
|
94
|
+
* jitter in it is not that promise.
|
|
79
95
|
*/
|
|
80
96
|
export const RETRY_DEFAULTS = {
|
|
81
97
|
enabled: true,
|
|
@@ -84,6 +100,8 @@ export const RETRY_DEFAULTS = {
|
|
|
84
100
|
jitter: 0,
|
|
85
101
|
breakAfter: 5,
|
|
86
102
|
cooldownMs: 60000,
|
|
103
|
+
maxRetries: 5,
|
|
104
|
+
chainGapMs: 600_000,
|
|
87
105
|
};
|
|
88
106
|
const isOff = (raw) => /^(off|false|0|no)$/i.test(String(raw ?? "").trim());
|
|
89
107
|
/**
|
|
@@ -99,6 +117,8 @@ export function resolveRetryConfig(config = resolveConfig()) {
|
|
|
99
117
|
jitter: config.retryJitter,
|
|
100
118
|
breakAfter: config.retryBreakAfter,
|
|
101
119
|
cooldownMs: config.retryCooldownMs,
|
|
120
|
+
maxRetries: config.retryMaxRetries,
|
|
121
|
+
chainGapMs: config.retryChainGapMs,
|
|
102
122
|
};
|
|
103
123
|
}
|
|
104
124
|
/**
|
|
@@ -112,6 +132,23 @@ export function backoffMs(attempt, baseMs, maxMs, jitter, random) {
|
|
|
112
132
|
const factor = 1 + (random() * 2 - 1) * j;
|
|
113
133
|
return Math.max(0, Math.round(raw * factor));
|
|
114
134
|
}
|
|
135
|
+
/** The turn-ending event spellings across host generations (2.0.28 renamed them). */
|
|
136
|
+
export function isTurnEndEvent(type) {
|
|
137
|
+
return type === "session.idle" || type === "session.execution.succeeded" || type === "session.execution.interrupted";
|
|
138
|
+
}
|
|
139
|
+
/** The session→role question, pure so a test can pin it without the host. */
|
|
140
|
+
export function roleOfSession(sid, children, agentOf, leadAgent = "team") {
|
|
141
|
+
const kid = children.find((c) => c.sessionID === sid);
|
|
142
|
+
if (kid)
|
|
143
|
+
return { role: "sub", parentSessionID: kid.parentSessionID, agent: kid.agent };
|
|
144
|
+
const agent = agentOf(sid);
|
|
145
|
+
// A Team specialist that has not reached the child registry yet (registration lands
|
|
146
|
+
// in the dispatch tool's execute.after, so there is a window) is still a SUB-agent:
|
|
147
|
+
// scope.agentOf only ever stores Team names, and only the lead is "team".
|
|
148
|
+
if (agent && agent !== leadAgent)
|
|
149
|
+
return { role: "sub", agent };
|
|
150
|
+
return { role: agent === leadAgent ? "main" : "unknown", agent };
|
|
151
|
+
}
|
|
115
152
|
const isThrottle = (kind) => kind === "quota" || kind === "rate";
|
|
116
153
|
const kindLabel = (kind) => kind === "quota" ? "配额" : kind === "rate" ? "限流" : kind === "transient" ? "临时" : "未知";
|
|
117
154
|
/**
|
|
@@ -143,30 +180,85 @@ export function createRetryGovernor(opts = {}) {
|
|
|
143
180
|
cooldownDenied: 0,
|
|
144
181
|
foreignSkipped: 0,
|
|
145
182
|
threw: 0,
|
|
183
|
+
childExhausted: 0,
|
|
184
|
+
mainExhausted: 0,
|
|
185
|
+
chainsReset: 0,
|
|
186
|
+
leadNotices: 0,
|
|
187
|
+
hostRetries: 0,
|
|
188
|
+
pluginRetries: 0,
|
|
189
|
+
};
|
|
190
|
+
const ANON = "~anon";
|
|
191
|
+
const MAX_CHAINS = 512;
|
|
192
|
+
const chains = new Map();
|
|
193
|
+
const pending = new Map();
|
|
194
|
+
/** The exact ladder this config promises, for the messages ("5s/10s/20s/40s/60s"). */
|
|
195
|
+
const ladderLabel = () => {
|
|
196
|
+
const out = [];
|
|
197
|
+
for (let i = 0; i < cfg.maxRetries; i++) {
|
|
198
|
+
out.push(`${Math.round(Math.min(cfg.baseMs * Math.pow(2, i), cfg.maxMs) / 1000)}s`);
|
|
199
|
+
}
|
|
200
|
+
return out.join("/");
|
|
201
|
+
};
|
|
202
|
+
const chainOf = (sid) => {
|
|
203
|
+
let c = chains.get(sid);
|
|
204
|
+
if (c)
|
|
205
|
+
return c;
|
|
206
|
+
if (chains.size >= MAX_CHAINS) {
|
|
207
|
+
// The map is keyed by session id; an unbounded map in a long-lived host is a
|
|
208
|
+
// leak. Evict the stalest session — its ladder state is worthless anyway.
|
|
209
|
+
let oldest = "";
|
|
210
|
+
let at = Infinity;
|
|
211
|
+
for (const [k, v] of chains) {
|
|
212
|
+
if (v.lastAt < at) {
|
|
213
|
+
at = v.lastAt;
|
|
214
|
+
oldest = k;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
if (oldest) {
|
|
218
|
+
chains.delete(oldest);
|
|
219
|
+
pending.delete(oldest);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
c = { attempts: 0, lastAt: 0, streak: 0, hardStreak: 0, cooldownUntil: 0, role: "unknown", done: false, redrives: 0 };
|
|
223
|
+
chains.set(sid, c);
|
|
224
|
+
return c;
|
|
146
225
|
};
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
let pending = null;
|
|
151
|
-
const inCooldown = () => now() < cooldownUntil;
|
|
152
|
-
const buildDirective = (cls, delayMs) => {
|
|
226
|
+
const inCooldown = (sessionID) => now() < chainOf(sidOf(sessionID)).cooldownUntil;
|
|
227
|
+
const cooldownRemainingMs = (sessionID) => Math.max(0, chainOf(sidOf(sessionID)).cooldownUntil - now());
|
|
228
|
+
const buildDirective = (cls, delayMs, attempt) => {
|
|
153
229
|
if (cls.hardQuota) {
|
|
154
230
|
const hint = cls.retryAfterMs > 0 ? `(错误文本给出的重试时间:约 ${Math.round(cls.retryAfterMs / 1000)} 秒)` : "";
|
|
155
231
|
return {
|
|
156
232
|
text: `${RETRY_MARKER}(配额耗尽,不是故障)\n检测到 provider 配额耗尽(hard quota):等待与重试都无法解决——请停止向该 provider 重发${hint},等其配额重置,或请用户提升配额/更换模型。需要完成的工作请主代理自己执行(再派子代理只会被同样的配额打回)。这是配额保护,不是插件故障;可用 retry: "off" 关闭。`,
|
|
157
233
|
delayMs: 0,
|
|
234
|
+
kind: "hard",
|
|
158
235
|
};
|
|
159
236
|
}
|
|
160
|
-
return buildLadderDirective(cls.kind, delayMs);
|
|
237
|
+
return buildLadderDirective(cls.kind, delayMs, attempt);
|
|
238
|
+
};
|
|
239
|
+
const buildExhaustedDirective = (cls, role) => {
|
|
240
|
+
const text = role === "sub"
|
|
241
|
+
? `${RETRY_MARKER}(重试预算耗尽)\n本请求已重试 ${cfg.maxRetries} 次(${ladderLabel()} 阶梯)仍失败(${kindLabel(cls.kind)}):停止重试,把失败原因交回主代理——由主代理接手该步骤,或换模型/换来源。这是配额保护,不是插件故障;可用 retry: "off" 关闭。`
|
|
242
|
+
: `${RETRY_MARKER}(重试预算耗尽)\n同一请求已重试 ${cfg.maxRetries} 次(${ladderLabel()} 阶梯)仍失败(${kindLabel(cls.kind)}):停止重试——向用户如实报告失败原因与已等待时间,并给出选项(换模型 / 补充配额 / 稍后再试)。这是配额保护,不是插件故障;可用 retry: "off" 关闭。`;
|
|
243
|
+
return { text, delayMs: 0, kind: "exhausted" };
|
|
244
|
+
};
|
|
245
|
+
const buildLeadNotice = (cls, sid, agent) => {
|
|
246
|
+
const who = String(agent ?? "").trim() || "子代理";
|
|
247
|
+
return {
|
|
248
|
+
text: `${RETRY_MARKER}(子代理被配额打回)\n子代理 ${who}(${sid.slice(-6)})同一请求已重试 ${cfg.maxRetries} 次仍失败(${kindLabel(cls.kind)}):不要再派发该步骤——请主代理自己执行,或换模型/等配额恢复。这是配额保护,不是插件故障;可用 retry: "off" 关闭。`,
|
|
249
|
+
delayMs: 0,
|
|
250
|
+
kind: "notice",
|
|
251
|
+
};
|
|
161
252
|
};
|
|
162
|
-
const buildLadderDirective = (kind, delayMs) => {
|
|
253
|
+
const buildLadderDirective = (kind, delayMs, attempt) => {
|
|
163
254
|
const secs = Math.max(1, Math.round(delayMs / 1000));
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
return { text, delayMs };
|
|
255
|
+
// Native wording (the desktop renders `尝试 N - 在 X 秒内重试` from the host's own
|
|
256
|
+
// retry event). We reuse the shape so the model reads one consistent cadence.
|
|
257
|
+
const text = `${RETRY_MARKER}(配额保护,不是故障)\n检测到 provider ${kindLabel(kind)} 错误:尝试 ${attempt} - 在 ${secs} 秒内重试。这是配额保护,不是插件故障;可用 retry: "off" 关闭。`;
|
|
258
|
+
return { text, delayMs, kind: "ladder" };
|
|
168
259
|
};
|
|
169
|
-
const observe = (text,
|
|
260
|
+
const observe = (text, rawOpts) => {
|
|
261
|
+
const o = typeof rawOpts === "number" ? { structuredRetryAfterMs: rawOpts } : (rawOpts ?? {});
|
|
170
262
|
const cls = classifyProviderError(text);
|
|
171
263
|
if (cls.kind === "unknown") {
|
|
172
264
|
report.unknownSeen++;
|
|
@@ -178,52 +270,185 @@ export function createRetryGovernor(opts = {}) {
|
|
|
178
270
|
report.rateSeen++;
|
|
179
271
|
else
|
|
180
272
|
report.transientSeen++;
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
273
|
+
// With `retry: "off"` the layer registers no hook, so arming directives would only
|
|
274
|
+
// grow the map nobody reads.
|
|
275
|
+
if (!cfg.enabled)
|
|
276
|
+
return { ...cls, delayMs: 0 };
|
|
277
|
+
const sid = sidOf(o.sessionID);
|
|
278
|
+
const chain = chainOf(sid);
|
|
279
|
+
// A chain that already spent its budget is finished: a further error starts a fresh
|
|
280
|
+
// chain (back at 5s). Otherwise a long quiet gap starts one. One reset, one count.
|
|
281
|
+
if (chain.done) {
|
|
282
|
+
chain.attempts = 0;
|
|
283
|
+
chain.done = false;
|
|
284
|
+
chain.redrives = 0;
|
|
285
|
+
report.chainsReset++;
|
|
286
|
+
emit({ event: "chain_reset", session: sid, why: "post-exhausted" });
|
|
287
|
+
}
|
|
288
|
+
else if (chain.lastAt > 0 && now() - chain.lastAt > cfg.chainGapMs) {
|
|
289
|
+
chain.attempts = 0;
|
|
290
|
+
chain.redrives = 0;
|
|
291
|
+
report.chainsReset++;
|
|
292
|
+
emit({ event: "chain_reset", session: sid, why: "gap" });
|
|
293
|
+
}
|
|
294
|
+
if (o.role && o.role !== "unknown")
|
|
295
|
+
chain.role = o.role;
|
|
296
|
+
chain.lastAt = now();
|
|
297
|
+
chain.streak++;
|
|
298
|
+
chain.hardStreak = cls.hardQuota ? chain.hardStreak + 1 : 0;
|
|
299
|
+
const structured = Number.isFinite(o.structuredRetryAfterMs) && o.structuredRetryAfterMs > 0 ? o.structuredRetryAfterMs : 0;
|
|
184
300
|
const retryAfterSource = structured > 0 ? "structured" : cls.retryAfterMs > 0 ? "text" : "none";
|
|
185
|
-
//
|
|
186
|
-
const
|
|
301
|
+
// This error is the one PAST the ladder when the request already spent all its retries.
|
|
302
|
+
const exhausted = !cls.hardQuota && chain.attempts >= cfg.maxRetries;
|
|
303
|
+
// The ladder is indexed by THIS request's cumulative retries: #1 → 5s, #2 → 10s, #3 → 20s.
|
|
304
|
+
const delayMs = cls.hardQuota || exhausted
|
|
305
|
+
? 0
|
|
306
|
+
: Math.max(backoffMs(chain.attempts, cfg.baseMs, cfg.maxMs, cfg.jitter, random), structured, cls.retryAfterMs);
|
|
307
|
+
if (!cls.hardQuota && !exhausted)
|
|
308
|
+
chain.attempts++;
|
|
187
309
|
report.lastDelayMs = delayMs;
|
|
188
|
-
emit({
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
310
|
+
emit({
|
|
311
|
+
event: "classified",
|
|
312
|
+
kind: cls.kind,
|
|
313
|
+
hard_quota: cls.hardQuota,
|
|
314
|
+
delay_ms: delayMs,
|
|
315
|
+
attempt: chain.attempts,
|
|
316
|
+
exhausted,
|
|
317
|
+
session: sid,
|
|
318
|
+
role: chain.role,
|
|
319
|
+
retry_after_source: retryAfterSource,
|
|
320
|
+
});
|
|
321
|
+
// The breaker is the SESSION's provider-health signal (independent of the
|
|
322
|
+
// per-request budget): consecutive throttle errors cool that session's dispatches.
|
|
323
|
+
// A hard quota is different work — two in a row means the provider is out of
|
|
324
|
+
// allocation, so waiting for breakAfter only burns tokens on a dead endpoint.
|
|
325
|
+
if (isThrottle(cls.kind) && (cls.hardQuota ? chain.hardStreak >= 2 : chain.streak >= cfg.breakAfter)) {
|
|
326
|
+
const wasIn = now() < chain.cooldownUntil;
|
|
327
|
+
chain.cooldownUntil = now() + cfg.cooldownMs;
|
|
195
328
|
report.cooldowns++;
|
|
196
329
|
if (!wasIn)
|
|
197
330
|
report.breakTrips++;
|
|
198
|
-
streak = 0;
|
|
199
|
-
hardStreak = 0;
|
|
200
|
-
emit({ event: "cooldown", break_trips: report.breakTrips, cooldowns: report.cooldowns, cooldown_ms: cfg.cooldownMs });
|
|
331
|
+
chain.streak = 0;
|
|
332
|
+
chain.hardStreak = 0;
|
|
333
|
+
emit({ event: "cooldown", break_trips: report.breakTrips, cooldowns: report.cooldowns, cooldown_ms: cfg.cooldownMs, session: sid });
|
|
334
|
+
}
|
|
335
|
+
if (exhausted) {
|
|
336
|
+
chain.done = true;
|
|
337
|
+
if (chain.role === "sub") {
|
|
338
|
+
report.childExhausted++;
|
|
339
|
+
pending.set(sid, buildExhaustedDirective(cls, "sub"));
|
|
340
|
+
const target = sidOf(o.parentSessionID);
|
|
341
|
+
if (target !== ANON && !pending.has(target)) {
|
|
342
|
+
pending.set(target, buildLeadNotice(cls, sid, o.agent));
|
|
343
|
+
report.leadNotices++;
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
else {
|
|
347
|
+
report.mainExhausted++;
|
|
348
|
+
pending.set(sid, buildExhaustedDirective(cls, chain.role));
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
else {
|
|
352
|
+
pending.set(sid, buildDirective(cls, delayMs, chain.attempts > 0 ? chain.attempts : 1));
|
|
353
|
+
if (cls.hardQuota && chain.role === "sub") {
|
|
354
|
+
const target = sidOf(o.parentSessionID);
|
|
355
|
+
if (target !== ANON && !pending.has(target)) {
|
|
356
|
+
pending.set(target, buildLeadNotice(cls, sid, o.agent));
|
|
357
|
+
report.leadNotices++;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
// ③ PLUGIN-DRIVEN RE-DRIVE: the ladder's wait is not just advice — the layer
|
|
362
|
+
// schedules a synthetic continuation for this session (ctx.session.synthetic,
|
|
363
|
+
// verified callable on 2.1.2). Hard quota re-drives ONLY when the provider
|
|
364
|
+
// stated a retry time; otherwise waiting is futile and the stop directive is
|
|
365
|
+
// all we can honestly say. Budget: one re-drive per rung of the chain.
|
|
366
|
+
if (opts.redrive && cfg.enabled && sid !== ANON) {
|
|
367
|
+
const floor = Math.max(structured, cls.retryAfterMs);
|
|
368
|
+
if (!cls.hardQuota && !exhausted && delayMs > 0 && chain.redrives < cfg.maxRetries) {
|
|
369
|
+
chain.redrives++;
|
|
370
|
+
opts.redrive({ sessionID: sid, delayMs, attempt: chain.redrives, kind: cls.kind, hardQuota: false });
|
|
371
|
+
}
|
|
372
|
+
else if (cls.hardQuota && floor > 0 && chain.redrives < cfg.maxRetries) {
|
|
373
|
+
chain.redrives++;
|
|
374
|
+
opts.redrive({ sessionID: sid, delayMs: floor, attempt: chain.redrives, kind: cls.kind, hardQuota: true });
|
|
375
|
+
}
|
|
201
376
|
}
|
|
202
|
-
pending = buildDirective(cls, delayMs);
|
|
203
377
|
return { ...cls, delayMs };
|
|
204
378
|
};
|
|
205
379
|
return {
|
|
206
380
|
observe,
|
|
207
381
|
observeEvent(raw) {
|
|
208
382
|
const text = errorTextOf(raw);
|
|
209
|
-
if (text)
|
|
210
|
-
|
|
383
|
+
if (!text)
|
|
384
|
+
return;
|
|
385
|
+
const sid = sessionIDOf(raw);
|
|
386
|
+
const info = sid && opts.roleOf ? opts.roleOf(sid) : undefined;
|
|
387
|
+
observe(text, {
|
|
388
|
+
sessionID: sid,
|
|
389
|
+
structuredRetryAfterMs: structuredRetryAfterOf(raw),
|
|
390
|
+
role: info?.role,
|
|
391
|
+
parentSessionID: info?.parentSessionID,
|
|
392
|
+
agent: info?.agent,
|
|
393
|
+
});
|
|
394
|
+
},
|
|
395
|
+
observeHostRetry(raw) {
|
|
396
|
+
if (!raw || typeof raw !== "object")
|
|
397
|
+
return;
|
|
398
|
+
const o = raw;
|
|
399
|
+
const props = (o.properties ?? o.payload ?? o.data);
|
|
400
|
+
const p = props && typeof props === "object" ? props : o;
|
|
401
|
+
const sid = sidOf(sessionIDOf(raw) || (typeof p.sessionID === "string" ? p.sessionID : ""));
|
|
402
|
+
const attempt = typeof p.attempt === "number" && Number.isFinite(p.attempt) ? Math.max(0, Math.floor(p.attempt)) : 0;
|
|
403
|
+
const at = typeof p.at === "number" && Number.isFinite(p.at) ? p.at : 0;
|
|
404
|
+
const err = p.error;
|
|
405
|
+
const retryAfterMs = err && typeof err === "object" && typeof err.retryAfterMs === "number" && err.retryAfterMs > 0 ? err.retryAfterMs : 0;
|
|
406
|
+
const chain = chainOf(sid);
|
|
407
|
+
// The host is already retrying this request: align our bookkeeping with its
|
|
408
|
+
// attempt count and stand the ladder directive down. A second “wait N
|
|
409
|
+
// seconds” from us would contradict the countdown the UI is showing; the
|
|
410
|
+
// exhausted/hard directives stay — those are the cases the host does NOT retry.
|
|
411
|
+
chain.attempts = Math.max(chain.attempts, attempt);
|
|
412
|
+
chain.lastAt = now();
|
|
413
|
+
const d = pending.get(sid);
|
|
414
|
+
if (d && d.kind === "ladder")
|
|
415
|
+
pending.delete(sid);
|
|
416
|
+
report.hostRetries++;
|
|
417
|
+
emit({ event: "host_retry", session: sid, attempt, at, retry_after_ms: retryAfterMs });
|
|
418
|
+
},
|
|
419
|
+
observeIdle(sessionID) {
|
|
420
|
+
const sid = sidOf(sessionID);
|
|
421
|
+
const c = chains.get(sid);
|
|
422
|
+
if (!c)
|
|
423
|
+
return;
|
|
424
|
+
// The turn ended: the REQUEST's ladder starts over, but the session-level
|
|
425
|
+
// breaker state (streak / hardStreak / cooldown) is a provider-health signal
|
|
426
|
+
// and must survive — a hard quota in one turn and another in the next are
|
|
427
|
+
// still two consecutive hits on the same provider.
|
|
428
|
+
const meaningful = c.attempts > 0 || c.done;
|
|
429
|
+
c.attempts = 0;
|
|
430
|
+
c.done = false;
|
|
431
|
+
if (meaningful) {
|
|
432
|
+
report.chainsReset++;
|
|
433
|
+
emit({ event: "chain_reset", session: sid, why: "idle" });
|
|
434
|
+
}
|
|
211
435
|
},
|
|
212
|
-
directive: () => pending,
|
|
213
|
-
takeDirective() {
|
|
214
|
-
const
|
|
215
|
-
|
|
436
|
+
directive: (sessionID) => pending.get(sidOf(sessionID)) ?? null,
|
|
437
|
+
takeDirective(sessionID) {
|
|
438
|
+
const sid = sidOf(sessionID);
|
|
439
|
+
const d = pending.get(sid) ?? null;
|
|
440
|
+
if (d)
|
|
441
|
+
pending.delete(sid);
|
|
216
442
|
return d;
|
|
217
443
|
},
|
|
218
444
|
inCooldown,
|
|
219
|
-
cooldownRemainingMs
|
|
445
|
+
cooldownRemainingMs,
|
|
220
446
|
breakAfter: cfg.breakAfter,
|
|
447
|
+
maxRetries: cfg.maxRetries,
|
|
221
448
|
report,
|
|
222
449
|
reset() {
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
cooldownUntil = 0;
|
|
226
|
-
pending = null;
|
|
450
|
+
chains.clear();
|
|
451
|
+
pending.clear();
|
|
227
452
|
},
|
|
228
453
|
};
|
|
229
454
|
}
|
|
@@ -301,6 +526,19 @@ export function structuredRetryAfterOf(raw) {
|
|
|
301
526
|
}
|
|
302
527
|
return 0;
|
|
303
528
|
}
|
|
529
|
+
/** The session id an event refers to — defensive, the payload shape is not promised. */
|
|
530
|
+
export function sessionIDOf(raw) {
|
|
531
|
+
if (!raw || typeof raw !== "object")
|
|
532
|
+
return "";
|
|
533
|
+
const o = raw;
|
|
534
|
+
if (typeof o.sessionID === "string" && o.sessionID)
|
|
535
|
+
return o.sessionID;
|
|
536
|
+
const props = (o.properties ?? o.payload ?? o.data);
|
|
537
|
+
if (props && typeof props === "object" && typeof props.sessionID === "string")
|
|
538
|
+
return props.sessionID;
|
|
539
|
+
return "";
|
|
540
|
+
}
|
|
541
|
+
const sidOf = (v) => (typeof v === "string" && v.trim() ? v.trim() : "~anon");
|
|
304
542
|
const rank = { allow: 0, ask: 1, deny: 2 };
|
|
305
543
|
/**
|
|
306
544
|
* Register the two hooks the retry layer owns:
|
|
@@ -316,22 +554,102 @@ export async function applyV2RetryGovernor(ctx, input) {
|
|
|
316
554
|
const { scope, governor } = input;
|
|
317
555
|
const report = governor.report;
|
|
318
556
|
const registrations = [];
|
|
557
|
+
const timers = input.timers ?? {
|
|
558
|
+
set: (cb, ms) => setTimeout(cb, ms),
|
|
559
|
+
clear: (h) => clearTimeout(h),
|
|
560
|
+
};
|
|
561
|
+
// ③ The re-drive machinery lives HERE because this layer owns the session domain,
|
|
562
|
+
// the scope and the trajectory sink; the governor only decides WHEN (with budget).
|
|
563
|
+
const pendingRedrive = new Map();
|
|
564
|
+
const emit = (row) => {
|
|
565
|
+
try {
|
|
566
|
+
input.onEvent?.(row);
|
|
567
|
+
}
|
|
568
|
+
catch {
|
|
569
|
+
/* the trajectory line is an extra, never a reason to fail the call */
|
|
570
|
+
}
|
|
571
|
+
};
|
|
572
|
+
const scheduleRedrive = (req) => {
|
|
573
|
+
// Only sessions this plugin has actually seen as Team (lead or a specialist).
|
|
574
|
+
// An unknown id is never re-driven — a throughput helper may not push input
|
|
575
|
+
// into somebody else's conversation.
|
|
576
|
+
if (!scope || !scope.agentOf(req.sessionID))
|
|
577
|
+
return;
|
|
578
|
+
const prev = pendingRedrive.get(req.sessionID);
|
|
579
|
+
if (prev !== undefined)
|
|
580
|
+
timers.clear(prev);
|
|
581
|
+
const secs = Math.max(1, Math.round(req.delayMs / 1000));
|
|
582
|
+
const text = `[Team 自动重试 ${req.attempt}] 上一次请求因 provider ${kindLabel(req.kind)} 失败` +
|
|
583
|
+
(req.hardQuota ? "(配额,已按服务端给出的重试时间等待)" : "") +
|
|
584
|
+
`,已等待 ${secs} 秒。请继续该会话未完成的工作;若再次失败,按 Team Retry 指令处理。`;
|
|
585
|
+
const handle = timers.set(() => {
|
|
586
|
+
pendingRedrive.delete(req.sessionID);
|
|
587
|
+
void (async () => {
|
|
588
|
+
const synthetic = ctx.session?.synthetic;
|
|
589
|
+
if (typeof synthetic !== "function") {
|
|
590
|
+
emit({ event: "plugin_retry_failed", session: req.sessionID, attempt: req.attempt, reason: "no seam: ctx.session.synthetic missing" });
|
|
591
|
+
return;
|
|
592
|
+
}
|
|
593
|
+
try {
|
|
594
|
+
await synthetic.call(ctx.session, { sessionID: req.sessionID, text, description: "Team retry", resume: true });
|
|
595
|
+
report.pluginRetries++;
|
|
596
|
+
emit({ event: "plugin_retry_admitted", session: req.sessionID, attempt: req.attempt, delay_ms: req.delayMs, kind: req.kind });
|
|
597
|
+
}
|
|
598
|
+
catch (err) {
|
|
599
|
+
emit({ event: "plugin_retry_failed", session: req.sessionID, attempt: req.attempt, reason: String(err?.message ?? err).slice(0, 140) });
|
|
600
|
+
}
|
|
601
|
+
})();
|
|
602
|
+
}, req.delayMs);
|
|
603
|
+
pendingRedrive.set(req.sessionID, handle);
|
|
604
|
+
emit({ event: "plugin_retry_scheduled", session: req.sessionID, attempt: req.attempt, delay_ms: req.delayMs, kind: req.kind, hard_quota: req.hardQuota });
|
|
605
|
+
};
|
|
606
|
+
const cancelRedrive = (sessionID) => {
|
|
607
|
+
const sid = typeof sessionID === "string" && sessionID.trim() ? sessionID.trim() : "";
|
|
608
|
+
if (!sid)
|
|
609
|
+
return;
|
|
610
|
+
const h = pendingRedrive.get(sid);
|
|
611
|
+
if (h === undefined)
|
|
612
|
+
return;
|
|
613
|
+
timers.clear(h);
|
|
614
|
+
pendingRedrive.delete(sid);
|
|
615
|
+
emit({ event: "plugin_retry_cancelled", session: sid });
|
|
616
|
+
};
|
|
617
|
+
input.bindRedrive?.(scheduleRedrive);
|
|
618
|
+
registrations.push({
|
|
619
|
+
dispose: async () => {
|
|
620
|
+
for (const h of pendingRedrive.values()) {
|
|
621
|
+
try {
|
|
622
|
+
timers.clear(h);
|
|
623
|
+
}
|
|
624
|
+
catch {
|
|
625
|
+
/* a timer that will not clear is already gone */
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
pendingRedrive.clear();
|
|
629
|
+
},
|
|
630
|
+
});
|
|
319
631
|
const session = ctx.session;
|
|
320
632
|
if (session && typeof session.hook === "function") {
|
|
321
633
|
registrations.push(await session.hook("context", (event) => {
|
|
322
634
|
try {
|
|
323
635
|
if (scope && scope.count(scope.decide(event)) !== "ours")
|
|
324
636
|
return;
|
|
325
|
-
|
|
637
|
+
const sid = typeof event.sessionID === "string" ? String(event.sessionID) : "";
|
|
638
|
+
scope?.learn(event?.agent, sid);
|
|
639
|
+
// The session is assembling a request on its own (host retry, or the model
|
|
640
|
+
// re-issued): stand the scheduled re-drive down — the revival happened.
|
|
641
|
+
cancelRedrive(sid);
|
|
326
642
|
if (!Array.isArray(event?.system))
|
|
327
643
|
return;
|
|
328
|
-
|
|
644
|
+
// Per-session delivery: a child's wait directive must never ride the
|
|
645
|
+
// lead's request (and vice versa) — the two chains are independent.
|
|
646
|
+
const d = governor.takeDirective(sid);
|
|
329
647
|
if (!d)
|
|
330
648
|
return;
|
|
331
649
|
event.system.push({ type: "text", text: d.text });
|
|
332
650
|
report.directivesInjected++;
|
|
333
651
|
try {
|
|
334
|
-
input.onEvent?.({ event: "injected", delay_ms: d.delayMs });
|
|
652
|
+
input.onEvent?.({ event: "injected", delay_ms: d.delayMs, session: sid });
|
|
335
653
|
}
|
|
336
654
|
catch {
|
|
337
655
|
/* the trajectory line is an extra */
|
|
@@ -352,16 +670,17 @@ export async function applyV2RetryGovernor(ctx, input) {
|
|
|
352
670
|
report.foreignSkipped++;
|
|
353
671
|
return;
|
|
354
672
|
}
|
|
355
|
-
|
|
673
|
+
const sid = typeof event.sessionID === "string" ? String(event.sessionID) : "";
|
|
674
|
+
if (!governor.inCooldown(sid))
|
|
356
675
|
return;
|
|
357
676
|
report.cooldownDenied++;
|
|
358
677
|
try {
|
|
359
|
-
input.onEvent?.({ event: "denied", remaining_ms: governor.cooldownRemainingMs() });
|
|
678
|
+
input.onEvent?.({ event: "denied", remaining_ms: governor.cooldownRemainingMs(sid), session: sid });
|
|
360
679
|
}
|
|
361
680
|
catch {
|
|
362
681
|
/* the trajectory line is an extra */
|
|
363
682
|
}
|
|
364
|
-
const secs = Math.max(1, Math.round(governor.cooldownRemainingMs() / 1000));
|
|
683
|
+
const secs = Math.max(1, Math.round(governor.cooldownRemainingMs(sid) / 1000));
|
|
365
684
|
const message = `配额保护:上游配额/限流已连续打回请求,冷却中(剩余约 ${secs} 秒)。期间不派发新的子代理——子代理的模型请求会继续被打回,需要完成的工作请主代理自己执行,或等冷却结束/提升配额。这是配额保护,不是故障;可用 retry: "off" 关闭。`;
|
|
366
685
|
// Only ever stricter: a host that already denied keeps its decision.
|
|
367
686
|
if (rank.deny <= rank[event.effect ?? "allow"])
|
|
@@ -374,6 +693,6 @@ export async function applyV2RetryGovernor(ctx, input) {
|
|
|
374
693
|
}
|
|
375
694
|
}));
|
|
376
695
|
}
|
|
377
|
-
return { registrations, report };
|
|
696
|
+
return { registrations, report, cancelRedrive };
|
|
378
697
|
}
|
|
379
698
|
//# sourceMappingURL=v2-retry.js.map
|