dsh-unlimited-retry 2.5.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -20
- package/README.zh.md +61 -10
- package/client.js +158 -3
- package/cordis.patch.yml +19 -6
- package/index.js +136 -12
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -117,7 +117,7 @@ the shipped contracts, not assumed:
|
|
|
117
117
|
|
|
118
118
|
### Retry behaviour
|
|
119
119
|
|
|
120
|
-
- 🔁 **
|
|
120
|
+
- 🔁 **Unbounded where that is honest**: no per-step attempt limit for codes that can recover, and a hard per-turn ceiling so a permanent fault cannot hold a turn open
|
|
121
121
|
- ⏱️ **Exponential backoff with jitter**: starts at `initialDelayMs`, doubles, capped at `maxDelayMs`, with per-delay randomization so synchronized storms cannot form
|
|
122
122
|
- 🚧 **The ceiling is a hard constraint**: `maxDelayMs` wins any conflict. Ask for 10 ms and you get 10 ms — it is never silently raised because `initialDelayMs` was larger
|
|
123
123
|
- 🛑 **A cancellation is never retried**: `ABORTED` is terminal at any configuration (see below)
|
|
@@ -172,30 +172,53 @@ silently ignored.
|
|
|
172
172
|
|
|
173
173
|
```yaml
|
|
174
174
|
autoRetry: true
|
|
175
|
-
maxAttempts: null # no attempt limit
|
|
175
|
+
maxAttempts: null # no per-step attempt limit
|
|
176
176
|
initialDelayMs: 1000
|
|
177
177
|
maxDelayMs: 60000
|
|
178
178
|
jitterRatio: 0.2
|
|
179
179
|
retryWithoutProviderPolicy: true
|
|
180
|
-
|
|
180
|
+
retryCodes: # the set the Harness itself retries
|
|
181
|
+
- EMPTY_RESPONSE
|
|
182
|
+
- RATE_LIMIT
|
|
183
|
+
- SERVER
|
|
184
|
+
- TIMEOUT
|
|
185
|
+
- TRANSPORT
|
|
186
|
+
retryAllCodes: false # true = retry everything that is not excluded
|
|
187
|
+
maxRetriesPerTurn: 40 # hard stop across all steps of one turn
|
|
188
|
+
excludeCodes: []
|
|
181
189
|
recordSessionEvents: true
|
|
182
190
|
log: true
|
|
183
191
|
markerFile: null
|
|
184
192
|
```
|
|
185
193
|
|
|
186
|
-
**
|
|
187
|
-
|
|
188
|
-
it is
|
|
189
|
-
|
|
190
|
-
|
|
194
|
+
**Only the codes the Harness itself declares retryable are retried.** That list is
|
|
195
|
+
copied from `dsh-llm`'s own `DEFAULT_RETRYABLE_CODES` rather than invented here:
|
|
196
|
+
it is the judgement of the layer that knows what each failure means. Everything
|
|
197
|
+
else — auth, quota, billing, context overflow, and **any code no adapter
|
|
198
|
+
classified** — is handed straight back to the loop.
|
|
191
199
|
|
|
192
|
-
|
|
200
|
+
Version 2.x defaulted to the opposite, and it was wrong in a specific and
|
|
201
|
+
expensive way: an adapter that cannot classify a stream error reports an opaque
|
|
202
|
+
code (`dsh-llm-pi-ai` calls it `PI_AI_ERROR`) that also covers its explicitly
|
|
203
|
+
non-retryable states, so 2.x retried a permanent fault with no ceiling while the
|
|
204
|
+
turn never ended. A code nobody recognised is not evidence that a retry helps.
|
|
205
|
+
|
|
206
|
+
Two more bounds:
|
|
207
|
+
|
|
208
|
+
1. **`maxRetriesPerTurn`** (default 40) is a hard stop across all steps of a turn.
|
|
209
|
+
The step ladder restarts every step, so without it a fault that survives each
|
|
210
|
+
step would begin again at attempt 1 forever.
|
|
211
|
+
2. **`maxAttempts`** caps per step, exactly as before.
|
|
212
|
+
|
|
213
|
+
Ways to adjust the scope:
|
|
193
214
|
|
|
194
215
|
1. **Set `maxAttempts`** to a positive integer — this plugin then stops after
|
|
195
216
|
that many tries per step and delegates, exactly as if it were not installed.
|
|
196
|
-
2. **
|
|
197
|
-
|
|
198
|
-
|
|
217
|
+
2. **Widen the scope** with `retryAllCodes: true`, or add a specific code to
|
|
218
|
+
`retryCodes`. Then narrow it with `excludeCodes`, which always wins.
|
|
219
|
+
3. **List codes in `excludeCodes`.** The panel has a one-click *Use recommended
|
|
220
|
+
set* action for the codes where a retry re-sends a request that fails
|
|
221
|
+
identically:
|
|
199
222
|
|
|
200
223
|
`AUTH`, `UNAUTHORIZED`, `FORBIDDEN`, `BILLING`, `QUOTA`, `ACCOUNT_QUOTA`,
|
|
201
224
|
`MISSING_CREDENTIAL`, `INVALID_CREDENTIAL`, `MODEL_NOT_FOUND`, `NO_ADAPTER`,
|
|
@@ -293,7 +316,10 @@ The plugin also exports a `Config` schema, so its fields appear in
|
|
|
293
316
|
| `maxDelayMs` | number | `60000` | **Single-delay ceiling; wins any conflict** |
|
|
294
317
|
| `jitterRatio` | number | `0.2` | Randomization per delay (0–1) |
|
|
295
318
|
| `retryWithoutProviderPolicy` | boolean | `true` | Also retry when the adapter published no policy |
|
|
296
|
-
| `
|
|
319
|
+
| `retryCodes` | string[] | the five retryable Harness codes | Codes this plugin retries; anything else is left to the loop |
|
|
320
|
+
| `retryAllCodes` | boolean | `false` | Retry every code that is not a cancellation or an exclusion |
|
|
321
|
+
| `maxRetriesPerTurn` | number \| null | `40` | Hard stop across all steps of one turn; `null` = no turn ceiling |
|
|
322
|
+
| `excludeCodes` | string[] | `[]` | Always-terminal codes (case/whitespace-insensitive); wins over `retryCodes` |
|
|
297
323
|
| `recordSessionEvents` | boolean | `true` | Append `llm/retry` / `llm/retry-started` |
|
|
298
324
|
| `log` | boolean | `true` | Write retry decisions to the DSH log |
|
|
299
325
|
| `markerFile` | string \| null | `null` | Optional mount-probe path (diagnostics) |
|
|
@@ -347,7 +373,7 @@ curl -s http://127.0.0.1:3080/unlimited-retry/stats | head -c 400
|
|
|
347
373
|
| Requirement | Value |
|
|
348
374
|
|---|---|
|
|
349
375
|
| Node.js | **>= 20** |
|
|
350
|
-
| DSH | **>= 0.2.0-0 < 0.3.0-0** (`engines.dsh`, enforced by the host's compatibility preflight). Verified against 0.2.1-rc.3, 0.2.1-rc.5, 0.2.2-alpha.2 and 0.2.2-rc.
|
|
376
|
+
| DSH | **>= 0.2.0-0 < 0.3.0-0** (`engines.dsh`, enforced by the host's compatibility preflight). Verified against 0.2.1-rc.3, 0.2.1-rc.5, 0.2.2-alpha.2, 0.2.2-rc.1 and 0.2.2-rc.3 |
|
|
351
377
|
| Ships `dsh-agent-loop` and `dsh-llm-retry` | Yes — the retry listener hangs off that event |
|
|
352
378
|
|
|
353
379
|
The session-projection reader adapts to both registration shapes the Host has
|
|
@@ -364,8 +390,59 @@ every `-rc` build.
|
|
|
364
390
|
- A 1.x `config` still mounts: `retryAllCodes` is ignored and `terminalCodes` is read as `excludeCodes`, each with a one-line warning naming the replacement.
|
|
365
391
|
- The Web panel additionally needs `configEditor` and the `settings.section` slot. Without them the policy still applies; the page is simply absent or read-only.
|
|
366
392
|
|
|
393
|
+
## What this plugin does not touch: tool calls and MCP calls
|
|
394
|
+
|
|
395
|
+
A retry here **re-issues the model request only.** It never re-runs a tool.
|
|
396
|
+
|
|
397
|
+
That is structural, not a promise: `agent/request-error` is emitted by
|
|
398
|
+
`dsh-agent-loop` at exactly one place — when the assistant **stream** finished
|
|
399
|
+
with `kind === "error"` or `"aborted"` — and the loop's tool execution happens
|
|
400
|
+
after a message has been produced. A tool that throws never becomes a
|
|
401
|
+
`request-error`; it becomes a `tool/result` error and is thrown by the loop.
|
|
402
|
+
|
|
403
|
+
Verified against real transcripts rather than by reading code: across 41 recorded
|
|
404
|
+
retries in the local session store, **41 were preceded by `assistant/attempt`** —
|
|
405
|
+
the failed stream settlement — and none by `tool/result`.
|
|
406
|
+
|
|
407
|
+
So if a turn appears to loop while calling the same tool, the loop belongs to the
|
|
408
|
+
model or to the tool, not to this plugin:
|
|
409
|
+
|
|
410
|
+
- a **model** that keeps choosing a failing tool is what `dsh-repeat-tool-reminder`
|
|
411
|
+
exists for;
|
|
412
|
+
- an **MCP server** that returns errors will keep returning them until it is fixed;
|
|
413
|
+
- this plugin only decides whether to re-send the *request*, and since 3.0.0 it only
|
|
414
|
+
does so for codes that can plausibly recover.
|
|
415
|
+
|
|
416
|
+
### 3.0.0
|
|
417
|
+
|
|
418
|
+
**Breaking: the retry scope was inverted.** 2.x retried *every* code except a
|
|
419
|
+
cancellation. In practice that meant an adapter that could not classify a stream
|
|
420
|
+
error (`PI_AI_ERROR`) — which also covers its explicitly non-retryable states —
|
|
421
|
+
was retried with no ceiling, and a turn whose fault survived each step never
|
|
422
|
+
ended. Real transcripts show the ladder advancing step after step against the same
|
|
423
|
+
`tier_limit_exceeded` condition.
|
|
424
|
+
|
|
425
|
+
- Default is now the Harness's own retryable set (`EMPTY_RESPONSE`, `RATE_LIMIT`,
|
|
426
|
+
`SERVER`, `TIMEOUT`, `TRANSPORT`). Anything else — including an unclassified
|
|
427
|
+
code — is left to the loop.
|
|
428
|
+
- `retryCodes` and `retryAllCodes` make the scope explicit and reversible.
|
|
429
|
+
`excludeCodes` still wins over both.
|
|
430
|
+
- `maxRetriesPerTurn` (default 40) stops a fault that survives every step, which
|
|
431
|
+
the per-step ladder alone could never catch.
|
|
432
|
+
- Tool calls and MCP calls were never retried — see the section above, which is
|
|
433
|
+
backed by 41 recorded retries with no `tool/result` predecessor among them.
|
|
434
|
+
|
|
435
|
+
If you relied on 2.x's behaviour, set `retryAllCodes: true`. The panel's
|
|
436
|
+
"What gets retried" card makes that a switch rather than a config edit.
|
|
437
|
+
|
|
367
438
|
## Changelog
|
|
368
439
|
|
|
440
|
+
### 2.5.1
|
|
441
|
+
|
|
442
|
+
- Infrastructure release. The release workflow now publishes through npm **trusted
|
|
443
|
+
publishing** (GitHub OIDC) instead of a static token, so there is no credential to
|
|
444
|
+
store, rotate, or lose, and the long-lived bypass-2FA token is gone.
|
|
445
|
+
|
|
369
446
|
### 2.5.0
|
|
370
447
|
|
|
371
448
|
- The ladder's last row printed the plugin's **first** delay beside a label that
|
|
@@ -380,7 +457,10 @@ every `-rc` build.
|
|
|
380
457
|
|
|
381
458
|
## Releasing
|
|
382
459
|
|
|
383
|
-
Publishing is tag-driven; nothing publishes from a branch
|
|
460
|
+
Publishing is tag-driven; nothing publishes from a branch, and no npm credential
|
|
461
|
+
is stored anywhere. The workflow authenticates with **trusted publishing**: npm
|
|
462
|
+
verifies a short-lived OIDC token that GitHub issues to this job, matched against
|
|
463
|
+
the repository and workflow file registered on the package.
|
|
384
464
|
|
|
385
465
|
```bash
|
|
386
466
|
# 1. bump the version in package.json (and PLUGIN_VERSION in index.js)
|
|
@@ -389,11 +469,11 @@ npm version patch --no-git-tag-version # or minor / major
|
|
|
389
469
|
git tag v2.1.3 && git push origin v2.1.3
|
|
390
470
|
```
|
|
391
471
|
|
|
392
|
-
The `Release` workflow then refuses a tag that disagrees with `package.json
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
472
|
+
The `Release` workflow then refuses a tag that disagrees with `package.json`, tests
|
|
473
|
+
the tree, packs, publishes with npm provenance, creates the GitHub release with the
|
|
474
|
+
tarball attached, and finally re-installs the published version from the registry
|
|
475
|
+
to prove that copy mounts. It needs no secret: only `id-token: write`, which the
|
|
476
|
+
workflow already declares.
|
|
397
477
|
|
|
398
478
|
`CI` runs the suite on Node 20, 22 and 24, then packs the tarball, asserts the
|
|
399
479
|
published file set has not drifted, installs it into a clean project, and mounts
|
package/README.zh.md
CHANGED
|
@@ -90,7 +90,7 @@ DSH 自带的 `@deepseek-ai/dsh-llm-retry` 只负责**执行**重试,真正的
|
|
|
90
90
|
|
|
91
91
|
### 重试行为
|
|
92
92
|
|
|
93
|
-
- 🔁
|
|
93
|
+
- 🔁 **在诚实的地方才无限**:对「有可能恢复」的错误码不设每步次数上限,但每轮有一道硬上限,永久故障无法把一轮拖住
|
|
94
94
|
- ⏱️ **指数退避 + 抖动**:`initialDelayMs` 起算,每次翻倍,封顶 `maxDelayMs`,每轮叠加随机抖动避免同步风暴
|
|
95
95
|
- 🚧 **上限是硬约束**:`maxDelayMs` 是天花板,**任何配置冲突都以它为准**。设了 10ms 就是 10ms,不会因为 `initialDelayMs` 更大而被悄悄抬高
|
|
96
96
|
- 🛑 **取消永不重试**:`ABORTED` 在任何配置下都终止(见下)
|
|
@@ -140,18 +140,34 @@ initialDelayMs: 1000
|
|
|
140
140
|
maxDelayMs: 60000
|
|
141
141
|
jitterRatio: 0.2
|
|
142
142
|
retryWithoutProviderPolicy: true
|
|
143
|
-
|
|
143
|
+
retryCodes: # Harness 自己认定可重试的那一组
|
|
144
|
+
- EMPTY_RESPONSE
|
|
145
|
+
- RATE_LIMIT
|
|
146
|
+
- SERVER
|
|
147
|
+
- TIMEOUT
|
|
148
|
+
- TRANSPORT
|
|
149
|
+
retryAllCodes: false # true = 除终止码外全部重试
|
|
150
|
+
maxRetriesPerTurn: 40 # 一轮内所有步骤合计的硬上限
|
|
151
|
+
excludeCodes: []
|
|
144
152
|
recordSessionEvents: true
|
|
145
153
|
log: true
|
|
146
154
|
markerFile: null
|
|
147
155
|
```
|
|
148
156
|
|
|
149
|
-
|
|
157
|
+
**默认只重试 Harness 自己认定的可重试错误码。** 这份列表直接取自 `dsh-llm` 的 `DEFAULT_RETRYABLE_CODES`,不是这里自己编的:它来自真正知道每种失败含义的那一层。其余一律立刻交回上层——认证、配额、计费、上下文溢出,以及**任何适配器没能归类的错误码**。
|
|
150
158
|
|
|
151
|
-
|
|
159
|
+
2.x 的默认正好相反,而且错得很具体、代价很高:适配器在无法归类流错误时会报一个不透明的码(`dsh-llm-pi-ai` 叫它 `PI_AI_ERROR`),而这个码同时覆盖它明确标记为不可重试的状态;于是 2.x 拿着一个永久故障、没有上限地重试,整轮永远结束不了。**一个没人认出来的错误码,不能当作「重试会有用」的依据。**
|
|
160
|
+
|
|
161
|
+
另外两道闸:
|
|
162
|
+
|
|
163
|
+
1. **`maxRetriesPerTurn`**(默认 40)是同一轮内所有步骤合计的硬上限。阶梯每步都会重置,没有它,一个跨步骤持续存在的故障会每步重新从第 1 次开始,无穷无尽。
|
|
164
|
+
2. **`maxAttempts`** 照旧限制每个 step。
|
|
165
|
+
|
|
166
|
+
调整范围的方式:
|
|
152
167
|
|
|
153
168
|
1. **设置 `maxAttempts`** 为正整数 —— 本插件在每个 step 重试这么多次后交回上层,效果等同于没装。
|
|
154
|
-
2.
|
|
169
|
+
2. **放宽范围**:`retryAllCodes: true`,或把某个具体码加进 `retryCodes`;再用 `excludeCodes` 收窄,它永远优先。
|
|
170
|
+
3. **把错误码写进 `excludeCodes`。** 面板里有一键*使用推荐集合*,这些码的共同点是「重发一次仍然同样失败」:
|
|
155
171
|
|
|
156
172
|
`AUTH`、`UNAUTHORIZED`、`FORBIDDEN`、`BILLING`、`QUOTA`、`ACCOUNT_QUOTA`、
|
|
157
173
|
`MISSING_CREDENTIAL`、`INVALID_CREDENTIAL`、`MODEL_NOT_FOUND`、`NO_ADAPTER`、
|
|
@@ -234,7 +250,10 @@ dsh-unlimited-retry/
|
|
|
234
250
|
| `maxDelayMs` | number | `60000` | **单次延迟天花板,冲突时以它为准** |
|
|
235
251
|
| `jitterRatio` | number | `0.2` | 每轮延迟的随机浮动比例(0–1) |
|
|
236
252
|
| `retryWithoutProviderPolicy` | boolean | `true` | 适配器没发布 `retryPolicy` 时也重试 |
|
|
237
|
-
| `
|
|
253
|
+
| `retryCodes` | string[] | Harness 那 5 个可重试码 | 本插件会重试的错误码;其他一律交回上层 |
|
|
254
|
+
| `retryAllCodes` | boolean | `false` | 除取消与终止码之外全部重试 |
|
|
255
|
+
| `maxRetriesPerTurn` | number \| null | `40` | 同一轮内所有步骤合计的硬上限;`null` = 本轮不限 |
|
|
256
|
+
| `excludeCodes` | string[] | `[]` | 始终终止的错误码(大小写/空白不敏感),优先于 `retryCodes` |
|
|
238
257
|
| `recordSessionEvents` | boolean | `true` | 是否写入 `llm/retry` / `llm/retry-started` |
|
|
239
258
|
| `log` | boolean | `true` | 是否把重试决定写进 DSH 日志 |
|
|
240
259
|
| `markerFile` | string \| null | `null` | 可选:挂载探针文件路径(诊断用) |
|
|
@@ -272,7 +291,7 @@ curl -s http://127.0.0.1:3080/unlimited-retry/stats | head -c 400
|
|
|
272
291
|
| 依赖 | 要求 |
|
|
273
292
|
|---|---|
|
|
274
293
|
| Node.js | **>= 20** |
|
|
275
|
-
| DSH | **>= 0.2.0-0 < 0.3.0-0**(`package.json` 的 `engines.dsh`,由宿主的兼容性预检强制执行)。已在 0.2.1-rc.3、0.2.1-rc.5、0.2.2-alpha.2、0.2.2-rc.1 上验证 |
|
|
294
|
+
| DSH | **>= 0.2.0-0 < 0.3.0-0**(`package.json` 的 `engines.dsh`,由宿主的兼容性预检强制执行)。已在 0.2.1-rc.3、0.2.1-rc.5、0.2.2-alpha.2、0.2.2-rc.1、0.2.2-rc.3 上验证 |
|
|
276
295
|
| 带 `dsh-agent-loop` 与 `dsh-llm-retry` | 是(重试监听器挂在那条事件上) |
|
|
277
296
|
|
|
278
297
|
会话投影读取对宿主先后提供过的两种注册形态都做了适配:0.2.2 之前 `register`
|
|
@@ -284,8 +303,38 @@ curl -s http://127.0.0.1:3080/unlimited-retry/stats | head -c 400
|
|
|
284
303
|
- 1.x 的 `config` 仍可挂载:`retryAllCodes` 被忽略(除非显式排除,否则每个码都会重试),`terminalCodes` 按 `excludeCodes` 读取,两者各打一条替代提示。
|
|
285
304
|
- Web 面板另需 `configEditor` 与 `settings.section` 槽位;缺这些时策略照常生效,页面只是不存在或只读。
|
|
286
305
|
|
|
306
|
+
## 本插件不碰的东西:工具调用与 MCP 调用
|
|
307
|
+
|
|
308
|
+
这里的重试**只重发模型请求**,从不重跑工具。
|
|
309
|
+
|
|
310
|
+
这是结构决定的,不是承诺:`agent/request-error` 在 `dsh-agent-loop` 里只有一处触发点 —— 助手**流**以 `kind === "error"` 或 `"aborted"` 结束时 —— 而循环里的工具执行发生在消息产出之后。工具抛错不会变成 `request-error`,它会变成 `tool/result` 的失败,并由循环抛出。
|
|
311
|
+
|
|
312
|
+
这一点是对真实会话验证的,不是靠读代码推断:本机会话库里 41 条重试记录,**41 条前一条事件都是 `assistant/attempt`**(失败的流结算),没有一条前面是 `tool/result`。
|
|
313
|
+
|
|
314
|
+
所以,如果某一轮看起来在反复调用同一个工具,那个循环属于模型或工具,不属于本插件:
|
|
315
|
+
|
|
316
|
+
- **模型**一直挑那个失败的工具 —— 这是 `dsh-repeat-tool-reminder` 负责的事;
|
|
317
|
+
- **MCP 服务**返回错误,在修好之前会一直返回错误;
|
|
318
|
+
- 本插件只决定要不要重发**请求**,而且从 3.0.0 起只对那些「有可能恢复」的错误码这么做。
|
|
319
|
+
|
|
320
|
+
### 3.0.0
|
|
321
|
+
|
|
322
|
+
**破坏性变更:重试范围反过来了。** 2.x 除取消之外「什么都重试」。实际后果是:适配器无法归类流错误时给出的 `PI_AI_ERROR`(这个码同时覆盖它明确标记为不可重试的状态)被无上限地重试,于是一个跨步骤持续存在的故障让整轮永远结束不了。真实会话记录里可以看到阶梯一步一步往前走,而同一条 `tier_limit_exceeded` 一直在。
|
|
323
|
+
|
|
324
|
+
- 默认改成 Harness 自己认定的可重试集合(`EMPTY_RESPONSE`、`RATE_LIMIT`、`SERVER`、`TIMEOUT`、`TRANSPORT`)。其他一律交回上层——包括未能归类的错误码。
|
|
325
|
+
- `retryCodes` 与 `retryAllCodes` 把范围变成显式且可逆的设置;`excludeCodes` 始终优先。
|
|
326
|
+
- `maxRetriesPerTurn`(默认 40)挡住「每个步骤都重来一遍」的持续故障——只靠按步阶梯永远挡不住。
|
|
327
|
+
- 工具调用与 MCP 调用从来就不在本插件的重试范围内——见上一节,依据是 41 条重试记录里没有一条前面是 `tool/result`。
|
|
328
|
+
|
|
329
|
+
如果你确实依赖 2.x 的行为,设 `retryAllCodes: true`。面板的「哪些错误会被重试」卡片把它做成了一个开关,不需要改配置文件。
|
|
330
|
+
|
|
287
331
|
## 更新记录
|
|
288
332
|
|
|
333
|
+
### 2.5.1
|
|
334
|
+
|
|
335
|
+
- 基础设施版本。发布流程改用 npm 的 **trusted publishing**(GitHub OIDC),不再依赖静态 token ——
|
|
336
|
+
没有凭证需要保存、轮换或续期,那个绕过 2FA 的长效 token 已经移除。
|
|
337
|
+
|
|
289
338
|
### 2.5.0
|
|
290
339
|
|
|
291
340
|
- 重试阶梯最后一行把插件的**首个**延迟印在一个标明上限的标签旁边,读起来是「此后每一次延迟均封顶 1.0 分钟 —— 1.0 秒」。现在显示上限本身。
|
|
@@ -294,7 +343,9 @@ curl -s http://127.0.0.1:3080/unlimited-retry/stats | head -c 400
|
|
|
294
343
|
|
|
295
344
|
## 发布流程
|
|
296
345
|
|
|
297
|
-
发布由 tag
|
|
346
|
+
发布由 tag 驱动,任何分支都不会触发发布,且**不保存任何 npm 凭证**。工作流使用
|
|
347
|
+
**trusted publishing**:npm 校验 GitHub 为本任务签发的短期 OIDC token,并与登记在包上的
|
|
348
|
+
仓库与工作流文件比对。
|
|
298
349
|
|
|
299
350
|
```bash
|
|
300
351
|
# 1. 改 package.json 的版本号(以及 index.js 里的 PLUGIN_VERSION)
|
|
@@ -305,8 +356,8 @@ git tag v2.1.3 && git push origin v2.1.3
|
|
|
305
356
|
|
|
306
357
|
`Release` 工作流会:拒绝与 `package.json` 不一致的 tag → 跑测试 → 打包 →
|
|
307
358
|
带 npm provenance 发布 → 创建 GitHub Release 并附上 tarball → 最后从 registry
|
|
308
|
-
|
|
309
|
-
|
|
359
|
+
重新安装已发布版本,证明那一份也能正常挂载。它**不需要任何 secret**,只要
|
|
360
|
+
`id-token: write`,工作流里已声明。
|
|
310
361
|
|
|
311
362
|
`CI` 在 Node 20 / 22 / 24 上跑测试套件,然后打包、断言发布文件清单没有漂移、
|
|
312
363
|
装进一个干净工程,并挂载安装后的宿主端——包括「取消永不重试」这条规则。
|
package/client.js
CHANGED
|
@@ -78,9 +78,17 @@ window.__ModuleLoader__.load({
|
|
|
78
78
|
withoutPolicyHint: "Also retry when the serving adapter published no retryPolicy.",
|
|
79
79
|
recordEvents: "Record session events",
|
|
80
80
|
recordEventsHint: "Append llm/retry and llm/retry-started so the chat surface shows the chain.",
|
|
81
|
-
|
|
81
|
+
scopeTitle: "What gets retried",
|
|
82
|
+
scopeHint: "Only codes in this list are retried. Anything else — including a code no adapter classified — is left to the loop, because an unrecognised failure is not evidence that a retry helps.",
|
|
83
|
+
retryAll: "Retry every code",
|
|
84
|
+
retryAllHint: "Retry anything that is not a cancellation or an exclusion, including unclassified adapter codes.",
|
|
85
|
+
retryAllNote: "Every code is eligible. A code nobody classified can then retry until the turn ceiling.",
|
|
86
|
+
retryNone: "Nothing is retried: every failure is left to the loop.",
|
|
87
|
+
turnCap: "Retries per turn",
|
|
88
|
+
turnCapHint: "Hard stop across all steps of one turn. The step ladder restarts each step, so this is what stops a fault that survives every step.",
|
|
89
|
+
excludeTitle: "Always terminal",
|
|
82
90
|
excludeHint: "Failures with these codes are left to the loop instead of retried.",
|
|
83
|
-
excludeEmpty: "No
|
|
91
|
+
excludeEmpty: "No extra exclusions: the retry set alone decides what is retried.",
|
|
84
92
|
excludeAdd: "Add a code",
|
|
85
93
|
excludePlaceholder: "e.g. INVALID_REQUEST",
|
|
86
94
|
excludeRemove: "Remove {code}",
|
|
@@ -165,7 +173,15 @@ window.__ModuleLoader__.load({
|
|
|
165
173
|
withoutPolicyHint: "当 provider 适配器没有发布 retryPolicy 时同样重试。",
|
|
166
174
|
recordEvents: "写入会话事件",
|
|
167
175
|
recordEventsHint: "追加 llm/retry 与 llm/retry-started,聊天界面才会显示重试链路。",
|
|
168
|
-
|
|
176
|
+
scopeTitle: "哪些错误会被重试",
|
|
177
|
+
scopeHint: "只有在列表里的错误码会重试。其他一律交回上层——包括适配器没能归类的错误码:一个无法识别的失败,并不能作为「重试会有用」的依据。",
|
|
178
|
+
retryAll: "重试所有错误码",
|
|
179
|
+
retryAllHint: "除取消和终止码之外全部重试,包括适配器未能归类的码。",
|
|
180
|
+
retryAllNote: "所有错误码都可重试。未能归类的码会一直重试到本轮上限。",
|
|
181
|
+
retryNone: "不重试任何错误码:所有失败都交回上层。",
|
|
182
|
+
turnCap: "每轮重试上限",
|
|
183
|
+
turnCapHint: "同一轮内所有步骤合计的硬上限。阶梯每步都会重置,所以这才是挡住「跨步骤持续失败」的那道闸。",
|
|
184
|
+
excludeTitle: "始终终止",
|
|
169
185
|
excludeHint: "命中这些错误码的失败交回上层,不再重试。",
|
|
170
186
|
excludeEmpty: "没有终止错误码:所有失败都会重试。",
|
|
171
187
|
excludeAdd: "添加错误码",
|
|
@@ -650,6 +666,8 @@ window.__ModuleLoader__.load({
|
|
|
650
666
|
const [draft, setDraft] = React.useState(null);
|
|
651
667
|
const [codeInput, setCodeInput] = React.useState("");
|
|
652
668
|
const [codeError, setCodeError] = React.useState(null);
|
|
669
|
+
const [retryInput, setRetryInput] = React.useState("");
|
|
670
|
+
const [retryError, setRetryError] = React.useState(null);
|
|
653
671
|
const [pending, setPending] = React.useState(false);
|
|
654
672
|
const [outcome, setOutcome] = React.useState(null);
|
|
655
673
|
const [confirmReset, setConfirmReset] = React.useState(false);
|
|
@@ -699,6 +717,9 @@ window.__ModuleLoader__.load({
|
|
|
699
717
|
maxDelayMs: draft.maxDelayMs,
|
|
700
718
|
jitterRatio: draft.jitterRatio,
|
|
701
719
|
retryWithoutProviderPolicy: draft.retryWithoutProviderPolicy,
|
|
720
|
+
retryCodes: draft.retryCodes,
|
|
721
|
+
retryAllCodes: draft.retryAllCodes,
|
|
722
|
+
maxRetriesPerTurn: draft.maxRetriesPerTurn,
|
|
702
723
|
excludeCodes: draft.excludeCodes,
|
|
703
724
|
recordSessionEvents: draft.recordSessionEvents,
|
|
704
725
|
};
|
|
@@ -758,6 +779,21 @@ window.__ModuleLoader__.load({
|
|
|
758
779
|
patch({ excludeCodes: [...(draft?.excludeCodes ?? []), code] });
|
|
759
780
|
}, [codeInput, draft, fmt, patch, t]);
|
|
760
781
|
|
|
782
|
+
const addRetryCode = React.useCallback(() => {
|
|
783
|
+
const code = retryInput.trim().toUpperCase();
|
|
784
|
+
if (code.length === 0) {
|
|
785
|
+
setRetryError(t("excludeUnknown"));
|
|
786
|
+
return;
|
|
787
|
+
}
|
|
788
|
+
if (Array.isArray(draft?.retryCodes) && draft.retryCodes.includes(code)) {
|
|
789
|
+
setRetryError(fmt.fill(t("excludeDuplicate"), { code }));
|
|
790
|
+
return;
|
|
791
|
+
}
|
|
792
|
+
setRetryError(null);
|
|
793
|
+
setRetryInput("");
|
|
794
|
+
patch({ retryCodes: [...(draft?.retryCodes ?? []), code] });
|
|
795
|
+
}, [draft, fmt, patch, retryInput, t]);
|
|
796
|
+
|
|
761
797
|
const styleTag = React.createElement("style", { key: "css", children: CSS });
|
|
762
798
|
|
|
763
799
|
const heading = React.createElement("div", { className: "duHead", key: "head" }, [
|
|
@@ -910,6 +946,124 @@ window.__ModuleLoader__.load({
|
|
|
910
946
|
React.createElement("span", { key: "note", className: "duRungNote" }, maxLabel),
|
|
911
947
|
]));
|
|
912
948
|
|
|
949
|
+
// What this layer will and will not retry. 2.x retried every code except a
|
|
950
|
+
// cancellation, which is how an unclassified adapter code kept a turn alive.
|
|
951
|
+
const retrySet = Array.isArray(config.retryCodes) ? config.retryCodes : [];
|
|
952
|
+
const retryingEverything = config.retryAllCodes === true;
|
|
953
|
+
const scopeCard = React.createElement("div", { key: "scope", className: "duCard" }, [
|
|
954
|
+
React.createElement("div", { key: "h", className: "duCardHead" }, [
|
|
955
|
+
React.createElement("div", { key: "t", className: "duCardTitle" }, t("scopeTitle")),
|
|
956
|
+
]),
|
|
957
|
+
React.createElement("div", { key: "hint", className: "duCardHint" }, t("scopeHint")),
|
|
958
|
+
React.createElement("div", { key: "b", className: "duCardBody" }, [
|
|
959
|
+
React.createElement(Field, {
|
|
960
|
+
key: "all",
|
|
961
|
+
id: "du-retry-all",
|
|
962
|
+
label: t("retryAll"),
|
|
963
|
+
hint: t("retryAllHint"),
|
|
964
|
+
}, React.createElement(Switch, {
|
|
965
|
+
id: "du-retry-all",
|
|
966
|
+
checked: retryingEverything,
|
|
967
|
+
disabled: !writable || pending || config.autoRetry !== true,
|
|
968
|
+
onChange: (next) => patch({ retryAllCodes: next }),
|
|
969
|
+
labelId: "du-retry-all-label",
|
|
970
|
+
describedById: "du-retry-all-hint",
|
|
971
|
+
})),
|
|
972
|
+
|
|
973
|
+
retryingEverything
|
|
974
|
+
? React.createElement("div", { key: "allnote", className: "duFieldHint" }, t("retryAllNote"))
|
|
975
|
+
: React.createElement("div", { key: "chips", className: "duChips" }, retrySet.length === 0
|
|
976
|
+
? React.createElement("div", { className: "duEmptyLine" }, t("retryNone"))
|
|
977
|
+
: retrySet.map((code) => React.createElement("span", {
|
|
978
|
+
key: code,
|
|
979
|
+
className: "duChip",
|
|
980
|
+
"data-mono": "true",
|
|
981
|
+
}, [
|
|
982
|
+
React.createElement("span", { key: "c" }, code),
|
|
983
|
+
React.createElement("button", {
|
|
984
|
+
key: "x",
|
|
985
|
+
type: "button",
|
|
986
|
+
className: "duChipX",
|
|
987
|
+
disabled: !writable || pending || config.autoRetry !== true,
|
|
988
|
+
"aria-label": fmt.fill(t("excludeRemove"), { code }),
|
|
989
|
+
title: fmt.fill(t("excludeRemove"), { code }),
|
|
990
|
+
onClick: () => patch({ retryCodes: retrySet.filter((entry) => entry !== code) }),
|
|
991
|
+
}, "×"),
|
|
992
|
+
]))),
|
|
993
|
+
|
|
994
|
+
React.createElement("div", {
|
|
995
|
+
key: "add",
|
|
996
|
+
style: { display: "flex", gap: 6, flexWrap: "wrap", alignItems: "center", marginTop: 4 },
|
|
997
|
+
}, [
|
|
998
|
+
React.createElement("input", {
|
|
999
|
+
key: "i",
|
|
1000
|
+
id: "du-retry-input",
|
|
1001
|
+
className: "duInput",
|
|
1002
|
+
type: "text",
|
|
1003
|
+
autoComplete: "off",
|
|
1004
|
+
spellCheck: false,
|
|
1005
|
+
disabled: !writable || pending || config.autoRetry !== true || retryingEverything,
|
|
1006
|
+
value: retryInput,
|
|
1007
|
+
placeholder: t("excludePlaceholder"),
|
|
1008
|
+
list: "du-code-options-scope",
|
|
1009
|
+
"aria-label": t("retryCodes"),
|
|
1010
|
+
onChange: (event) => {
|
|
1011
|
+
setRetryInput(event.target.value);
|
|
1012
|
+
setRetryError(null);
|
|
1013
|
+
},
|
|
1014
|
+
onKeyDown: (event) => {
|
|
1015
|
+
if (event.key === "Enter") {
|
|
1016
|
+
event.preventDefault();
|
|
1017
|
+
addRetryCode();
|
|
1018
|
+
}
|
|
1019
|
+
},
|
|
1020
|
+
}),
|
|
1021
|
+
React.createElement("datalist", { key: "codes", id: "du-code-options-scope" },
|
|
1022
|
+
(data.knownFailureCodes ?? []).concat(data.harnessRetryableCodes ?? []).map((code) =>
|
|
1023
|
+
React.createElement("option", { key: code, value: code }))),
|
|
1024
|
+
React.createElement("button", {
|
|
1025
|
+
key: "b",
|
|
1026
|
+
type: "button",
|
|
1027
|
+
className: "duBtn",
|
|
1028
|
+
disabled: !writable || pending || config.autoRetry !== true || retryingEverything,
|
|
1029
|
+
onClick: addRetryCode,
|
|
1030
|
+
}, t("excludeAddAction")),
|
|
1031
|
+
]),
|
|
1032
|
+
retryError === null
|
|
1033
|
+
? null
|
|
1034
|
+
: React.createElement("div", { key: "e", className: "duFieldHint", "data-tone": "error", role: "alert" }, retryError),
|
|
1035
|
+
|
|
1036
|
+
React.createElement(Field, {
|
|
1037
|
+
key: "turncap",
|
|
1038
|
+
id: "du-turn-cap",
|
|
1039
|
+
label: t("turnCap"),
|
|
1040
|
+
hint: t("turnCapHint"),
|
|
1041
|
+
}, [
|
|
1042
|
+
React.createElement("input", {
|
|
1043
|
+
key: "i",
|
|
1044
|
+
id: "du-turn-cap",
|
|
1045
|
+
className: "duInput",
|
|
1046
|
+
type: "number",
|
|
1047
|
+
min: 1,
|
|
1048
|
+
step: 1,
|
|
1049
|
+
inputMode: "numeric",
|
|
1050
|
+
placeholder: t("unlimited"),
|
|
1051
|
+
disabled: !writable || pending || config.autoRetry !== true,
|
|
1052
|
+
value: config.maxRetriesPerTurn === null || config.maxRetriesPerTurn === undefined
|
|
1053
|
+
? ""
|
|
1054
|
+
: String(config.maxRetriesPerTurn),
|
|
1055
|
+
onChange: (event) => {
|
|
1056
|
+
const raw = event.target.value.trim();
|
|
1057
|
+
const parsed = raw.length === 0 ? null : parseField(raw, 1);
|
|
1058
|
+
if (raw.length === 0 || parsed !== undefined) patch({ maxRetriesPerTurn: parsed });
|
|
1059
|
+
},
|
|
1060
|
+
}),
|
|
1061
|
+
React.createElement("span", { key: "u", className: "duUnit" },
|
|
1062
|
+
config.maxRetriesPerTurn === null || config.maxRetriesPerTurn === undefined ? t("unlimited") : ""),
|
|
1063
|
+
]),
|
|
1064
|
+
]),
|
|
1065
|
+
]);
|
|
1066
|
+
|
|
913
1067
|
const policyCard = React.createElement("div", { key: "policy", className: "duCard" }, [
|
|
914
1068
|
React.createElement("div", { key: "h", className: "duCardHead" }, [
|
|
915
1069
|
React.createElement("div", { key: "t", className: "duCardTitle" }, t("policyTitle")),
|
|
@@ -1229,6 +1383,7 @@ window.__ModuleLoader__.load({
|
|
|
1229
1383
|
]),
|
|
1230
1384
|
]),
|
|
1231
1385
|
|
|
1386
|
+
scopeCard,
|
|
1232
1387
|
policyCard,
|
|
1233
1388
|
|
|
1234
1389
|
React.createElement("div", { key: "dist", className: "duGrid2" }, [
|
package/cordis.patch.yml
CHANGED
|
@@ -6,13 +6,18 @@
|
|
|
6
6
|
#
|
|
7
7
|
# Defaults are deliberate:
|
|
8
8
|
# autoRetry: true — the plugin's entire reason to exist
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
9
|
+
# retryCodes: the five the Harness itself retries (EMPTY_RESPONSE, RATE_LIMIT,
|
|
10
|
+
# SERVER, TIMEOUT, TRANSPORT). Anything else is the upstream
|
|
11
|
+
# layer's judgement that re-sending will not help: a code nobody
|
|
12
|
+
# classified is how 2.x retried the same dead request forever.
|
|
13
|
+
# retryAllCodes: false — set true to retry everything except exclusions
|
|
14
|
+
# maxAttempts: null — no per-step attempt limit
|
|
15
|
+
# maxRetriesPerTurn: 40 — hard stop, so a fault that survives every step
|
|
16
|
+
# cannot keep a turn alive indefinitely
|
|
17
|
+
# maxDelayMs: 60000 — ceiling for any single wait
|
|
12
18
|
#
|
|
13
|
-
# If you want a
|
|
14
|
-
#
|
|
15
|
-
# exactly that set).
|
|
19
|
+
# If you want a wider or narrower policy, change retryCodes (or retryAllCodes),
|
|
20
|
+
# and use excludeCodes for codes that must stay terminal either way.
|
|
16
21
|
|
|
17
22
|
- insert:
|
|
18
23
|
- id: unlimited-retry
|
|
@@ -24,6 +29,14 @@
|
|
|
24
29
|
maxDelayMs: 60000
|
|
25
30
|
jitterRatio: 0.2
|
|
26
31
|
retryWithoutProviderPolicy: true
|
|
32
|
+
retryCodes:
|
|
33
|
+
- EMPTY_RESPONSE
|
|
34
|
+
- RATE_LIMIT
|
|
35
|
+
- SERVER
|
|
36
|
+
- TIMEOUT
|
|
37
|
+
- TRANSPORT
|
|
38
|
+
retryAllCodes: false
|
|
39
|
+
maxRetriesPerTurn: 40
|
|
27
40
|
excludeCodes: []
|
|
28
41
|
recordSessionEvents: true
|
|
29
42
|
log: true
|
package/index.js
CHANGED
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
/** Cordis function-plugin name; also the Loader row id used by the patch. */
|
|
41
41
|
export const name = "unlimited-retry";
|
|
42
42
|
|
|
43
|
-
const PLUGIN_VERSION = "
|
|
43
|
+
const PLUGIN_VERSION = "3.0.0";
|
|
44
44
|
const STATS_PATH = "/unlimited-retry/stats";
|
|
45
45
|
const CONFIG_PATH = "/unlimited-retry/config";
|
|
46
46
|
const RESET_PATH = "/unlimited-retry/reset";
|
|
@@ -75,6 +75,30 @@ const ALWAYS_TERMINAL_CODES = Object.freeze(["ABORTED"]);
|
|
|
75
75
|
/** Fast membership test for {@link ALWAYS_TERMINAL_CODES}. */
|
|
76
76
|
const ALWAYS_TERMINAL = new Set(ALWAYS_TERMINAL_CODES);
|
|
77
77
|
|
|
78
|
+
/**
|
|
79
|
+
* The failure codes a retry can plausibly recover from.
|
|
80
|
+
*
|
|
81
|
+
* Copied from the shipped `dsh-llm` layer's own `DEFAULT_RETRYABLE_CODES`. That
|
|
82
|
+
* list is the Harness's explicit judgement about which failures are worth
|
|
83
|
+
* re-sending, and a plugin that retries *outside* it is overriding a decision
|
|
84
|
+
* that upstream made deliberately.
|
|
85
|
+
*
|
|
86
|
+
* This is the whole reason 2.x over-retried: it defaulted to "nothing is
|
|
87
|
+
* terminal", so an adapter code such as `PI_AI_ERROR` — which pi-ai emits for
|
|
88
|
+
* both unclassified stream errors *and* explicitly non-retryable `pending` /
|
|
89
|
+
* `deferred` states — was retried with no ceiling until the turn was killed.
|
|
90
|
+
*/
|
|
91
|
+
const HARNESS_RETRYABLE_CODES = Object.freeze([
|
|
92
|
+
"EMPTY_RESPONSE",
|
|
93
|
+
"RATE_LIMIT",
|
|
94
|
+
"SERVER",
|
|
95
|
+
"TIMEOUT",
|
|
96
|
+
"TRANSPORT",
|
|
97
|
+
]);
|
|
98
|
+
|
|
99
|
+
/** Default allow-list; see {@link HARNESS_RETRYABLE_CODES}. */
|
|
100
|
+
const DEFAULT_RETRY_CODES = HARNESS_RETRYABLE_CODES;
|
|
101
|
+
|
|
78
102
|
/**
|
|
79
103
|
* Every failure code the Harness itself can produce, for the panel's picker.
|
|
80
104
|
*
|
|
@@ -138,7 +162,13 @@ const DEFAULTS = Object.freeze({
|
|
|
138
162
|
jitterRatio: 0.2,
|
|
139
163
|
/** Retry even when the serving adapter published no retryPolicy at all. */
|
|
140
164
|
retryWithoutProviderPolicy: true,
|
|
141
|
-
/** Codes
|
|
165
|
+
/** Codes this layer will retry; everything else is left to the loop. */
|
|
166
|
+
retryCodes: DEFAULT_RETRY_CODES,
|
|
167
|
+
/** Retry any code that is not always-terminal and not excluded. */
|
|
168
|
+
retryAllCodes: false,
|
|
169
|
+
/** Hard ceiling on retries inside one turn, whatever the codes say. */
|
|
170
|
+
maxRetriesPerTurn: 40,
|
|
171
|
+
/** Codes that stay terminal even when they are in `retryCodes`. */
|
|
142
172
|
excludeCodes: Object.freeze([]),
|
|
143
173
|
/** Append `llm/retry` + `llm/retry-started` so the chat surface shows the chain. */
|
|
144
174
|
recordSessionEvents: true,
|
|
@@ -155,6 +185,9 @@ const KNOWN_KEYS = Object.freeze([
|
|
|
155
185
|
"maxDelayMs",
|
|
156
186
|
"jitterRatio",
|
|
157
187
|
"retryWithoutProviderPolicy",
|
|
188
|
+
"retryCodes",
|
|
189
|
+
"retryAllCodes",
|
|
190
|
+
"maxRetriesPerTurn",
|
|
158
191
|
"excludeCodes",
|
|
159
192
|
"recordSessionEvents",
|
|
160
193
|
"log",
|
|
@@ -196,7 +229,10 @@ async function loadConfigSchema() {
|
|
|
196
229
|
// `normalize()` instead, which is stricter than a hint either way.
|
|
197
230
|
jitterRatio: schemastery.number().min(0).max(1).default(0.2).description("Randomization applied to each delay, as a fraction of the nominal value."),
|
|
198
231
|
retryWithoutProviderPolicy: schemastery.boolean().default(true).description("Retry even when the serving adapter published no retryPolicy."),
|
|
199
|
-
|
|
232
|
+
retryCodes: schemastery.array(schemastery.string()).default([...DEFAULT_RETRY_CODES]).description("Failure codes this plugin retries. Any other code is left to the loop."),
|
|
233
|
+
retryAllCodes: schemastery.boolean().default(false).description("Retry every code that is not always-terminal and not excluded, including unclassified adapter codes."),
|
|
234
|
+
maxRetriesPerTurn: nullableCount.default(DEFAULTS.maxRetriesPerTurn).description("Hard ceiling on retries inside one turn. Empty or null means no turn ceiling."),
|
|
235
|
+
excludeCodes: schemastery.array(schemastery.string()).default([]).description("Failure codes that stay terminal even when listed in retryCodes."),
|
|
200
236
|
recordSessionEvents: schemastery.boolean().default(true).description("Append llm/retry and llm/retry-started session events."),
|
|
201
237
|
log: schemastery.boolean().default(true).description("Write retry decisions to the Harness log."),
|
|
202
238
|
markerFile: nullableString.description("Optional absolute path for a one-line JSON mount probe."),
|
|
@@ -277,6 +313,9 @@ const CHAIN_SCHEMA = await (async () => {
|
|
|
277
313
|
return zod.z.object({
|
|
278
314
|
attempt: zod.z.number().int().nonnegative().default(0),
|
|
279
315
|
retryId: zod.z.string().optional(),
|
|
316
|
+
turn: zod.z.number().int().nonnegative().optional(),
|
|
317
|
+
// Retries already spent inside the current turn, across all its steps.
|
|
318
|
+
turnAttempts: zod.z.number().int().nonnegative().default(0),
|
|
280
319
|
});
|
|
281
320
|
})();
|
|
282
321
|
|
|
@@ -353,12 +392,24 @@ function normalize(raw) {
|
|
|
353
392
|
// 0 and negatives mean "no cap", matching how an empty panel field is read.
|
|
354
393
|
const maxAttempts = parsedAttempts === null || parsedAttempts < 1 ? null : parsedAttempts;
|
|
355
394
|
|
|
395
|
+
// 0 and negatives mean "no ceiling", matching how an empty panel field is read.
|
|
396
|
+
const rawTurnLimit = source.maxRetriesPerTurn;
|
|
397
|
+
const parsedTurnLimit = rawTurnLimit === null || rawTurnLimit === undefined
|
|
398
|
+
? DEFAULTS.maxRetriesPerTurn
|
|
399
|
+
: Math.trunc(finite(rawTurnLimit, 0));
|
|
400
|
+
const maxRetriesPerTurn = parsedTurnLimit < 1 ? null : parsedTurnLimit;
|
|
401
|
+
|
|
356
402
|
// Canonical on both sides, so `Auth`, `auth` and ` AUTH ` are one exclusion.
|
|
357
403
|
const legacyExcludes = Array.isArray(source.terminalCodes) ? source.terminalCodes : [];
|
|
358
404
|
const excludeCodes = canonicalCodeSet([
|
|
359
405
|
...legacyExcludes,
|
|
360
406
|
...(Array.isArray(source.excludeCodes) ? source.excludeCodes : []),
|
|
361
407
|
]);
|
|
408
|
+
const retryCodes = canonicalCodeSet(
|
|
409
|
+
Array.isArray(source.retryCodes) && source.retryCodes.length > 0
|
|
410
|
+
? source.retryCodes
|
|
411
|
+
: DEFAULT_RETRY_CODES,
|
|
412
|
+
);
|
|
362
413
|
|
|
363
414
|
const retired = Object.keys(RETIRED_KEYS).filter((key) => Object.hasOwn(source, key));
|
|
364
415
|
|
|
@@ -369,6 +420,9 @@ function normalize(raw) {
|
|
|
369
420
|
maxDelayMs,
|
|
370
421
|
jitterRatio,
|
|
371
422
|
retryWithoutProviderPolicy: source.retryWithoutProviderPolicy !== false,
|
|
423
|
+
retryAllCodes: source.retryAllCodes === true,
|
|
424
|
+
retryCodes,
|
|
425
|
+
maxRetriesPerTurn,
|
|
372
426
|
excludeCodes,
|
|
373
427
|
recordSessionEvents: source.recordSessionEvents !== false,
|
|
374
428
|
log: source.log !== false,
|
|
@@ -390,6 +444,9 @@ function configPayload(cfg) {
|
|
|
390
444
|
maxDelayMs: cfg.maxDelayMs,
|
|
391
445
|
jitterRatio: cfg.jitterRatio,
|
|
392
446
|
retryWithoutProviderPolicy: cfg.retryWithoutProviderPolicy,
|
|
447
|
+
retryCodes: [...cfg.retryCodes],
|
|
448
|
+
retryAllCodes: cfg.retryAllCodes,
|
|
449
|
+
maxRetriesPerTurn: cfg.maxRetriesPerTurn,
|
|
393
450
|
excludeCodes: [...cfg.excludeCodes],
|
|
394
451
|
recordSessionEvents: cfg.recordSessionEvents,
|
|
395
452
|
log: cfg.log,
|
|
@@ -782,8 +839,13 @@ export function apply(ctx, config) {
|
|
|
782
839
|
/** One retry series; see `CHAIN_SCHEMA` for why it is a real zod schema. */
|
|
783
840
|
|
|
784
841
|
/** Boundaries that end the request series owning a ladder. */
|
|
785
|
-
|
|
786
|
-
|
|
842
|
+
// Boundaries that end the *step* ladder. A turn boundary resets the turn
|
|
843
|
+
// budget too; a step boundary must not, or the ceiling would be per-step and
|
|
844
|
+
// a turn could still loop across steps.
|
|
845
|
+
const STEP_RESETS = new Set(["step/start", "assistant/message"]);
|
|
846
|
+
const TURN_RESETS = new Set(["turn/start", "turn/end"]);
|
|
847
|
+
const CHAIN_RESETS = new Set([...STEP_RESETS, ...TURN_RESETS]);
|
|
848
|
+
const EMPTY_CHAIN = Object.freeze({ attempt: 0, retryId: undefined, turn: undefined, turnAttempts: 0 });
|
|
787
849
|
|
|
788
850
|
let chainsRegistered = false;
|
|
789
851
|
/**
|
|
@@ -796,13 +858,18 @@ export function apply(ctx, config) {
|
|
|
796
858
|
try {
|
|
797
859
|
const registered = ctx.sessionProjections.register({
|
|
798
860
|
key: CHAIN_KEY,
|
|
799
|
-
stateVersion:
|
|
861
|
+
stateVersion: 2,
|
|
800
862
|
stateSchema: CHAIN_SCHEMA,
|
|
801
863
|
init: () => EMPTY_CHAIN,
|
|
802
864
|
apply: (state, event) => {
|
|
803
865
|
const type = event?.type;
|
|
804
866
|
if (typeof type !== "string") return state;
|
|
805
|
-
if (
|
|
867
|
+
if (TURN_RESETS.has(type)) {
|
|
868
|
+
return state.attempt === 0 && (state.turnAttempts ?? 0) === 0 ? state : EMPTY_CHAIN;
|
|
869
|
+
}
|
|
870
|
+
if (STEP_RESETS.has(type)) {
|
|
871
|
+
return state.attempt === 0 ? state : { ...state, attempt: 0 };
|
|
872
|
+
}
|
|
806
873
|
if (type !== "llm/retry") return state;
|
|
807
874
|
// Only this plugin's own retries own this ladder.
|
|
808
875
|
if (event.data?.policyKey !== POLICY_KEY) return state;
|
|
@@ -810,7 +877,14 @@ export function apply(ctx, config) {
|
|
|
810
877
|
if (attempt === 0) return state;
|
|
811
878
|
const retryId = typeof event.data?.retryId === "string" ? event.data.retryId : state.retryId;
|
|
812
879
|
if (attempt === state.attempt && retryId === state.retryId) return state;
|
|
813
|
-
|
|
880
|
+
const turn = Number.isSafeInteger(event.data?.turn) ? event.data.turn : state.turn;
|
|
881
|
+
const sameTurn = turn !== undefined && turn === state.turn;
|
|
882
|
+
return {
|
|
883
|
+
attempt,
|
|
884
|
+
retryId,
|
|
885
|
+
turn,
|
|
886
|
+
turnAttempts: (sameTurn ? (state.turnAttempts ?? 0) : 0) + 1,
|
|
887
|
+
};
|
|
814
888
|
},
|
|
815
889
|
});
|
|
816
890
|
chainsRegistered = true;
|
|
@@ -860,7 +934,13 @@ export function apply(ctx, config) {
|
|
|
860
934
|
const type = event?.type;
|
|
861
935
|
if (typeof type !== "string" || !CHAIN_RESETS.has(type)) return;
|
|
862
936
|
const id = session?.id ?? "unidentified-session";
|
|
863
|
-
fallbackChains.
|
|
937
|
+
const current = fallbackChains.get(id);
|
|
938
|
+
if (current === undefined) return;
|
|
939
|
+
// A step boundary ends the step ladder but must keep the turn budget; a
|
|
940
|
+
// turn boundary ends both.
|
|
941
|
+
fallbackChains.set(id, TURN_RESETS.has(type)
|
|
942
|
+
? EMPTY_CHAIN
|
|
943
|
+
: { ...current, attempt: 0 });
|
|
864
944
|
});
|
|
865
945
|
}
|
|
866
946
|
|
|
@@ -907,9 +987,13 @@ export function apply(ctx, config) {
|
|
|
907
987
|
* Record that one attempt is being scheduled.
|
|
908
988
|
* @param session - the owning session.
|
|
909
989
|
* @param attempt - the attempt number being scheduled.
|
|
990
|
+
* @param retryId - the identity shared by the retry's two durable events.
|
|
991
|
+
* @param turn - the turn this attempt belongs to, for the turn ceiling.
|
|
992
|
+
* @param session - the owning session.
|
|
993
|
+
* @param attempt - the attempt number being scheduled.
|
|
910
994
|
* @param retryId - the series identity.
|
|
911
995
|
*/
|
|
912
|
-
function rememberChain(session, attempt, retryId) {
|
|
996
|
+
function rememberChain(session, attempt, retryId, turn) {
|
|
913
997
|
if (chainsRegistered) {
|
|
914
998
|
// The projection folds the `llm/retry` event appended by `backoff()`;
|
|
915
999
|
// nothing else may write here.
|
|
@@ -920,7 +1004,14 @@ export function apply(ctx, config) {
|
|
|
920
1004
|
const oldest = fallbackChains.keys().next().value;
|
|
921
1005
|
if (oldest !== undefined) fallbackChains.delete(oldest);
|
|
922
1006
|
}
|
|
923
|
-
fallbackChains.
|
|
1007
|
+
const previous = fallbackChains.get(id);
|
|
1008
|
+
const sameTurn = previous?.turn !== undefined && previous.turn === turn;
|
|
1009
|
+
fallbackChains.set(id, {
|
|
1010
|
+
attempt,
|
|
1011
|
+
retryId,
|
|
1012
|
+
turn,
|
|
1013
|
+
turnAttempts: (sameTurn ? previous.turnAttempts ?? 0 : 0) + 1,
|
|
1014
|
+
});
|
|
924
1015
|
}
|
|
925
1016
|
|
|
926
1017
|
// ---------------------------------------------------------------------
|
|
@@ -960,6 +1051,9 @@ export function apply(ctx, config) {
|
|
|
960
1051
|
// asking an operator to spell a constant from memory. Providers still
|
|
961
1052
|
// surface their own codes, which is why the input stays free text too.
|
|
962
1053
|
knownFailureCodes: [...KNOWN_FAILURE_CODES],
|
|
1054
|
+
// The codes the Harness itself declares retryable: the panel shows these as
|
|
1055
|
+
// the default scope and offers them in the picker.
|
|
1056
|
+
harnessRetryableCodes: [...DEFAULT_RETRY_CODES],
|
|
963
1057
|
// Rules that apply regardless of configuration, published so the panel can
|
|
964
1058
|
// explain them rather than looking like it silently ignored a setting.
|
|
965
1059
|
alwaysTerminalCodes: [...ALWAYS_TERMINAL_CODES],
|
|
@@ -1080,6 +1174,20 @@ export function apply(ctx, config) {
|
|
|
1080
1174
|
return downstream;
|
|
1081
1175
|
}
|
|
1082
1176
|
|
|
1177
|
+
// Retry scope. A code outside the allow-list is the Harness's own signal that
|
|
1178
|
+
// re-sending will not help, or an adapter code nobody classified — and an
|
|
1179
|
+
// unclassified code is exactly what produced the endless turns under 2.x,
|
|
1180
|
+
// where everything except a cancellation was retried.
|
|
1181
|
+
if (!cfg.retryAllCodes && !cfg.retryCodes.has(code)) {
|
|
1182
|
+
bump(stats.terminalSeen, code);
|
|
1183
|
+
log(
|
|
1184
|
+
"info",
|
|
1185
|
+
`failure "${code}" is outside the retry set; left to the loop `
|
|
1186
|
+
+ `(retryCodes: ${[...cfg.retryCodes].join(", ") || "none"})`,
|
|
1187
|
+
);
|
|
1188
|
+
return downstream;
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1083
1191
|
// The ladder is keyed by the session, never by the agent object: the session
|
|
1084
1192
|
// is created before the agent that owns it, and the agent itself carries no
|
|
1085
1193
|
// `id`. `chainFor` falls back to a single shared key when a session is
|
|
@@ -1093,6 +1201,22 @@ export function apply(ctx, config) {
|
|
|
1093
1201
|
return downstream;
|
|
1094
1202
|
}
|
|
1095
1203
|
|
|
1204
|
+
// Turn ceiling. The step ladder resets every step, so a fault that survives
|
|
1205
|
+
// each step would otherwise restart at attempt 1 forever and the turn would
|
|
1206
|
+
// never end. This is the backstop that makes that impossible even for a code
|
|
1207
|
+
// the allow-list accepts.
|
|
1208
|
+
if (cfg.maxRetriesPerTurn !== null) {
|
|
1209
|
+
const spentThisTurn = chain.turn !== undefined && chain.turn === turn ? chain.turnAttempts ?? 0 : 0;
|
|
1210
|
+
if (spentThisTurn >= cfg.maxRetriesPerTurn) {
|
|
1211
|
+
bump(stats.terminalSeen, code);
|
|
1212
|
+
log(
|
|
1213
|
+
"warn",
|
|
1214
|
+
`turn retry ceiling ${cfg.maxRetriesPerTurn} reached for "${code}"; left to the loop`,
|
|
1215
|
+
);
|
|
1216
|
+
return downstream;
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1096
1220
|
// One identity per retry series, so the two durable events pair up.
|
|
1097
1221
|
const retryId = chain.retryId ?? newRetryId();
|
|
1098
1222
|
|
|
@@ -1104,7 +1228,7 @@ export function apply(ctx, config) {
|
|
|
1104
1228
|
|
|
1105
1229
|
// Record before waiting: a wait that is cancelled must still be visible, and
|
|
1106
1230
|
// when the ladder is projection-folded this append IS the state update.
|
|
1107
|
-
rememberChain(session, attempt, retryId);
|
|
1231
|
+
rememberChain(session, attempt, retryId, turn);
|
|
1108
1232
|
if (!(await backoff(agent, turn, step, provider, failure, attempt, retryId, delayMs, session, signal))) {
|
|
1109
1233
|
return downstream;
|
|
1110
1234
|
}
|