agent-nuvira 3.5.0 → 3.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -13
- package/dist/agent-sdk/src/scaffold.d.ts +1 -1
- package/dist/agent-sdk/src/scaffold.js +1 -1
- package/dist/cli/chat.d.ts +159 -6
- package/dist/cli/chat.d.ts.map +1 -1
- package/dist/cli/chat.js +583 -35
- package/dist/cli/chat.js.map +1 -1
- package/dist/cli/duties.d.ts +11 -0
- package/dist/cli/duties.d.ts.map +1 -1
- package/dist/cli/duties.js +31 -0
- package/dist/cli/duties.js.map +1 -1
- package/dist/cli/edit.d.ts.map +1 -1
- package/dist/cli/edit.js +3 -2
- package/dist/cli/edit.js.map +1 -1
- package/dist/cli/execute.d.ts.map +1 -1
- package/dist/cli/execute.js +10 -3
- package/dist/cli/execute.js.map +1 -1
- package/dist/cli/loop-executor.d.ts +16 -0
- package/dist/cli/loop-executor.d.ts.map +1 -1
- package/dist/cli/loop-executor.js +9 -0
- package/dist/cli/loop-executor.js.map +1 -1
- package/dist/cli/models.d.ts.map +1 -1
- package/dist/cli/models.js +55 -0
- package/dist/cli/models.js.map +1 -1
- package/dist/cli/plan.d.ts.map +1 -1
- package/dist/cli/plan.js +18 -6
- package/dist/cli/plan.js.map +1 -1
- package/dist/cli/run.d.ts.map +1 -1
- package/dist/cli/run.js +23 -8
- package/dist/cli/run.js.map +1 -1
- package/dist/cli/ui/answer-view.d.ts +38 -0
- package/dist/cli/ui/answer-view.d.ts.map +1 -0
- package/dist/cli/ui/answer-view.js +72 -0
- package/dist/cli/ui/answer-view.js.map +1 -0
- package/dist/cli/ui/cards.d.ts +61 -0
- package/dist/cli/ui/cards.d.ts.map +1 -0
- package/dist/cli/ui/cards.js +141 -0
- package/dist/cli/ui/cards.js.map +1 -0
- package/dist/cli/ui/highlight.d.ts +32 -0
- package/dist/cli/ui/highlight.d.ts.map +1 -0
- package/dist/cli/ui/highlight.js +261 -0
- package/dist/cli/ui/highlight.js.map +1 -0
- package/dist/cli/ui/index.d.ts +21 -0
- package/dist/cli/ui/index.d.ts.map +1 -0
- package/dist/cli/ui/index.js +14 -0
- package/dist/cli/ui/index.js.map +1 -0
- package/dist/cli/ui/markdown.d.ts +90 -0
- package/dist/cli/ui/markdown.d.ts.map +1 -0
- package/dist/cli/ui/markdown.js +292 -0
- package/dist/cli/ui/markdown.js.map +1 -0
- package/dist/cli/ui/presenter.d.ts +147 -0
- package/dist/cli/ui/presenter.d.ts.map +1 -0
- package/dist/cli/ui/presenter.js +313 -0
- package/dist/cli/ui/presenter.js.map +1 -0
- package/dist/cli/ui/render.d.ts +52 -0
- package/dist/cli/ui/render.d.ts.map +1 -0
- package/dist/cli/ui/render.js +384 -0
- package/dist/cli/ui/render.js.map +1 -0
- package/dist/cli/ui/theme.d.ts +36 -0
- package/dist/cli/ui/theme.d.ts.map +1 -0
- package/dist/cli/ui/theme.js +92 -0
- package/dist/cli/ui/theme.js.map +1 -0
- package/dist/inference/model-entitlement.d.ts +10 -0
- package/dist/inference/model-entitlement.d.ts.map +1 -1
- package/dist/inference/model-entitlement.js +1 -1
- package/dist/inference/model-entitlement.js.map +1 -1
- package/dist/learning/auto-router.d.ts +38 -1
- package/dist/learning/auto-router.d.ts.map +1 -1
- package/dist/learning/auto-router.js +139 -8
- package/dist/learning/auto-router.js.map +1 -1
- package/dist/learning/duty-lease.d.ts +163 -0
- package/dist/learning/duty-lease.d.ts.map +1 -0
- package/dist/learning/duty-lease.js +301 -0
- package/dist/learning/duty-lease.js.map +1 -0
- package/dist/learning/model-first-router.d.ts +14 -0
- package/dist/learning/model-first-router.d.ts.map +1 -1
- package/dist/learning/model-first-router.js +54 -17
- package/dist/learning/model-first-router.js.map +1 -1
- package/dist/learning/model-warmup.d.ts +92 -0
- package/dist/learning/model-warmup.d.ts.map +1 -1
- package/dist/learning/model-warmup.js +137 -2
- package/dist/learning/model-warmup.js.map +1 -1
- package/dist/learning/reasoning-trace.d.ts +21 -1
- package/dist/learning/reasoning-trace.d.ts.map +1 -1
- package/dist/learning/reasoning-trace.js.map +1 -1
- package/dist/learning/resilient-call.d.ts.map +1 -1
- package/dist/learning/resilient-call.js +70 -4
- package/dist/learning/resilient-call.js.map +1 -1
- package/dist/learning/retry-patience.d.ts +139 -0
- package/dist/learning/retry-patience.d.ts.map +1 -0
- package/dist/learning/retry-patience.js +169 -0
- package/dist/learning/retry-patience.js.map +1 -0
- package/dist/tools/process-liveness.d.ts +227 -0
- package/dist/tools/process-liveness.d.ts.map +1 -0
- package/dist/tools/process-liveness.js +486 -0
- package/dist/tools/process-liveness.js.map +1 -0
- package/dist/tools/registry.d.ts.map +1 -1
- package/dist/tools/registry.js +4 -2
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/run-registry.d.ts +282 -0
- package/dist/tools/run-registry.d.ts.map +1 -0
- package/dist/tools/run-registry.js +544 -0
- package/dist/tools/run-registry.js.map +1 -0
- package/dist/tools/run-terminal.d.ts +100 -2
- package/dist/tools/run-terminal.d.ts.map +1 -1
- package/dist/tools/run-terminal.js +841 -67
- package/dist/tools/run-terminal.js.map +1 -1
- package/dist/tools/tool-loop.d.ts +87 -3
- package/dist/tools/tool-loop.d.ts.map +1 -1
- package/dist/tools/tool-loop.js +375 -6
- package/dist/tools/tool-loop.js.map +1 -1
- package/dist/web-dashboard/chat-console.d.ts +25 -0
- package/dist/web-dashboard/chat-console.d.ts.map +1 -1
- package/dist/web-dashboard/chat-console.js +16 -1
- package/dist/web-dashboard/chat-console.js.map +1 -1
- package/dist/web-dashboard/server.d.ts.map +1 -1
- package/dist/web-dashboard/server.js +7 -1
- package/dist/web-dashboard/server.js.map +1 -1
- package/dist/web-dashboard/src/types.d.ts +32 -1
- package/dist/web-dashboard/src/types.d.ts.map +1 -1
- package/dist/web-dashboard/task-registry.d.ts +98 -0
- package/dist/web-dashboard/task-registry.d.ts.map +1 -0
- package/dist/web-dashboard/task-registry.js +159 -0
- package/dist/web-dashboard/task-registry.js.map +1 -0
- package/dist/web-dashboard/task-runner.d.ts +155 -5
- package/dist/web-dashboard/task-runner.d.ts.map +1 -1
- package/dist/web-dashboard/task-runner.js +505 -14
- package/dist/web-dashboard/task-runner.js.map +1 -1
- package/package.json +4 -1
- package/src/web-dashboard/public/assets/index-DycGBWwy.js +236 -0
- package/src/web-dashboard/public/assets/index-DycGBWwy.js.map +1 -0
- package/src/web-dashboard/public/index.html +1 -1
- package/src/web-dashboard/public/assets/index-yky0dw5e.js +0 -236
- package/src/web-dashboard/public/assets/index-yky0dw5e.js.map +0 -1
package/dist/cli/chat.js
CHANGED
|
@@ -16,6 +16,7 @@ import { decisionsRecordingEnabled, recallDecisionBlock } from '../learning/deci
|
|
|
16
16
|
import { getMemoryManager } from '../memory/manager.js';
|
|
17
17
|
import { logger } from '../utils/logger.js';
|
|
18
18
|
import { printOrchestrationResult } from './execute.js';
|
|
19
|
+
import { AnswerStream, attachShellCards, gitDiffCard, presentAnswer, shouldUseRichUi, toolCallCard, } from './ui/index.js';
|
|
19
20
|
// WS5 (#27) — isolation (a git worktree around the turn) and resume (replaying
|
|
20
21
|
// recorded steps instead of re-paying for them). See the module headers for why
|
|
21
22
|
// the worktree is created HERE, around the whole turn, and why a replay is
|
|
@@ -29,7 +30,7 @@ import { recordActionFailure, } from '../learning/failure-bookkeeping.js';
|
|
|
29
30
|
import { getRouterBandit } from '../learning/router-bandit.js';
|
|
30
31
|
import { UnusableModelResponseError } from '../learning/response-usability.js';
|
|
31
32
|
import { resolveThreadBudgetChars } from '../learning/context-budget.js';
|
|
32
|
-
import { getAutoRouter, isAgentAutoRoute, isAutoModel, isAutoProvider, governanceVerdict, adminBudgetVerdict } from '../learning/auto-router.js';
|
|
33
|
+
import { getAutoRouter, isAgentAutoRoute, isAutoModel, isAutoProvider, governanceVerdict, adminBudgetVerdict, isAgenticTask } from '../learning/auto-router.js';
|
|
33
34
|
import { continuationSoftwareText } from '../learning/continuation-intent.js';
|
|
34
35
|
import { TaskModelContinuity, taskSignature } from '../learning/task-model-continuity.js';
|
|
35
36
|
import { describeFailureKind, renderExhaustionReport, } from '../learning/exhaustion-report.js';
|
|
@@ -62,8 +63,9 @@ import { assertAgenticRoute, setWeakModelConsent, resolveWeakModelPolicy, weakRo
|
|
|
62
63
|
import { resolvePromptBudget, measurePromptBudget, formatPromptBudgetBreakdown } from '../learning/prompt-budget.js';
|
|
63
64
|
import { buildTurnReport, formatTurnReport } from '../learning/turn-report.js';
|
|
64
65
|
import { turnOutcomeObservation } from '../learning/outcome-observation.js';
|
|
65
|
-
import { resolveAdapterDefault, hasCredentials, preferredModelsFor } from '../learning/model-selection.js';
|
|
66
|
+
import { resolveAdapterDefault, hasCredentials, preferredModelsFor, rankAvailableProviders } from '../learning/model-selection.js';
|
|
66
67
|
import { sameClassFallback, capabilityClassOf } from '../learning/model-capability.js';
|
|
68
|
+
import { planBlipPatience, EMPTY_PATIENCE_MS, TRANSIENT_BLIP_DELAYS_MS, } from '../learning/retry-patience.js';
|
|
67
69
|
import { buildLoopProjectContext } from '../tools/loop-project-context.js';
|
|
68
70
|
import { sweepTransientFailures, collectionRevivalStore } from '../learning/provider-revival.js';
|
|
69
71
|
import { analyzeComplexity } from '../learning/hybrid-router.js';
|
|
@@ -261,12 +263,208 @@ export async function runDeveloperMode(goal, configManager, options) {
|
|
|
261
263
|
* caller, never to skip the loop).
|
|
262
264
|
*/
|
|
263
265
|
/**
|
|
264
|
-
* Backoff schedule for a SAME-provider retry on a
|
|
265
|
-
*
|
|
266
|
-
*
|
|
267
|
-
*
|
|
266
|
+
* Backoff schedule for a SAME-provider retry on a TRANSIENT failure
|
|
267
|
+
* (server/network/timeout): dense early, sparse late — attempts at ≈1s, 3s, 7s,
|
|
268
|
+
* 15s, 30s, 60s.
|
|
269
|
+
*
|
|
270
|
+
* WHY THE SHAPE CHANGED (it was `[5s, 10s, 20s, 30s]`, front-loaded). A fixed
|
|
271
|
+
* front-loaded schedule is wrong in BOTH directions at once:
|
|
272
|
+
*
|
|
273
|
+
* - A TWO-SECOND BLINK WAS NOT CAUGHT. A cloud endpoint routinely returns 200,
|
|
274
|
+
* then a couple of empty/5xx answers for a handful of seconds, then serves
|
|
275
|
+
* normally again (measured repeatedly on `deepseek`). The first retry sat at
|
|
276
|
+
* t=5s, so the wait outlived the blink AND, because exhausting this schedule
|
|
277
|
+
* is what starts the failover walk, a healthy strong model was declared dead
|
|
278
|
+
* and the turn folded onto a weaker one. A retry at t≈1s catches the blink
|
|
279
|
+
* outright.
|
|
280
|
+
* - THE TOTAL STILL HAS TO RIDE OUT A REAL SPIKE. A capacity spike, an
|
|
281
|
+
* upstream batch job or a rolling restart lasts tens of seconds, so the late
|
|
282
|
+
* delays stay sparse and the total stays ~60s. Nothing that used to be
|
|
283
|
+
* ridden out is given up on; the wait is just spent where the evidence is.
|
|
284
|
+
*
|
|
285
|
+
* THIS IS THE NO-EVIDENCE DEFAULT, NOT THE WHOLE POLICY. The schedule used per
|
|
286
|
+
* call comes from {@link planBlipPatience} — which also decides whether walking
|
|
287
|
+
* away is a remedy at all, from the pair's alternatives, its track record, and
|
|
288
|
+
* whether it already failed an earlier step of this turn. This constant is what
|
|
289
|
+
* every caller with no evidence gets, and it is what the `models`/parity
|
|
290
|
+
* harnesses see.
|
|
291
|
+
*
|
|
292
|
+
* Every sleep is CANCELLABLE (see `abortableSleep`), so an Esc during a 30s wait
|
|
293
|
+
* stops the turn immediately instead of after it.
|
|
294
|
+
*/
|
|
295
|
+
export const TRANSIENT_RETRY_DELAYS_MS = TRANSIENT_BLIP_DELAYS_MS;
|
|
296
|
+
/**
|
|
297
|
+
* Resolve the backoff schedule actually used (env override → default).
|
|
298
|
+
*
|
|
299
|
+
* `NUVIRA_TRANSIENT_RETRY_DELAYS_MS` takes a comma-separated list of ms
|
|
300
|
+
* (`"5000,10000"`), so how long a foreground turn waits on a flapping provider
|
|
301
|
+
* is a CONFIGURATION decision rather than a property of the build — the same
|
|
302
|
+
* rule the warmup sweep's bounds follow. A headless gateway, a CI contract
|
|
303
|
+
* test or a user on a metered connection can dial it down (or to `0`) without a
|
|
304
|
+
* source edit and a rebuild; nonsense is ignored and the default stands.
|
|
305
|
+
*
|
|
306
|
+
* Read PER CALL, not captured at import, so a test observes the value in force
|
|
307
|
+
* at that moment.
|
|
308
|
+
*/
|
|
309
|
+
export function transientRetryDelaysMs(env = process.env) {
|
|
310
|
+
const raw = env.NUVIRA_TRANSIENT_RETRY_DELAYS_MS;
|
|
311
|
+
if (raw === undefined || raw.trim() === '')
|
|
312
|
+
return TRANSIENT_RETRY_DELAYS_MS;
|
|
313
|
+
const parsed = raw
|
|
314
|
+
.split(',')
|
|
315
|
+
.map((part) => Number(part.trim()))
|
|
316
|
+
.filter((n) => Number.isFinite(n) && n >= 0);
|
|
317
|
+
return parsed.length > 0 ? parsed : TRANSIENT_RETRY_DELAYS_MS;
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Is the transient schedule overridden by the environment right now?
|
|
321
|
+
*
|
|
322
|
+
* The dial is authoritative when set (it is an explicit configuration decision,
|
|
323
|
+
* and it is what makes attempt COUNTS deterministic in the parity/E2E
|
|
324
|
+
* contracts), so callers ask this BEFORE substituting an evidence-derived
|
|
325
|
+
* schedule — otherwise a test would silently get the learned schedule and its
|
|
326
|
+
* exact call-count assertions would be about a different policy than the one it
|
|
327
|
+
* dialed.
|
|
328
|
+
*
|
|
329
|
+
* A set-but-nonsense value does NOT count as an override: `transientRetryDelaysMs`
|
|
330
|
+
* ignores it and falls back to the shipped schedule, so the evidence-derived
|
|
331
|
+
* schedule would be overridden by nothing.
|
|
332
|
+
*/
|
|
333
|
+
export function transientRetryScheduleOverridden(env = process.env) {
|
|
334
|
+
const raw = env.NUVIRA_TRANSIENT_RETRY_DELAYS_MS;
|
|
335
|
+
if (raw === undefined || raw.trim() === '')
|
|
336
|
+
return false;
|
|
337
|
+
return raw.split(',').some((part) => {
|
|
338
|
+
const n = Number(part.trim());
|
|
339
|
+
return Number.isFinite(n) && n >= 0;
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Bounded extra same-model patience for an `empty-response` — the "final verse"
|
|
344
|
+
* case, granted ONLY to a pair that has already delivered real work this turn.
|
|
345
|
+
*
|
|
346
|
+
* THE TWO CASES LOOK IDENTICAL TO THE CLASSIFIER AND MUST NOT BE TREATED ALIKE:
|
|
347
|
+
*
|
|
348
|
+
* - a BROKEN pair answers nothing, forever. The codebase already measured the
|
|
349
|
+
* cost of re-asking it: "5 empty responses on one model" in one turn, no
|
|
350
|
+
* failover, which is why `empty-response` was removed from
|
|
351
|
+
* `TRANSIENT_RETRY_TYPES` and why the handoff/walk is what remedies it.
|
|
352
|
+
* - a pair that just finished four GOOD steps and then returns nothing is
|
|
353
|
+
* having a blip at the END of real work — the expensive case, because the
|
|
354
|
+
* turn already paid for the context, the tool results and the edits, and a
|
|
355
|
+
* single empty reply would otherwise discard all of it.
|
|
356
|
+
*
|
|
357
|
+
* The discriminator is EVIDENCE, not a heuristic on the reply. Two halves, both
|
|
358
|
+
* required to be absent before we give up on a pair: a completed tool call in
|
|
359
|
+
* THIS turn (`threadHasDeliveredWork`), and the registry's own record for the
|
|
360
|
+
* pair — verified by TELEMETRY or a spot-check, with a low error rate
|
|
361
|
+
* (`hasProvenTrackRecord` in `retry-patience.ts`, which is where the rule now
|
|
362
|
+
* lives so it cannot drift). So a pair that answers nothing from the start with
|
|
363
|
+
* no track record still fails over immediately, a proven pair's FIRST empty
|
|
364
|
+
* reply gets these two short waits, and a proven pair whose only alternatives
|
|
365
|
+
* would DE-ESCALATE gets the longer
|
|
366
|
+
* `EMPTY_PATIENCE_WAIT_BEATS_FOLDING_MS` schedule rather than folding down a
|
|
367
|
+
* capability band. Bounded by design, and always finite.
|
|
368
|
+
*
|
|
369
|
+
* The value is re-exported from `retry-patience.ts`: ONE table for the whole
|
|
370
|
+
* blip policy, so the CLI and any other caller cannot drift apart.
|
|
371
|
+
*/
|
|
372
|
+
export const EMPTY_RESPONSE_PATIENCE_MS = EMPTY_PATIENCE_MS;
|
|
373
|
+
/**
|
|
374
|
+
* Sleep that a CANCELLED turn interrupts.
|
|
375
|
+
*
|
|
376
|
+
* The old retry slept with a bare `setTimeout`, so an Esc during a backoff was
|
|
377
|
+
* only honored when the timer happened to fire — with a 30s step in the
|
|
378
|
+
* schedule that is a 30s delay before the turn admits it was cancelled. This
|
|
379
|
+
* resolves the moment the signal aborts, so the caller's abort checks see the
|
|
380
|
+
* cancellation immediately.
|
|
381
|
+
*/
|
|
382
|
+
function abortableSleep(ms, signal) {
|
|
383
|
+
if (!signal)
|
|
384
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
385
|
+
if (signal.aborted)
|
|
386
|
+
return Promise.resolve();
|
|
387
|
+
return new Promise((resolve) => {
|
|
388
|
+
const finish = () => {
|
|
389
|
+
clearTimeout(timer);
|
|
390
|
+
signal.removeEventListener('abort', finish);
|
|
391
|
+
resolve();
|
|
392
|
+
};
|
|
393
|
+
const timer = setTimeout(finish, ms);
|
|
394
|
+
signal.addEventListener('abort', finish, { once: true });
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Has this turn already produced real, OBSERVED work on this thread?
|
|
399
|
+
*
|
|
400
|
+
* Structural, not a counter: an assistant step that ASKED for a tool and the
|
|
401
|
+
* tool result that came back for it. That is the cheapest honest evidence that
|
|
402
|
+
* the pair under us has already served this turn — a pair that has not is given
|
|
403
|
+
* no `empty-response` patience at all, so the "ask a broken model five times"
|
|
404
|
+
* loop stays impossible.
|
|
405
|
+
*/
|
|
406
|
+
export function threadHasDeliveredWork(thread) {
|
|
407
|
+
const asked = thread.some((m) => m.role === 'assistant' && (m.toolCalls?.length ?? 0) > 0);
|
|
408
|
+
const answered = thread.some((m) => m.role === 'tool');
|
|
409
|
+
return asked && answered;
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* The pairs this surface could actually switch to right now, best-first, for
|
|
413
|
+
* {@link planBlipPatience}.
|
|
414
|
+
*
|
|
415
|
+
* DELIBERATELY NARROW. Only providers with at least one registry-VERIFIED model
|
|
416
|
+
* count, and the model offered is the one that provider would actually lead
|
|
417
|
+
* with (`preferredModelsFor`'s health order). A provider we cannot name a model
|
|
418
|
+
* for is not evidence of a same-band alternative — treating a cold-start or
|
|
419
|
+
* unverified provider as one would SHORTEN patience on the strength of a guess,
|
|
420
|
+
* and the whole point of the policy is that waiting is better than an
|
|
421
|
+
* unproven downgrade. Unknown therefore reads as "no known alternative", the
|
|
422
|
+
* conservative direction.
|
|
423
|
+
*
|
|
424
|
+
* Pure read, best-effort: any failure to rank means the same thing (no known
|
|
425
|
+
* alternative), never a thrown turn.
|
|
268
426
|
*/
|
|
269
|
-
|
|
427
|
+
function failoverAlternatives(configManager, excludeProvider) {
|
|
428
|
+
const out = [];
|
|
429
|
+
try {
|
|
430
|
+
for (const ranked of rankAvailableProviders(configManager)) {
|
|
431
|
+
if (ranked.provider === excludeProvider)
|
|
432
|
+
continue;
|
|
433
|
+
const model = ranked.verifiedModels[0];
|
|
434
|
+
if (!model)
|
|
435
|
+
continue;
|
|
436
|
+
out.push({ provider: ranked.provider, model });
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
catch {
|
|
440
|
+
// Best-effort — see the note above: unknown is the conservative reading.
|
|
441
|
+
}
|
|
442
|
+
return out;
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Registry evidence for one pair, shaped for {@link planBlipPatience}.
|
|
446
|
+
* `undefined` when nothing is recorded — an untracked pair earns no patience.
|
|
447
|
+
*
|
|
448
|
+
* The `default` sentinel is not a model id, so it has no evidence by definition
|
|
449
|
+
* (the same reasoning that keeps it out of `preferredModelsFor`).
|
|
450
|
+
*/
|
|
451
|
+
function registryPatienceEvidence(provider, model) {
|
|
452
|
+
if (!model || model === 'default')
|
|
453
|
+
return undefined;
|
|
454
|
+
try {
|
|
455
|
+
const entry = getModelRegistry().getEntry(provider, model);
|
|
456
|
+
if (!entry)
|
|
457
|
+
return undefined;
|
|
458
|
+
return {
|
|
459
|
+
verified: entry.status === 'verified',
|
|
460
|
+
source: entry.source,
|
|
461
|
+
errorRate: entry.errorRate,
|
|
462
|
+
};
|
|
463
|
+
}
|
|
464
|
+
catch {
|
|
465
|
+
return undefined;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
270
468
|
/**
|
|
271
469
|
* How long a PINNED + STRICT pair is given to answer before the user is asked
|
|
272
470
|
* what to do next (ms).
|
|
@@ -333,18 +531,48 @@ const PINNED_DEFINITIVE_ERROR_RE = /(?:does not exist|not found|no such model|in
|
|
|
333
531
|
*
|
|
334
532
|
* Never retries: non-transient classes (auth, rate-limit, model/harness faults),
|
|
335
533
|
* a cancelled turn, or once the schedule is exhausted.
|
|
534
|
+
*
|
|
535
|
+
* `emptyResponseDelaysMs` is the ONE narrowly-scoped exception, and it exists in
|
|
536
|
+
* this helper rather than in `TRANSIENT_RETRY_TYPES` because retryability here
|
|
537
|
+
* depends on the TURN, not only on the error class: pass it only when the pair
|
|
538
|
+
* has a track record this turn (see {@link threadHasDeliveredWork}). Omitted →
|
|
539
|
+
* an empty response is not retried at all, byte for byte the old behaviour.
|
|
540
|
+
*
|
|
541
|
+
* `transientDelaysMs` is the caller's EVIDENCE-DERIVED schedule
|
|
542
|
+
* ({@link planBlipPatience}) for server/network/timeout failures. Omitted → the
|
|
543
|
+
* shipped default / env dial (`transientRetryDelaysMs`), which is what a caller
|
|
544
|
+
* with no evidence (a test, a headless path) gets.
|
|
336
545
|
*/
|
|
337
|
-
export async function generateWithTransientRetry(attempt, signal, onRetry) {
|
|
338
|
-
|
|
546
|
+
export async function generateWithTransientRetry(attempt, signal, onRetry, opts = {}) {
|
|
547
|
+
let transientRetries = 0;
|
|
548
|
+
let emptyRetries = 0;
|
|
549
|
+
const emptySchedule = opts.emptyResponseDelaysMs ?? [];
|
|
550
|
+
const transientSchedule = opts.transientDelaysMs ?? transientRetryDelaysMs();
|
|
551
|
+
for (;;) {
|
|
339
552
|
try {
|
|
340
553
|
return await attempt();
|
|
341
554
|
}
|
|
342
555
|
catch (err) {
|
|
343
|
-
|
|
344
|
-
|
|
556
|
+
if (signal?.aborted)
|
|
557
|
+
throw err;
|
|
558
|
+
// The attempt number a human sees: 1-based, counting every retry so far.
|
|
559
|
+
const attemptNumber = transientRetries + emptyRetries + 1;
|
|
560
|
+
let delay;
|
|
561
|
+
if (isTransientForRetry(err)) {
|
|
562
|
+
delay = transientRetries < transientSchedule.length
|
|
563
|
+
? transientSchedule[transientRetries++]
|
|
564
|
+
: undefined;
|
|
565
|
+
}
|
|
566
|
+
else if (classifyFallbackError(err) === 'empty-response' && emptyRetries < emptySchedule.length) {
|
|
567
|
+
delay = emptySchedule[emptyRetries++];
|
|
568
|
+
}
|
|
569
|
+
if (delay === undefined)
|
|
570
|
+
throw err;
|
|
571
|
+
onRetry?.(attemptNumber, err);
|
|
572
|
+
await abortableSleep(delay, signal);
|
|
573
|
+
// A cancelled turn must not pay for another attempt.
|
|
574
|
+
if (signal?.aborted)
|
|
345
575
|
throw err;
|
|
346
|
-
onRetry?.(i + 1, err);
|
|
347
|
-
await new Promise((resolve) => setTimeout(resolve, TRANSIENT_RETRY_DELAYS_MS[i]));
|
|
348
576
|
}
|
|
349
577
|
}
|
|
350
578
|
}
|
|
@@ -1065,6 +1293,63 @@ export class ChatCommand extends BaseCommand {
|
|
|
1065
1293
|
resume: options?.resume,
|
|
1066
1294
|
};
|
|
1067
1295
|
}
|
|
1296
|
+
/**
|
|
1297
|
+
* Live answer streaming for the interactive REPL.
|
|
1298
|
+
*
|
|
1299
|
+
* Returns the per-turn context overrides that paint the answer as it is
|
|
1300
|
+
* generated, plus a `stop()` the caller MUST invoke once the turn returns.
|
|
1301
|
+
* `stop()` unmounts the live view WITHOUT archiving, because the turn's own
|
|
1302
|
+
* answer-printing path renders the authoritative final `answer.content` —
|
|
1303
|
+
* archiving here as well would print the answer twice. The streamed tokens
|
|
1304
|
+
* are a PREVIEW; the final frame is what stays in the transcript.
|
|
1305
|
+
*
|
|
1306
|
+
* ON A REAL TERMINAL ONLY. A pipe, CI run or `--json-events` gets no Ink view
|
|
1307
|
+
* and no `onToken`, so a later step never streams and the bytes on stdout are
|
|
1308
|
+
* exactly what they were before this existed.
|
|
1309
|
+
*
|
|
1310
|
+
* WHY THE RESET ON `tool:started`. `onToken` is invoked for EVERY model call
|
|
1311
|
+
* in the tool loop, not only for the answer, so a turn that calls a tool
|
|
1312
|
+
* would otherwise append the next step's answer onto the previous step's
|
|
1313
|
+
* preamble. A tool starting is the boundary between steps, so the preview is
|
|
1314
|
+
* cleared there.
|
|
1315
|
+
*/
|
|
1316
|
+
liveAnswer(header, opts = {}) {
|
|
1317
|
+
if (!shouldUseRichUi(opts))
|
|
1318
|
+
return { overrides: {}, stop: () => { }, stream: null };
|
|
1319
|
+
const stream = new AnswerStream({ header, headerColor: 'assistant' }, opts);
|
|
1320
|
+
let begun = false;
|
|
1321
|
+
// Shell commands are emitted on the process-wide bus (utils/shell.ts owns
|
|
1322
|
+
// them), so the turn subscribes for its own duration and detaches on stop().
|
|
1323
|
+
const detachShell = attachShellCards((card) => stream.card(card));
|
|
1324
|
+
return {
|
|
1325
|
+
stream,
|
|
1326
|
+
overrides: {
|
|
1327
|
+
onToken: (token) => {
|
|
1328
|
+
if (!begun) {
|
|
1329
|
+
begun = true;
|
|
1330
|
+
stream.begin();
|
|
1331
|
+
}
|
|
1332
|
+
stream.append(token);
|
|
1333
|
+
},
|
|
1334
|
+
onToolCall: (phase, info) => {
|
|
1335
|
+
if (phase === 'started') {
|
|
1336
|
+
// The step's preamble is not the answer: clear it so the preview
|
|
1337
|
+
// shows the step that is actually producing the reply.
|
|
1338
|
+
if (begun)
|
|
1339
|
+
stream.reset();
|
|
1340
|
+
return;
|
|
1341
|
+
}
|
|
1342
|
+
// Completed call → an archived card (this is what the turn DID).
|
|
1343
|
+
stream.card(toolCallCard({ tool: info.tool, ok: info.ok, error: info.error, result: info.result, durationMs: info.durationMs, args: info.args }));
|
|
1344
|
+
},
|
|
1345
|
+
onGitDiff: (payload) => stream.card(gitDiffCard(payload)),
|
|
1346
|
+
},
|
|
1347
|
+
stop: () => {
|
|
1348
|
+
detachShell();
|
|
1349
|
+
stream.finish({ archive: false });
|
|
1350
|
+
},
|
|
1351
|
+
};
|
|
1352
|
+
}
|
|
1068
1353
|
async execute(prompt, options) {
|
|
1069
1354
|
// Apply the active model state from `nuvira model switch` as defaults
|
|
1070
1355
|
const activeOpts = applyActiveModel({ provider: options?.provider, model: options?.model });
|
|
@@ -1191,10 +1476,14 @@ export class ChatCommand extends BaseCommand {
|
|
|
1191
1476
|
// failed entirely), never as a bypass.
|
|
1192
1477
|
const parsed = parseRequestSync(prompt);
|
|
1193
1478
|
const dispatchDecision = resolvePipelineDispatch(parsed, { dev: options?.dev, text: prompt });
|
|
1479
|
+
// P7 — paint the answer live while it generates (TTY only; a no-op
|
|
1480
|
+
// elsewhere). `stop()` unmounts the preview once the turn returns; the
|
|
1481
|
+
// final frame is printed by the answer path below.
|
|
1482
|
+
const live = this.liveAnswer(`nuvira · ${provider.name}${model ? ` / ${model}` : ''}`);
|
|
1194
1483
|
const answer = await this.runChatAnswer(prompt, [], { type, provider, model }, options || {}, cacheEnabled, { auto: autoMode }, parsed,
|
|
1195
1484
|
// WS5 (#27) — isolation and resume ride on the CLI's own flags. Passed
|
|
1196
1485
|
// per turn: in the REPL each message is its own turn (see the manual).
|
|
1197
|
-
this.ws5Overrides(options));
|
|
1486
|
+
{ ...this.ws5Overrides(options), ...live.overrides }).finally(() => live.stop());
|
|
1198
1487
|
// No-model fallback: the tool loop could not generate a single response
|
|
1199
1488
|
// AND the rules assessed a high-confidence pipeline intent — run the
|
|
1200
1489
|
// pipeline directly (rules decide only when the model is unavailable;
|
|
@@ -1229,7 +1518,8 @@ export class ChatCommand extends BaseCommand {
|
|
|
1229
1518
|
// JSON blob, or the empty ```json fence the fallback transport leaves
|
|
1230
1519
|
// behind) — see stripToolCallArtifacts.
|
|
1231
1520
|
else if (answer.content.trim()) {
|
|
1232
|
-
console.log('
|
|
1521
|
+
console.log('');
|
|
1522
|
+
presentAnswer(stripToolCallArtifacts(answer.content));
|
|
1233
1523
|
}
|
|
1234
1524
|
if (!process.stdin.isTTY) {
|
|
1235
1525
|
await this.renderFollowups(answer.followups ?? [], false);
|
|
@@ -1243,12 +1533,14 @@ export class ChatCommand extends BaseCommand {
|
|
|
1243
1533
|
const picked = await this.renderFollowups(singleAnswer.followups ?? [], true);
|
|
1244
1534
|
if (!picked)
|
|
1245
1535
|
break;
|
|
1536
|
+
const liveNext = this.liveAnswer(`nuvira · ${provider.name}${model ? ` / ${model}` : ''}`);
|
|
1246
1537
|
const next = await this.runChatAnswer(picked, history, { type, provider, model }, options || {}, cacheEnabled, { auto: autoMode }, parseRequestSync(picked),
|
|
1247
1538
|
// P5 — a picked followup continues the previous execution.
|
|
1248
|
-
{ continuation: true, ...this.ws5Overrides(options) });
|
|
1539
|
+
{ continuation: true, ...this.ws5Overrides(options), ...liveNext.overrides }).finally(() => liveNext.stop());
|
|
1249
1540
|
const nextText = stripToolCallArtifacts(next.content);
|
|
1250
1541
|
if (nextText) {
|
|
1251
|
-
console.log('
|
|
1542
|
+
console.log('');
|
|
1543
|
+
presentAnswer(nextText);
|
|
1252
1544
|
// NOTE: no history.push here — runChatAnswer already recorded the
|
|
1253
1545
|
// assistant turn. The old duplicate push gave every subsequent turn
|
|
1254
1546
|
// TWO copies of the previous answer (relevance noise).
|
|
@@ -1338,13 +1630,16 @@ export class ChatCommand extends BaseCommand {
|
|
|
1338
1630
|
const parsed = recordMetricTime('rule.parse.ms', () => parseRequestSync(message));
|
|
1339
1631
|
const dispatchDecision = recordMetricTime('rule.dispatch.ms', () => resolvePipelineDispatch(parsed, { dev: this.devModeAuto, text: message }));
|
|
1340
1632
|
const session = { type, provider, model: effectiveModel };
|
|
1633
|
+
// P7 — live preview for this turn (see `liveAnswer`).
|
|
1634
|
+
const live = this.liveAnswer(`nuvira · ${provider.name}${effectiveModel ? ` / ${effectiveModel}` : ''}`);
|
|
1341
1635
|
const answer = await withLogCorrelation({ sessionId: chatSessionId }, () => recordMetricTime('llm.answer.ms', () => this.runChatAnswer(message, history, session, options || {}, cacheEnabled, { auto: autoMode }, parsed,
|
|
1342
1636
|
// P5 — a picked followup (or a typed one that matches the last
|
|
1343
1637
|
// suggestions) is a continuation, not a fresh independent request.
|
|
1344
1638
|
{
|
|
1345
1639
|
continuation: pickedFollowup || isSuggestedFollowup(message, lastFollowups),
|
|
1346
1640
|
...this.ws5Overrides(options),
|
|
1347
|
-
|
|
1641
|
+
...live.overrides,
|
|
1642
|
+
}))).finally(() => live.stop());
|
|
1348
1643
|
// No-model fallback: the tool loop could not generate a single response
|
|
1349
1644
|
// AND the rules assessed a high-confidence pipeline intent — run the
|
|
1350
1645
|
// pipeline directly (rules decide only when the model is unavailable).
|
|
@@ -1364,7 +1659,8 @@ export class ChatCommand extends BaseCommand {
|
|
|
1364
1659
|
// followup pick becomes the next message). The user asked for the
|
|
1365
1660
|
// content — the menu must never print before it.
|
|
1366
1661
|
if (answer.content.trim()) {
|
|
1367
|
-
console.log('
|
|
1662
|
+
console.log('');
|
|
1663
|
+
presentAnswer(answer.content);
|
|
1368
1664
|
}
|
|
1369
1665
|
lastFollowups = answer.followups ?? [];
|
|
1370
1666
|
const followupPrompt = await this.renderFollowups(lastFollowups, true);
|
|
@@ -2330,6 +2626,37 @@ export class ChatCommand extends BaseCommand {
|
|
|
2330
2626
|
callModel: callModelWithTrace,
|
|
2331
2627
|
// P2 — the empty-response handoff (see `requestModelSwitch` above).
|
|
2332
2628
|
requestModelSwitch,
|
|
2629
|
+
// P5 — the SAME blip policy the thrown-empty path uses, reachable from
|
|
2630
|
+
// the loop's empty-response path too. A provider that answers 200 with
|
|
2631
|
+
// no content never throws, so it never reached
|
|
2632
|
+
// `generateWithTransientRetry` and its empty schedule was computed but
|
|
2633
|
+
// never spent: both live failures (trace-1791906998502-txddq4,
|
|
2634
|
+
// trace-1791907393457-7seewg) handed off after the loop's single retry
|
|
2635
|
+
// and landed on the local last resort. The plan knows whether waiting
|
|
2636
|
+
// is worth it (a proven pair, or alternatives that would de-escalate),
|
|
2637
|
+
// so the loop asks the CALLER rather than inventing a backoff.
|
|
2638
|
+
emptyResponsePatienceMs: ({ emptyStreak, deliveredWorkThisTurn }) => {
|
|
2639
|
+
try {
|
|
2640
|
+
const pair = {
|
|
2641
|
+
provider: session.type,
|
|
2642
|
+
model: this.lastAttempt?.model ?? session.model ?? 'default',
|
|
2643
|
+
};
|
|
2644
|
+
return planBlipPatience({
|
|
2645
|
+
current: pair,
|
|
2646
|
+
alternatives: failoverAlternatives(this.configManager, pair.provider),
|
|
2647
|
+
evidence: registryPatienceEvidence(pair.provider, pair.model),
|
|
2648
|
+
// An entry here means an EARLIER step of this turn already failed
|
|
2649
|
+
// on this pair — the same pair failing twice is a pattern, not a
|
|
2650
|
+
// blip, and patience collapses there.
|
|
2651
|
+
failedEarlierStep: this.turnAttempts.some((a) => a.provider === pair.provider && a.model === pair.model),
|
|
2652
|
+
deliveredWorkThisTurn,
|
|
2653
|
+
}).emptyResponseDelaysMs;
|
|
2654
|
+
}
|
|
2655
|
+
catch {
|
|
2656
|
+
// No evidence → no extra patience (the pre-P5 behaviour).
|
|
2657
|
+
return [];
|
|
2658
|
+
}
|
|
2659
|
+
},
|
|
2333
2660
|
// C5 — `requestModelSwitch` installs the replacement on `session`, so
|
|
2334
2661
|
// re-resolve the thread budget from the session AFTER a handoff: a
|
|
2335
2662
|
// handoff to a smaller window is the case that overflows, and the
|
|
@@ -3287,6 +3614,10 @@ export class ChatCommand extends BaseCommand {
|
|
|
3287
3614
|
// (`answered()`, and the walks that install what they served) are the
|
|
3288
3615
|
// ones that own `session.model`.
|
|
3289
3616
|
this.lastAttempt = { provider: typ, model: effectiveModel };
|
|
3617
|
+
// G5/P6 — and the trace's routing snapshot follows the ATTEMPT, so a step
|
|
3618
|
+
// can never record a pair the request did not go to (see
|
|
3619
|
+
// `noteServedRoute`).
|
|
3620
|
+
this.noteServedRoute(typ, effectiveModel ?? 'default');
|
|
3290
3621
|
const nativeKey = `${typ}|${effectiveModel ?? ''}`;
|
|
3291
3622
|
// Answer-quality resilience: a CONFUSED reply — the model talking about
|
|
3292
3623
|
// the tool contract (e.g. apologizing that "the provided example call
|
|
@@ -3398,13 +3729,54 @@ export class ChatCommand extends BaseCommand {
|
|
|
3398
3729
|
// R2 — this IS the fallback transport, so say so: the same fact the
|
|
3399
3730
|
// subagent child announces, and what makes a chat turn comparable with it.
|
|
3400
3731
|
return answered({ content: text, toolCalls: calls, transport: 'json' });
|
|
3732
|
+
}; // ── HOW PATIENT TO BE, AND WHY ────────────────────────────────────────
|
|
3733
|
+
// A fixed schedule cannot tell a 2-second blink from a dead endpoint, so
|
|
3734
|
+
// the schedule is chosen per pair from evidence the caller ALREADY HAS:
|
|
3735
|
+
// is a same-or-better-band candidate reachable, does this pair have a
|
|
3736
|
+
// track record, and has it already failed an EARLIER step of this turn.
|
|
3737
|
+
// Both ways of getting this wrong are expensive — fail fast and a
|
|
3738
|
+
// blinking strong model is silently replaced by a weaker one nobody
|
|
3739
|
+
// chose; wait blindly and every user of a dead provider pays the full
|
|
3740
|
+
// backoff. The plan's `reason` goes to the debug log, because "why was
|
|
3741
|
+
// that turn slow / why did it change model" is asked after the fact.
|
|
3742
|
+
const patiencePair = {
|
|
3743
|
+
provider: session.type,
|
|
3744
|
+
model: this.lastAttempt?.model ?? session.model ?? 'default',
|
|
3401
3745
|
};
|
|
3746
|
+
const patience = planBlipPatience({
|
|
3747
|
+
current: patiencePair,
|
|
3748
|
+
alternatives: failoverAlternatives(this.configManager, patiencePair.provider),
|
|
3749
|
+
evidence: registryPatienceEvidence(patiencePair.provider, patiencePair.model),
|
|
3750
|
+
// An entry here means an EARLIER step of this same turn already failed
|
|
3751
|
+
// on this pair — the same pair failing twice is a pattern, not a blip,
|
|
3752
|
+
// and patience collapses there whatever the alternatives look like.
|
|
3753
|
+
failedEarlierStep: this.turnAttempts.some((a) => a.provider === patiencePair.provider && a.model === patiencePair.model),
|
|
3754
|
+
deliveredWorkThisTurn: threadHasDeliveredWork(messages),
|
|
3755
|
+
});
|
|
3402
3756
|
try {
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
|
|
3406
|
-
//
|
|
3407
|
-
|
|
3757
|
+
logger.debug(`[patience] ${patience.reason}`);
|
|
3758
|
+
}
|
|
3759
|
+
catch {
|
|
3760
|
+
// Logging must never break a turn.
|
|
3761
|
+
}
|
|
3762
|
+
try {
|
|
3763
|
+
return await generateWithTransientRetry(() => tryGenerate(session.provider, session.type, session.model), abort, (attempt, err) => logger.warn(` ⏳ ${session.provider.name} transient failure (attempt ${attempt}) — retrying shortly: ${err instanceof Error ? err.message.split('\n')[0] : String(err)}`),
|
|
3764
|
+
// Empty-response patience from the plan: granted for a proven pair
|
|
3765
|
+
// (registry track record or real work this turn), LONGER when every
|
|
3766
|
+
// alternative would de-escalate a capability band. A pair with neither
|
|
3767
|
+
// gets none — the measured "5 empty responses on one model" loop stays
|
|
3768
|
+
// impossible — while a pair that just finished real steps never has
|
|
3769
|
+
// them thrown away by a single empty reply.
|
|
3770
|
+
{
|
|
3771
|
+
// The env dial stays authoritative when set (tests, headless
|
|
3772
|
+
// gateways, metered connections) — it is an explicit configuration
|
|
3773
|
+
// decision, and the attempt COUNTS in the parity/E2E contracts are
|
|
3774
|
+
// derived from it.
|
|
3775
|
+
transientDelaysMs: transientRetryScheduleOverridden()
|
|
3776
|
+
? undefined
|
|
3777
|
+
: patience.transientDelaysMs,
|
|
3778
|
+
emptyResponseDelaysMs: patience.emptyResponseDelaysMs,
|
|
3779
|
+
});
|
|
3408
3780
|
}
|
|
3409
3781
|
catch (err) {
|
|
3410
3782
|
// P4 — the model this turn STARTED on failed. Recorded with a human
|
|
@@ -3419,14 +3791,67 @@ export class ChatCommand extends BaseCommand {
|
|
|
3419
3791
|
kind: primaryKind,
|
|
3420
3792
|
reason: describeFailureKind(primaryKind),
|
|
3421
3793
|
});
|
|
3794
|
+
// ── LEARN the failure, don't just survive it ───────────────────────────
|
|
3795
|
+
// A failed attempt was recorded in an in-memory per-turn list only, so
|
|
3796
|
+
// the registry's health score, the quota timeline and the shared
|
|
3797
|
+
// breaker all stayed untouched: the model kept its `verified` badge and
|
|
3798
|
+
// was re-picked on the next turn, and a degraded turn left no on-disk
|
|
3799
|
+
// evidence of WHICH candidate failed or WHY. `recordAutoProviderFailure`
|
|
3800
|
+
// runs the full shared composition (model-scoped session exclusion,
|
|
3801
|
+
// registry write-through, quota-timeline event, shared breaker) exactly
|
|
3802
|
+
// as the empty-response handoff already does.
|
|
3803
|
+
try {
|
|
3804
|
+
this.recordAutoProviderFailure(this.lastAttempt?.provider ?? session.type, err, this.lastAttempt?.model ?? session.model);
|
|
3805
|
+
}
|
|
3806
|
+
catch {
|
|
3807
|
+
// Best-effort — bookkeeping must never break the turn.
|
|
3808
|
+
}
|
|
3422
3809
|
// Auto mode: fail over across the ranked candidates (never stuck).
|
|
3423
3810
|
if (mode.auto) {
|
|
3424
3811
|
const firstType = session.type;
|
|
3425
3812
|
// Observable telemetry, identical to the shared single-shot runner
|
|
3426
3813
|
// (the E2E repair-failover contract): one warn per walk, one
|
|
3427
|
-
// success log per landed candidate
|
|
3428
|
-
|
|
3429
|
-
|
|
3814
|
+
// success log per landed candidate — and now the failure KIND in the
|
|
3815
|
+
// line, so the console says what went wrong instead of only that it did.
|
|
3816
|
+
logger.warn(` ⚠️ ${session.provider.name} failed (${describeFailureKind(primaryKind)}) — trying the next auto candidate...`);
|
|
3817
|
+
/**
|
|
3818
|
+
* ── PAIR-SCOPED WALK: follow what is ROUTABLE, not one model per
|
|
3819
|
+
* provider ────────────────────────────────────────────────────────
|
|
3820
|
+
*
|
|
3821
|
+
* The walk used to track PROVIDERS: the failing provider was put in
|
|
3822
|
+
* `failed` and every re-route passed it as `excludeProviders`, so each
|
|
3823
|
+
* iteration could only ever land on a DIFFERENT provider — one model
|
|
3824
|
+
* tried per provider no matter how many that provider serves. Measured
|
|
3825
|
+
* live (trace-1791906998502-txddq4): a per-model 429 on
|
|
3826
|
+
* `gemini/gemma-4-31b-it` ended gemini's part of the turn, a `model not
|
|
3827
|
+
* found` on one openrouter id ended openrouter's, and the walk reached
|
|
3828
|
+
* the local last resort while ~370 routable pairs were never tried.
|
|
3829
|
+
*
|
|
3830
|
+
* The registry, the quota ledger and the exclusion filter all treat
|
|
3831
|
+
* these failures as PAIR-scoped (`recordActionFailure` parks the pair,
|
|
3832
|
+
* `ledger.parkModel` parks the model, and only
|
|
3833
|
+
* `PROVIDER_RATE_LIMIT_ESCALATION_MODELS` distinct models 429ing
|
|
3834
|
+
* escalates to a provider park). The walk was the one place
|
|
3835
|
+
* contradicting that, so it is the one place fixed here: it follows the
|
|
3836
|
+
* pair-scoped exclusions the bookkeeping just wrote, and only leaves a
|
|
3837
|
+
* provider when the EVIDENCE says the provider itself is out — a
|
|
3838
|
+
* transient/server failure, a provider-wide rate limit, a refused
|
|
3839
|
+
* account or a provider the registry has blocked.
|
|
3840
|
+
*/
|
|
3841
|
+
const firstPair = `${this.lastAttempt?.provider ?? session.type}|${this.lastAttempt?.model ?? session.model ?? 'default'}`;
|
|
3842
|
+
const failedPairs = new Set([firstPair]);
|
|
3843
|
+
const failedProviders = new Set();
|
|
3844
|
+
try {
|
|
3845
|
+
// The bookkeeping already ran for the failure above: a provider still
|
|
3846
|
+
// carrying an ACTIVE provider-scoped exclusion is out for this walk,
|
|
3847
|
+
// whatever kind the failure was classified as.
|
|
3848
|
+
if ((this.sessionFailedProviders.get(session.type) ?? 0) > Date.now()) {
|
|
3849
|
+
failedProviders.add(session.type);
|
|
3850
|
+
}
|
|
3851
|
+
}
|
|
3852
|
+
catch {
|
|
3853
|
+
// Best-effort — evidence must never break the walk.
|
|
3854
|
+
}
|
|
3430
3855
|
// Intent-aware escalation for the FAILOVER walk (the live junk path):
|
|
3431
3856
|
// a mid-turn failure used to re-route on the bare message alone, so a
|
|
3432
3857
|
// software turn whose first cloud provider died fell through to a tiny
|
|
@@ -3436,25 +3861,34 @@ export class ChatCommand extends BaseCommand {
|
|
|
3436
3861
|
// applies here too — but NEVER drop a candidate the floor would keep,
|
|
3437
3862
|
// so auto still cannot dead-end (local stays the last resort).
|
|
3438
3863
|
const failoverRoutingText = routingText;
|
|
3439
|
-
// Try ALL
|
|
3440
|
-
//
|
|
3864
|
+
// Try ALL routable candidates (no 3-candidate cap) — bounded by a
|
|
3865
|
+
// fixed attempt count so a churning pool cannot spin the turn.
|
|
3441
3866
|
const maxAttempts = 10;
|
|
3442
3867
|
for (let i = 0; i < maxAttempts; i++) {
|
|
3443
3868
|
let next = null;
|
|
3444
3869
|
try {
|
|
3445
3870
|
next = failoverRoutingText
|
|
3446
|
-
? await this.routeMessageAuto(message, [...
|
|
3871
|
+
? await this.routeMessageAuto(message, [...failedProviders], {
|
|
3447
3872
|
routingText: failoverRoutingText,
|
|
3448
3873
|
fallbackFrom: firstType,
|
|
3874
|
+
excludePairs: [...failedPairs],
|
|
3449
3875
|
})
|
|
3450
|
-
: await this.routeMessageAuto(message, [...
|
|
3876
|
+
: await this.routeMessageAuto(message, [...failedProviders], {
|
|
3877
|
+
fallbackFrom: firstType,
|
|
3878
|
+
excludePairs: [...failedPairs],
|
|
3879
|
+
});
|
|
3451
3880
|
}
|
|
3452
3881
|
catch {
|
|
3453
3882
|
break;
|
|
3454
3883
|
}
|
|
3455
|
-
if (!next
|
|
3884
|
+
if (!next)
|
|
3456
3885
|
break;
|
|
3457
|
-
|
|
3886
|
+
const nextPair = `${next.type}|${next.model ?? 'default'}`;
|
|
3887
|
+
// A pair this walk already tried means nothing NEW is routable — stop
|
|
3888
|
+
// rather than re-paying for a known-bad call.
|
|
3889
|
+
if (failedPairs.has(nextPair))
|
|
3890
|
+
break;
|
|
3891
|
+
failedPairs.add(nextPair);
|
|
3458
3892
|
// Last-resort truthfulness: if even the agentic floor could not
|
|
3459
3893
|
// avoid a weak model, say so ONCE so a degraded answer is never
|
|
3460
3894
|
// mistaken for a real one (the tiny local model fabricated tool
|
|
@@ -3503,6 +3937,37 @@ export class ChatCommand extends BaseCommand {
|
|
|
3503
3937
|
kind: nextKind,
|
|
3504
3938
|
reason: describeFailureKind(nextKind),
|
|
3505
3939
|
});
|
|
3940
|
+
// ── DIAGNOSABILITY: every failed candidate gets a reason LINE and
|
|
3941
|
+
// a REGISTRY write, not only an in-memory entry. A live turn
|
|
3942
|
+
// walked gemini → openrouter → groq → local and left NO on-disk
|
|
3943
|
+
// record of why the first three died — the walk was undebuggable
|
|
3944
|
+
// and those candidates never lost health, so the router kept
|
|
3945
|
+
// offering them. `recordAutoProviderFailure` writes the pair's
|
|
3946
|
+
// own row (and, for a rate-limit, parks it for the real reset
|
|
3947
|
+
// window) while leaving a healthy provider's siblings routable.
|
|
3948
|
+
try {
|
|
3949
|
+
this.recordAutoProviderFailure(next.type, nextErr, next.model);
|
|
3950
|
+
}
|
|
3951
|
+
catch {
|
|
3952
|
+
// Best-effort — bookkeeping must never break the turn.
|
|
3953
|
+
}
|
|
3954
|
+
// Is the PROVIDER out, or only this PAIR? The bookkeeping just
|
|
3955
|
+
// wrote the answer; read it rather than re-deriving the rule, so
|
|
3956
|
+
// the walk and the exclusion filter can never disagree.
|
|
3957
|
+
let providerOut = false;
|
|
3958
|
+
try {
|
|
3959
|
+
providerOut = (this.sessionFailedProviders.get(next.type) ?? 0) > Date.now();
|
|
3960
|
+
}
|
|
3961
|
+
catch {
|
|
3962
|
+
// Unknown scope → treat it as pair-scoped (the conservative
|
|
3963
|
+
// direction for staying routable).
|
|
3964
|
+
}
|
|
3965
|
+
if (providerOut)
|
|
3966
|
+
failedProviders.add(next.type);
|
|
3967
|
+
logger.warn(` ⚠️ ${next.type}/${next.model ?? 'default'} failed (${describeFailureKind(nextKind)}) — ` +
|
|
3968
|
+
(providerOut
|
|
3969
|
+
? 'that provider is out; trying the next auto candidate...'
|
|
3970
|
+
: `trying ${next.type}'s next model, then the next provider...`));
|
|
3506
3971
|
// Next candidate.
|
|
3507
3972
|
}
|
|
3508
3973
|
}
|
|
@@ -3608,6 +4073,10 @@ export class ChatCommand extends BaseCommand {
|
|
|
3608
4073
|
// gate `loop-executor`'s pinned pool applies.
|
|
3609
4074
|
if (!hasCredentials(this.configManager, fbType))
|
|
3610
4075
|
continue;
|
|
4076
|
+
// The model THIS candidate actually used — hoisted so the failure
|
|
4077
|
+
// handler below can name the pair without relying on block-scoped
|
|
4078
|
+
// bindings from the `try` body.
|
|
4079
|
+
let fbAttemptedModel;
|
|
3611
4080
|
try {
|
|
3612
4081
|
const resolved = resolveProvider(this.configManager, fbType);
|
|
3613
4082
|
// The fallback provider gets ITS OWN model, not the primary's
|
|
@@ -3639,6 +4108,7 @@ export class ChatCommand extends BaseCommand {
|
|
|
3639
4108
|
`${pinnedType}/${pinnedModel ?? 'default'}.`);
|
|
3640
4109
|
}
|
|
3641
4110
|
const fallbackModel = pick?.model ?? baseModel;
|
|
4111
|
+
fbAttemptedModel = fallbackModel;
|
|
3642
4112
|
const resp = await tryGenerate(resolved.provider, resolved.type, fallbackModel);
|
|
3643
4113
|
// ── The pair that ANSWERED is the pair the turn ran on ───────
|
|
3644
4114
|
// This walk used to serve the fallback and leave `session` still
|
|
@@ -3673,7 +4143,27 @@ export class ChatCommand extends BaseCommand {
|
|
|
3673
4143
|
logger.warn(` 🔀 ${pinnedType} was unavailable — answered from ${resolved.provider.name} (${fallbackModel ?? 'default'})`);
|
|
3674
4144
|
return resp;
|
|
3675
4145
|
}
|
|
3676
|
-
catch {
|
|
4146
|
+
catch (fbErr) {
|
|
4147
|
+
// DIAGNOSABILITY (pinned walk): name and LEARN each failed
|
|
4148
|
+
// fallback candidate instead of swallowing it. The silent catch
|
|
4149
|
+
// meant a pinned turn that stepped through three dead providers
|
|
4150
|
+
// reported only the primary error, and none of those providers
|
|
4151
|
+
// lost health — the same undebuggable-walk defect the auto walk
|
|
4152
|
+
// had.
|
|
4153
|
+
const fbKind = classifyFallbackError(fbErr);
|
|
4154
|
+
this.recordTurnAttempt({
|
|
4155
|
+
provider: fbType,
|
|
4156
|
+
model: fbAttemptedModel ?? 'default',
|
|
4157
|
+
kind: fbKind,
|
|
4158
|
+
reason: describeFailureKind(fbKind),
|
|
4159
|
+
});
|
|
4160
|
+
try {
|
|
4161
|
+
this.recordAutoProviderFailure(fbType, fbErr, fbAttemptedModel);
|
|
4162
|
+
}
|
|
4163
|
+
catch {
|
|
4164
|
+
// Best-effort — bookkeeping must never break the turn.
|
|
4165
|
+
}
|
|
4166
|
+
logger.warn(` ⚠️ ${fbType}/${fbAttemptedModel ?? 'default'} failed (${describeFailureKind(fbKind)}) — trying the next fallback...`);
|
|
3677
4167
|
// Next fallback candidate.
|
|
3678
4168
|
}
|
|
3679
4169
|
}
|
|
@@ -3841,6 +4331,44 @@ export class ChatCommand extends BaseCommand {
|
|
|
3841
4331
|
* the ranked candidates and picks the first one whose isAvailable() passes —
|
|
3842
4332
|
* so Auto routing never sends a request to a provider that would 401.
|
|
3843
4333
|
*/
|
|
4334
|
+
/**
|
|
4335
|
+
* Point the routing snapshot at the pair this step will ACTUALLY use.
|
|
4336
|
+
*
|
|
4337
|
+
* THE DEFECT THIS CLOSES (measured 2026-10-13). The snapshot was assigned once
|
|
4338
|
+
* per `routeMessageAuto` call, from the router's DECISION — before the
|
|
4339
|
+
* candidate walk had a chance to substitute a pair — so a step could record
|
|
4340
|
+
* `routing: gemini/gemini-3.1-flash-lite, agenticCapable: true` while the
|
|
4341
|
+
* request went to `local/deepseek-coder:latest`: the Trace tab, the console
|
|
4342
|
+
* line and the capability verdict all named a model the turn never ran, and the
|
|
4343
|
+
* disagreement was invisible (trace-1791907393457-7seewg). The pair that really
|
|
4344
|
+
* ran was in `lastAttempt` only.
|
|
4345
|
+
*
|
|
4346
|
+
* So the snapshot now FOLLOWS the attempt: provider/model are the pair being
|
|
4347
|
+
* asked, the capability verdict is recomputed for THAT pair (the field asks
|
|
4348
|
+
* "may the final routed model hold an agentic software task", so it must
|
|
4349
|
+
* describe the model that ran), and the router's intended pair is kept once as
|
|
4350
|
+
* `substitutedFor` so the substitution stays auditable instead of being erased.
|
|
4351
|
+
* The first substitution wins, because that is the DECISION the walk moved away
|
|
4352
|
+
* from — a later hop does not overwrite it with an intermediate pair.
|
|
4353
|
+
*/
|
|
4354
|
+
noteServedRoute(provider, model) {
|
|
4355
|
+
const snap = this.lastRouteSnapshot;
|
|
4356
|
+
if (!snap)
|
|
4357
|
+
return;
|
|
4358
|
+
const served = `${provider}/${model}`;
|
|
4359
|
+
const intended = `${snap.provider}/${snap.model}`;
|
|
4360
|
+
const next = {
|
|
4361
|
+
...snap,
|
|
4362
|
+
provider,
|
|
4363
|
+
model,
|
|
4364
|
+
agenticCapable: isAgenticCapableModel(model, provider),
|
|
4365
|
+
};
|
|
4366
|
+
if (snap.substitutedFor)
|
|
4367
|
+
next.substitutedFor = snap.substitutedFor;
|
|
4368
|
+
else if (served !== intended)
|
|
4369
|
+
next.substitutedFor = intended;
|
|
4370
|
+
this.lastRouteSnapshot = next;
|
|
4371
|
+
}
|
|
3844
4372
|
async routeMessageAuto(message, excludeProviders = [], opts) {
|
|
3845
4373
|
// The text ROUTING is decided from — the prior software ask for a bare
|
|
3846
4374
|
// continuation, otherwise the message itself. `message` is used everywhere
|
|
@@ -3911,6 +4439,13 @@ export class ChatCommand extends BaseCommand {
|
|
|
3911
4439
|
const decision = opts?.noCache === true
|
|
3912
4440
|
? resolveFresh()
|
|
3913
4441
|
: withRoutingCache(cacheSignature, 30_000, resolveFresh);
|
|
4442
|
+
/**
|
|
4443
|
+
* Does this ask need CAPACITY (not just a healthy model)? One predicate for
|
|
4444
|
+
* the whole route, shared by the capability floor, the model-first override
|
|
4445
|
+
* and this walk's per-provider model resolution — so a `default` model is
|
|
4446
|
+
* resolved capability-first exactly when the ask is agentic.
|
|
4447
|
+
*/
|
|
4448
|
+
const agenticAsk = isAgenticTask(decision.complexity, decision.taskProfile);
|
|
3914
4449
|
// G5 — record the routing snapshot now (the winner + why), so the reasoning
|
|
3915
4450
|
// trace for this turn can show the decision instead of a blank column.
|
|
3916
4451
|
// P3 — the TASK this turn belongs to, as far as model continuity is
|
|
@@ -4038,6 +4573,10 @@ export class ChatCommand extends BaseCommand {
|
|
|
4038
4573
|
if (isModelExcluded(prov, model))
|
|
4039
4574
|
return;
|
|
4040
4575
|
const key = `${prov}|${model}`;
|
|
4576
|
+
// Pairs this WALK already tried (see `opts.excludePairs`) — a pair-scoped
|
|
4577
|
+
// exclusion, so a provider's remaining models stay reachable.
|
|
4578
|
+
if (opts?.excludePairs?.includes(key))
|
|
4579
|
+
return;
|
|
4041
4580
|
if (seenPairs.has(key))
|
|
4042
4581
|
return;
|
|
4043
4582
|
// NOTE: deliberately NO registry-usability skip here. `resolveWorkingModel`
|
|
@@ -4088,9 +4627,18 @@ export class ChatCommand extends BaseCommand {
|
|
|
4088
4627
|
try {
|
|
4089
4628
|
const resolved = resolveProvider(this.configManager, candidate.provider);
|
|
4090
4629
|
if (await resolved.provider.isAvailable()) {
|
|
4630
|
+
// A `default` model is resolved by the router, and for a task that
|
|
4631
|
+
// NEEDS capacity that resolution is told so: the health ranking it
|
|
4632
|
+
// otherwise follows picks the fastest model, which is how a keyless
|
|
4633
|
+
// local rung ended up on `local/deepseek-coder:latest` (a
|
|
4634
|
+
// confabulation) with a verified `gpt-oss:120b-cloud` unused
|
|
4635
|
+
// (trace-1791907393457-7seewg). `agenticAsk` is the same predicate the
|
|
4636
|
+
// router's own capability floor uses, so the two cannot disagree.
|
|
4091
4637
|
const desired = candidate.model !== 'default'
|
|
4092
4638
|
? candidate.model
|
|
4093
|
-
: getAutoRouter().resolveModel(candidate.provider, 'chat', this.configManager
|
|
4639
|
+
: getAutoRouter().resolveModel(candidate.provider, 'chat', this.configManager, undefined, {
|
|
4640
|
+
agenticAsk,
|
|
4641
|
+
});
|
|
4094
4642
|
// Model health: only use models that actually exist on the provider.
|
|
4095
4643
|
// A provider's pinned config.model can be deprecated or a placeholder
|
|
4096
4644
|
// (e.g. gemini-2.0-flash-exp → 404) — repair to a live model. The pair
|