@bivy/bivy 0.7.0-staging.97 → 0.7.0-staging.99
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/metadata.js +18 -0
- package/dist/policy/conditions.js +54 -3
- package/dist/policy/run-policy.js +2 -1
- package/dist/policy/session-reroute.js +52 -0
- package/dist/server.js +202 -15
- package/package.json +1 -1
package/dist/metadata.js
CHANGED
|
@@ -169,6 +169,24 @@ export class MetadataStore {
|
|
|
169
169
|
this.data.sessions[id] = { ...prev, resumePending: pending, updatedAt: nowIso() };
|
|
170
170
|
this.save();
|
|
171
171
|
}
|
|
172
|
+
/** Set/clear the durable auto-resume time (rate/usage-limit recovery). Pass
|
|
173
|
+
* null to clear. No-op when the row is missing or already in the requested
|
|
174
|
+
* state, so it never churns the file on the hot turn path. */
|
|
175
|
+
setResumeAt(id, resumeAt) {
|
|
176
|
+
const prev = this.data.sessions[id];
|
|
177
|
+
if (!prev)
|
|
178
|
+
return;
|
|
179
|
+
const next = resumeAt ?? undefined;
|
|
180
|
+
if ((prev.resumeAt ?? undefined) === next)
|
|
181
|
+
return;
|
|
182
|
+
this.data.sessions[id] = { ...prev, resumeAt: next, updatedAt: nowIso() };
|
|
183
|
+
this.save();
|
|
184
|
+
}
|
|
185
|
+
/** Sessions with a durable auto-resume time set — the resume sweep re-arms
|
|
186
|
+
* these after a restart. */
|
|
187
|
+
sessionsWithResumeAt() {
|
|
188
|
+
return Object.values(this.data.sessions).filter((s) => typeof s.resumeAt === "string" && s.resumeAt);
|
|
189
|
+
}
|
|
172
190
|
/** Look up a session's durable metadata by id, or by session-file path. */
|
|
173
191
|
getSession(idOrPath) {
|
|
174
192
|
if (!idOrPath)
|
|
@@ -50,6 +50,48 @@ export function parseResetsAt(raw) {
|
|
|
50
50
|
const iso = /(20\d\d-\d\d-\d\dT[\d:.]+(?:Z|[+-]\d\d:?\d\d))/.exec(raw);
|
|
51
51
|
return iso?.[1];
|
|
52
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Parse a bare wall-clock reset time — the shape Claude's subscription limits
|
|
55
|
+
* surface, e.g. `resets 12am (UTC)`, `resets at 3pm UTC`, `resets 09:00 UTC` —
|
|
56
|
+
* into the ISO timestamp of its NEXT occurrence (interpreted as UTC, which is
|
|
57
|
+
* what these messages state). Returns undefined when there's no clear clock
|
|
58
|
+
* time, so a relative phrase ("resets in 2 hours", handled by
|
|
59
|
+
* parseRetryAfterMs) or an unrelated number never masquerades as a reset.
|
|
60
|
+
*
|
|
61
|
+
* NB: a bare time-of-day can't say WHICH day, so for a multi-day window (a
|
|
62
|
+
* "weekly limit") this resolves to the nearest matching midnight, which may be
|
|
63
|
+
* earlier than the real reset. Prefer a structured resetsAtHint when available.
|
|
64
|
+
*/
|
|
65
|
+
export function parseResetClock(raw, nowMs) {
|
|
66
|
+
const m = /reset[a-z]*\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)?/i.exec(raw);
|
|
67
|
+
if (!m)
|
|
68
|
+
return undefined;
|
|
69
|
+
const meridiem = m[3]?.toLowerCase();
|
|
70
|
+
// Require a real clock signal — a meridiem, an explicit minutes field, or a
|
|
71
|
+
// trailing UTC/GMT marker — so a bare "resets 5 nodes" can't parse as 05:00.
|
|
72
|
+
const tzFollows = /\b(?:utc|gmt)\b/i.test(raw.slice(m.index));
|
|
73
|
+
if (!meridiem && m[2] === undefined && !tzFollows)
|
|
74
|
+
return undefined;
|
|
75
|
+
let hour = Number(m[1]);
|
|
76
|
+
const minute = m[2] ? Number(m[2]) : 0;
|
|
77
|
+
if (hour > 23 || minute > 59)
|
|
78
|
+
return undefined;
|
|
79
|
+
if (meridiem === "am") {
|
|
80
|
+
if (hour === 12)
|
|
81
|
+
hour = 0;
|
|
82
|
+
}
|
|
83
|
+
else if (meridiem === "pm") {
|
|
84
|
+
if (hour !== 12)
|
|
85
|
+
hour += 12;
|
|
86
|
+
}
|
|
87
|
+
if (hour > 23)
|
|
88
|
+
return undefined;
|
|
89
|
+
const now = new Date(nowMs);
|
|
90
|
+
let target = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hour, minute, 0, 0);
|
|
91
|
+
if (target <= nowMs)
|
|
92
|
+
target += 86_400_000; // already past today → next day's occurrence
|
|
93
|
+
return new Date(target).toISOString();
|
|
94
|
+
}
|
|
53
95
|
// Ordered classifiers: the FIRST match wins, so more-specific/actionable
|
|
54
96
|
// conditions are tested before broader ones (auth 401 before generic HTTP
|
|
55
97
|
// noise; explicit billing/quota before a bare rate-limit; context-window before
|
|
@@ -58,7 +100,12 @@ const CLASSIFIERS = [
|
|
|
58
100
|
{ condition: "auth_failed", test: (r) => isAnthropicAuthError(r) },
|
|
59
101
|
{
|
|
60
102
|
condition: "credits_exhausted",
|
|
61
|
-
|
|
103
|
+
// Includes subscription usage caps — a Claude "5-hour" or "weekly" window
|
|
104
|
+
// hit reads "you've hit your weekly limit · resets 12am (UTC)". The window
|
|
105
|
+
// qualifier (weekly/daily/5-hour/7-day) is optional and may sit between
|
|
106
|
+
// "your" and "limit", so it must not break the match (it did before — the
|
|
107
|
+
// word "weekly" left these limits classified "unknown" and never resumed).
|
|
108
|
+
test: (r) => /\b402\b|payment required|insufficient\s+(?:credit|quota|balance|funds)|credit balance (?:is )?too low|quota (?:exceeded|exhausted)|billing|(?:usage|session|weekly|daily|monthly|5[\s-]?hour|7[\s-]?day)[\s-]?limit(?:\s+(?:reached|hit))?|(?:you(?:'ve| have)\s+)?hit your (?:(?:weekly|daily|monthly|session|usage|5[\s-]?hour|7[\s-]?day)\s+)?limit|out of credits|plan (?:limit|allowance)/i.test(r),
|
|
62
109
|
},
|
|
63
110
|
{
|
|
64
111
|
condition: "rate_limited",
|
|
@@ -86,7 +133,7 @@ const CLASSIFIERS = [
|
|
|
86
133
|
* recovery metadata. Unmatched failures are `"unknown"` — deliberately left for
|
|
87
134
|
* a human rather than blindly retried.
|
|
88
135
|
*/
|
|
89
|
-
export function classifyFailure(error) {
|
|
136
|
+
export function classifyFailure(error, opts = {}) {
|
|
90
137
|
const raw = rawText(error).slice(0, 2000);
|
|
91
138
|
const condition = CLASSIFIERS.find((c) => c.test(raw))?.condition ?? "unknown";
|
|
92
139
|
const out = { condition, raw };
|
|
@@ -95,7 +142,11 @@ export function classifyFailure(error) {
|
|
|
95
142
|
const retryAfterMs = parseRetryAfterMs(raw);
|
|
96
143
|
if (retryAfterMs !== undefined)
|
|
97
144
|
out.retryAfterMs = retryAfterMs;
|
|
98
|
-
|
|
145
|
+
// Reset time, most-authoritative first: a structured hint the caller
|
|
146
|
+
// supplied (the provider's own usage snapshot), then an ISO stamp in the
|
|
147
|
+
// text, then a bare wall-clock ("resets 12am (UTC)") resolved to its next
|
|
148
|
+
// occurrence.
|
|
149
|
+
const resetsAt = opts.resetsAtHint ?? parseResetsAt(raw) ?? parseResetClock(raw, opts.now ?? Date.now());
|
|
99
150
|
if (resetsAt !== undefined)
|
|
100
151
|
out.resetsAt = resetsAt;
|
|
101
152
|
}
|
|
@@ -34,7 +34,7 @@ export function createRunPolicy(deps = {}) {
|
|
|
34
34
|
const now = deps.now ?? Date.now;
|
|
35
35
|
return {
|
|
36
36
|
decide(ctx) {
|
|
37
|
-
const classified = classifyFailure(ctx.error);
|
|
37
|
+
const classified = classifyFailure(ctx.error, { now: now(), resetsAtHint: ctx.resetsAtHint });
|
|
38
38
|
const { condition } = classified;
|
|
39
39
|
const rule = findRule(ruleset, condition, context);
|
|
40
40
|
if (!rule)
|
|
@@ -73,6 +73,7 @@ export function createRunPolicy(deps = {}) {
|
|
|
73
73
|
delayMs,
|
|
74
74
|
condition,
|
|
75
75
|
summary: `${condition}: transient — retrying (attempt ${nextAttempt}/${rule.maxAttempts})${timing}.`,
|
|
76
|
+
...(resetDelayMs !== undefined && classified.resetsAt ? { resetsAt: classified.resetsAt } : {}),
|
|
76
77
|
};
|
|
77
78
|
}
|
|
78
79
|
// action === "reroute": walk the chain from the current cursor, skipping
|
|
@@ -21,6 +21,17 @@
|
|
|
21
21
|
// whether to suppress the turn's error toast before kicking off the async swap +
|
|
22
22
|
// retry (`applyReroute`). Reroute happens only at the turn boundary, so there is
|
|
23
23
|
// no partial-work hazard.
|
|
24
|
+
//
|
|
25
|
+
// It also plans the OTHER in-place recovery a live session can do: waiting out a
|
|
26
|
+
// provider usage/rate limit and re-sending the same prompt when the window
|
|
27
|
+
// resets (`planResume`). Unlike a reroute (which the controller applies itself),
|
|
28
|
+
// a resume can be hours away and must survive a daemon restart, so scheduling +
|
|
29
|
+
// persistence live in the caller (src/server.ts) — the controller only decides
|
|
30
|
+
// whether a resume is warranted and by when.
|
|
31
|
+
/** Below this, a "retry" is ordinary backoff (seconds) — not worth deferring an
|
|
32
|
+
* interactive turn for; let it surface. A real usage/rate window reset is
|
|
33
|
+
* minutes-to-days out and always clears this bar. */
|
|
34
|
+
const MIN_RESUME_DELAY_MS = 60_000;
|
|
24
35
|
const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
25
36
|
export class SessionRerouteController {
|
|
26
37
|
deps;
|
|
@@ -50,6 +61,47 @@ export class SessionRerouteController {
|
|
|
50
61
|
attempt: this.attempt,
|
|
51
62
|
rerouteCount: this.rerouteCount,
|
|
52
63
|
});
|
|
64
|
+
if (decision.action !== "reroute")
|
|
65
|
+
return null;
|
|
66
|
+
return this.rerouteFrom(decision, currentModel);
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Decide whether this turn error should be recovered by WAITING for a provider
|
|
70
|
+
* usage/rate limit to reset and re-sending the same prompt. Returns a plan the
|
|
71
|
+
* caller should persist + schedule, or null (surface the error as usual).
|
|
72
|
+
*
|
|
73
|
+
* `resetsAtHint` is the authoritative reset time when the caller has one (the
|
|
74
|
+
* provider's structured usage snapshot) — essential for a multi-day "weekly"
|
|
75
|
+
* window, whose error text only states a time-of-day. `now` is injectable for
|
|
76
|
+
* deterministic tests. Pure w.r.t. the controller's counters.
|
|
77
|
+
*/
|
|
78
|
+
planResume(rawError, currentModel, opts = {}) {
|
|
79
|
+
if (this.applying)
|
|
80
|
+
return null;
|
|
81
|
+
const now = opts.now ?? Date.now();
|
|
82
|
+
const decision = this.deps.policy.decide({
|
|
83
|
+
routing: { model: currentModel },
|
|
84
|
+
error: rawError,
|
|
85
|
+
attempt: this.attempt,
|
|
86
|
+
rerouteCount: this.rerouteCount,
|
|
87
|
+
resetsAtHint: opts.resetsAtHint,
|
|
88
|
+
});
|
|
89
|
+
if (decision.action !== "retry")
|
|
90
|
+
return null;
|
|
91
|
+
// Only defer for a concrete recovery window — a provider reset, or a delay
|
|
92
|
+
// long enough that it's clearly a limit rather than routine backoff.
|
|
93
|
+
if (decision.resetsAt === undefined && decision.delayMs < MIN_RESUME_DELAY_MS)
|
|
94
|
+
return null;
|
|
95
|
+
const resumeAt = decision.resetsAt ?? new Date(now + decision.delayMs).toISOString();
|
|
96
|
+
return { condition: decision.condition, summary: decision.summary, delayMs: Math.max(0, decision.delayMs), resumeAt };
|
|
97
|
+
}
|
|
98
|
+
/** Advance the attempt budget once the caller has committed to a resume, so a
|
|
99
|
+
* limit that re-fires after the reset counts toward `maxAttempts` and can
|
|
100
|
+
* eventually exhaust (→ park) instead of looping forever. */
|
|
101
|
+
noteResumeApplied() {
|
|
102
|
+
this.attempt += 1;
|
|
103
|
+
}
|
|
104
|
+
rerouteFrom(decision, currentModel) {
|
|
53
105
|
if (decision.action !== "reroute")
|
|
54
106
|
return null;
|
|
55
107
|
const model = decision.routing.model;
|
package/dist/server.js
CHANGED
|
@@ -501,11 +501,11 @@ const terminals = new TerminalManager();
|
|
|
501
501
|
// and fixed for that session's life; switching agents in the UI starts a new one.
|
|
502
502
|
let defaultRuntimeId = (process.env.BIVY_RUNTIME ?? "pi").toLowerCase();
|
|
503
503
|
const runtimeHost = new RuntimeHost({ credsDir, piDir, sessionsDir, attachToChat: attachToChatForSession });
|
|
504
|
-
//
|
|
505
|
-
//
|
|
506
|
-
// hits an exhausted-credits / rate-limit turn error swaps down the
|
|
507
|
-
// runtime's live setModel) and retries
|
|
508
|
-
//
|
|
504
|
+
// A built-in in-session model-fallback ruleset from BIVY_SESSION_MODEL_FALLBACK
|
|
505
|
+
// (docs/rulesets.md). Opt-in: set it to a comma-separated model list and a
|
|
506
|
+
// session that hits an exhausted-credits / rate-limit turn error swaps down the
|
|
507
|
+
// list (via the runtime's live setModel) and retries. Used only when the user
|
|
508
|
+
// hasn't authored their own session-scoped ruleset in the UI.
|
|
509
509
|
function sessionModelFallbackRuleset() {
|
|
510
510
|
const models = (process.env.BIVY_SESSION_MODEL_FALLBACK ?? "")
|
|
511
511
|
.split(",")
|
|
@@ -529,9 +529,21 @@ function sessionModelFallbackRuleset() {
|
|
|
529
529
|
],
|
|
530
530
|
};
|
|
531
531
|
}
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
532
|
+
/** The ruleset in-session recovery runs under right now: the user's active
|
|
533
|
+
* ruleset if it applies to sessions, else the env model-fallback ruleset, else
|
|
534
|
+
* undefined (→ built-in DEFAULT_RULESET). Read lazily on each turn error so UI
|
|
535
|
+
* edits take effect without a restart, mirroring activeQueueRuleset. */
|
|
536
|
+
function activeSessionRuleset() {
|
|
537
|
+
return activeRulesetFor(rulesetsDir, "session") ?? sessionModelFallbackRuleset();
|
|
538
|
+
}
|
|
539
|
+
// The in-session recovery effector's policy. Always available: an interactive
|
|
540
|
+
// session can wait out a provider usage/rate limit and resume when it resets
|
|
541
|
+
// (planResume), or swap models down a fallback chain (planReroute). Thin wrapper
|
|
542
|
+
// so a freshly-saved active ruleset is picked up on the next turn error.
|
|
543
|
+
const sessionRunPolicy = {
|
|
544
|
+
decide: (ctx) => createRunPolicy({ context: "session", ruleset: activeSessionRuleset() }).decide(ctx),
|
|
545
|
+
};
|
|
546
|
+
if (process.env.BIVY_SESSION_MODEL_FALLBACK) {
|
|
535
547
|
console.log(`[policy] in-session model reroute enabled: ${process.env.BIVY_SESSION_MODEL_FALLBACK}`);
|
|
536
548
|
}
|
|
537
549
|
let lastUpdateCheckAt = 0;
|
|
@@ -3542,6 +3554,9 @@ const RELAY_COMMANDS = {
|
|
|
3542
3554
|
record.lastPrompt = agentPrompt;
|
|
3543
3555
|
record.lastPromptOptions = promptOptionsFor(record, msg.streamingBehavior, images);
|
|
3544
3556
|
record.reroute?.beginTurn();
|
|
3557
|
+
// The user is driving this turn manually — supersede any pending auto-resume
|
|
3558
|
+
// that was scheduled after a prior limit so it can't re-fire on top of them.
|
|
3559
|
+
clearSessionResume(record.id);
|
|
3545
3560
|
await promptWithWatchdog(record, agentPrompt, record.lastPromptOptions);
|
|
3546
3561
|
}).catch((error) => {
|
|
3547
3562
|
// Mirror the HTTP path (see the /prompt route): a rejected turn after
|
|
@@ -6862,6 +6877,20 @@ const idleCloseTimer = setInterval(() => { closeIdleSessions(); pruneGhostSessio
|
|
|
6862
6877
|
idleCloseTimer.unref?.();
|
|
6863
6878
|
const worktreeCleanupTimer = setInterval(() => void sweepDiskGuardrails(), worktreeCleanupSweepMs);
|
|
6864
6879
|
worktreeCleanupTimer.unref?.();
|
|
6880
|
+
// In-session auto-resume tunables (see the resume helpers below). setTimeout
|
|
6881
|
+
// can't be trusted past ~24.8 days and we don't want one timer owning a
|
|
6882
|
+
// multi-hour wait a restart would drop, so each timer is capped and the periodic
|
|
6883
|
+
// sweep re-arms the remainder from the persisted resumeAt.
|
|
6884
|
+
const SESSION_RESUME_MAX_TIMER_MS = 30 * 60_000;
|
|
6885
|
+
const SESSION_RESUME_SWEEP_MS = 60_000;
|
|
6886
|
+
/** Slack around "due": a capped timer may fire a touch early — drive only when
|
|
6887
|
+
* within this of the target, else re-arm. */
|
|
6888
|
+
const SESSION_RESUME_TICK_MS = 15_000;
|
|
6889
|
+
const sessionResumeTimers = new Map();
|
|
6890
|
+
// Fire due auto-resumes (a usage/rate limit that has since reset) and re-arm the
|
|
6891
|
+
// tail of long waits whose in-process timer was capped or lost to a restart.
|
|
6892
|
+
const sessionResumeTimer = setInterval(() => sessionResumeSweep(), SESSION_RESUME_SWEEP_MS);
|
|
6893
|
+
sessionResumeTimer.unref?.();
|
|
6865
6894
|
// --- server-side ephemeral teardown ----------------------------------------
|
|
6866
6895
|
// On a disposable machine (bootstrap set BIVY_EPHEMERAL=1) the daemon ends the
|
|
6867
6896
|
// machine ITSELF once it goes idle, so teardown no longer needs the launching
|
|
@@ -7111,6 +7140,125 @@ async function refreshSessionUsage(record) {
|
|
|
7111
7140
|
// Usage reporting must never affect the session it's reporting on.
|
|
7112
7141
|
}
|
|
7113
7142
|
}
|
|
7143
|
+
// ── In-session auto-resume after a usage/rate limit ─────────────────────────
|
|
7144
|
+
// When a turn ends because a provider window is exhausted ("you've hit your
|
|
7145
|
+
// weekly limit · resets 12am (UTC)") and the session's ruleset says retry, we
|
|
7146
|
+
// wait out the window and re-send the same prompt when it resets — instead of
|
|
7147
|
+
// leaving a dead error bubble. Durable: the due time is persisted (metadata
|
|
7148
|
+
// resumeAt) so a daemon restart re-arms it (sessionResumeSweep); an in-process
|
|
7149
|
+
// timer fires it promptly while the daemon is up. (Tunables + timer map are
|
|
7150
|
+
// declared up by the timer cluster so the sweep interval can reference them.)
|
|
7151
|
+
/** The authoritative reset time for the limit a session just hit: the soonest
|
|
7152
|
+
* future reset among its most-utilized usage windows (the binding one), from
|
|
7153
|
+
* the last snapshot the runtime reported. Essential for a multi-day "weekly"
|
|
7154
|
+
* window, whose error text states only a time-of-day. Undefined when unknown. */
|
|
7155
|
+
function limitResetHint(record, nowMs) {
|
|
7156
|
+
const windows = record.usage?.plan?.windows ?? [];
|
|
7157
|
+
let best;
|
|
7158
|
+
for (const w of windows) {
|
|
7159
|
+
if (!w.resetsAt)
|
|
7160
|
+
continue;
|
|
7161
|
+
const at = Date.parse(w.resetsAt);
|
|
7162
|
+
if (!Number.isFinite(at) || at <= nowMs)
|
|
7163
|
+
continue;
|
|
7164
|
+
const util = w.utilizationPct ?? 0;
|
|
7165
|
+
// Prefer the most-utilized window (the one being hit); tie-break on soonest reset.
|
|
7166
|
+
if (!best || util > best.util || (util === best.util && at < best.at))
|
|
7167
|
+
best = { at, util };
|
|
7168
|
+
}
|
|
7169
|
+
return best ? new Date(best.at).toISOString() : undefined;
|
|
7170
|
+
}
|
|
7171
|
+
/** Cancel a pending in-process resume timer (leaves the durable marker alone). */
|
|
7172
|
+
function cancelSessionResumeTimer(id) {
|
|
7173
|
+
const timer = sessionResumeTimers.get(id);
|
|
7174
|
+
if (timer) {
|
|
7175
|
+
clearTimeout(timer);
|
|
7176
|
+
sessionResumeTimers.delete(id);
|
|
7177
|
+
}
|
|
7178
|
+
}
|
|
7179
|
+
/** Clear both the durable resume marker and any armed timer — the session moved
|
|
7180
|
+
* on (a new user turn, or the resume itself started). */
|
|
7181
|
+
function clearSessionResume(id) {
|
|
7182
|
+
cancelSessionResumeTimer(id);
|
|
7183
|
+
metadata.setResumeAt(id, null);
|
|
7184
|
+
}
|
|
7185
|
+
function armSessionResumeTimer(id, dueMs) {
|
|
7186
|
+
cancelSessionResumeTimer(id);
|
|
7187
|
+
const delay = Math.min(Math.max(0, dueMs - Date.now()), SESSION_RESUME_MAX_TIMER_MS);
|
|
7188
|
+
const timer = setTimeout(() => {
|
|
7189
|
+
sessionResumeTimers.delete(id);
|
|
7190
|
+
void driveSessionResume(id);
|
|
7191
|
+
}, delay);
|
|
7192
|
+
timer.unref?.();
|
|
7193
|
+
sessionResumeTimers.set(id, timer);
|
|
7194
|
+
}
|
|
7195
|
+
/** Persist + arm an auto-resume decided by the session policy. Synchronous so
|
|
7196
|
+
* the caller can atomically suppress the turn's error toast. */
|
|
7197
|
+
function scheduleSessionResume(record, plan) {
|
|
7198
|
+
metadata.setResumeAt(record.id, plan.resumeAt);
|
|
7199
|
+
const when = Date.parse(plan.resumeAt);
|
|
7200
|
+
const cond = plan.condition.replace(/_/g, " ");
|
|
7201
|
+
broadcast({
|
|
7202
|
+
type: "session.notice",
|
|
7203
|
+
sessionId: record.id,
|
|
7204
|
+
level: "info",
|
|
7205
|
+
message: `Hit a ${cond} limit — I'll resume this automatically when it resets (${plan.resumeAt}).`,
|
|
7206
|
+
});
|
|
7207
|
+
armSessionResumeTimer(record.id, Number.isFinite(when) ? when : Date.now());
|
|
7208
|
+
}
|
|
7209
|
+
/** Fire a due auto-resume: re-open the session if needed and re-send the turn's
|
|
7210
|
+
* last prompt. Clears the durable marker BEFORE driving so a crash mid-resume
|
|
7211
|
+
* can't loop. Best-effort — never throws into a timer/sweep. */
|
|
7212
|
+
async function driveSessionResume(id) {
|
|
7213
|
+
const meta = metadata.getSession(id);
|
|
7214
|
+
if (!meta?.resumeAt)
|
|
7215
|
+
return; // cancelled or already resumed
|
|
7216
|
+
const due = Date.parse(meta.resumeAt);
|
|
7217
|
+
if (Number.isFinite(due) && due - Date.now() > SESSION_RESUME_TICK_MS) {
|
|
7218
|
+
// A capped timer fired before the real due time — re-arm for the remainder.
|
|
7219
|
+
armSessionResumeTimer(id, due);
|
|
7220
|
+
return;
|
|
7221
|
+
}
|
|
7222
|
+
clearSessionResume(id);
|
|
7223
|
+
try {
|
|
7224
|
+
const live = openSessions.get(id);
|
|
7225
|
+
if (live?.isWorking)
|
|
7226
|
+
return; // a user turn is already running — don't pile on
|
|
7227
|
+
const record = live ?? (await resolveOrResumeSession(id, meta.path));
|
|
7228
|
+
if (!record)
|
|
7229
|
+
return; // transcript gone / unresolvable
|
|
7230
|
+
if (record.isWorking)
|
|
7231
|
+
return;
|
|
7232
|
+
// In-memory lastPrompt is the exact user turn to retry; after a restart it's
|
|
7233
|
+
// gone, so fall back to the generic interrupted-turn continuation nudge.
|
|
7234
|
+
const prompt = record.lastPrompt ?? buildInteractiveResumePrompt();
|
|
7235
|
+
console.log(`[resume] auto-resuming session ${id} — provider limit has reset`);
|
|
7236
|
+
broadcast({ type: "session.notice", sessionId: id, level: "info", message: "The limit has reset — resuming now." });
|
|
7237
|
+
await promptWithWatchdog(record, prompt, record.lastPromptOptions);
|
|
7238
|
+
}
|
|
7239
|
+
catch (error) {
|
|
7240
|
+
console.warn(`[resume] auto-resume after a provider limit failed for ${id}`, error);
|
|
7241
|
+
}
|
|
7242
|
+
}
|
|
7243
|
+
/** Re-arm (or immediately fire) durable auto-resume markers. Runs once at boot
|
|
7244
|
+
* and on an interval, so a wait survives a restart and a capped timer's tail
|
|
7245
|
+
* still fires. */
|
|
7246
|
+
function sessionResumeSweep() {
|
|
7247
|
+
const now = Date.now();
|
|
7248
|
+
for (const meta of metadata.sessionsWithResumeAt()) {
|
|
7249
|
+
const due = Date.parse(meta.resumeAt);
|
|
7250
|
+
if (!Number.isFinite(due)) {
|
|
7251
|
+
metadata.setResumeAt(meta.id, null);
|
|
7252
|
+
continue;
|
|
7253
|
+
}
|
|
7254
|
+
if (sessionResumeTimers.has(meta.id))
|
|
7255
|
+
continue; // already armed this run
|
|
7256
|
+
if (due <= now + SESSION_RESUME_TICK_MS)
|
|
7257
|
+
void driveSessionResume(meta.id);
|
|
7258
|
+
else
|
|
7259
|
+
armSessionResumeTimer(meta.id, due);
|
|
7260
|
+
}
|
|
7261
|
+
}
|
|
7114
7262
|
/**
|
|
7115
7263
|
* Turn a raw provider/runtime error string into something a human can read.
|
|
7116
7264
|
* Model APIs commonly return `<status> {json}` (e.g. `400 {"error":{"message":
|
|
@@ -7184,9 +7332,12 @@ function maybeSignalAuthRequired(record, errorText) {
|
|
|
7184
7332
|
}
|
|
7185
7333
|
function attachSessionListeners(record) {
|
|
7186
7334
|
record.unsubscribe?.();
|
|
7187
|
-
// In-session
|
|
7188
|
-
//
|
|
7189
|
-
|
|
7335
|
+
// In-session recovery controller — waits out a usage/rate limit and resumes
|
|
7336
|
+
// (planResume), or swaps models down a fallback chain (planReroute). One per
|
|
7337
|
+
// session; its per-turn budget resets on each user prompt. The policy reads
|
|
7338
|
+
// the active session ruleset lazily, so it's inert until one authorizes a
|
|
7339
|
+
// retry/reroute for the failing condition.
|
|
7340
|
+
if (!record.reroute) {
|
|
7190
7341
|
record.reroute = new SessionRerouteController({
|
|
7191
7342
|
policy: sessionRunPolicy,
|
|
7192
7343
|
onNotice: (n) => broadcast({ type: "session.notice", sessionId: record.id, level: n.level, message: n.message }),
|
|
@@ -7306,7 +7457,18 @@ function attachSessionListeners(record) {
|
|
|
7306
7457
|
// credential or a 4xx from the API) otherwise vanished: working cleared,
|
|
7307
7458
|
// no reply, no signal. Surface it as a session-scoped error so the client
|
|
7308
7459
|
// can show it *inline in that chat*, and notify instead of "done".
|
|
7309
|
-
|
|
7460
|
+
// A terminal turn error reaches us two ways. pi-ai puts it on the last
|
|
7461
|
+
// assistant message (stopReason:"error" → terminalTurnError), and the
|
|
7462
|
+
// server owns surfacing it. Claude Code instead throws inside the SDK
|
|
7463
|
+
// query: it emits its OWN session.error to the client AND carries the raw
|
|
7464
|
+
// text on agent_end.error (e.g. "you've hit your weekly limit · resets 12am
|
|
7465
|
+
// (UTC)"). We read that too — but only to DRIVE recovery, since the runtime
|
|
7466
|
+
// already surfaced it; re-broadcasting would double the error bubble.
|
|
7467
|
+
const messageError = terminalTurnError(event);
|
|
7468
|
+
const agentEndError = typeof event.error === "string"
|
|
7469
|
+
? humanizeAgentError(event.error)
|
|
7470
|
+
: undefined;
|
|
7471
|
+
const turnError = messageError ?? (agentEndError?.trim() ? agentEndError : undefined);
|
|
7310
7472
|
// Before surfacing a turn error, see if the session's run policy can recover
|
|
7311
7473
|
// it in place by swapping to a fallback model and retrying the same prompt.
|
|
7312
7474
|
// planReroute is synchronous, so we can atomically suppress the error toast
|
|
@@ -7314,6 +7476,14 @@ function attachSessionListeners(record) {
|
|
|
7314
7476
|
const reroutePlan = turnError && record.lastPrompt !== undefined
|
|
7315
7477
|
? record.reroute?.planReroute(turnError, record.session.getCurrentModel()?.name) ?? null
|
|
7316
7478
|
: null;
|
|
7479
|
+
// If a reroute doesn't apply, a usage/rate limit that gave a reset time can
|
|
7480
|
+
// instead be waited out and resumed when the window clears (planResume is
|
|
7481
|
+
// synchronous too, so this stays atomic with suppressing the error toast).
|
|
7482
|
+
const resumePlan = !reroutePlan && turnError && record.lastPrompt !== undefined
|
|
7483
|
+
? record.reroute?.planResume(turnError, record.session.getCurrentModel()?.name, {
|
|
7484
|
+
resetsAtHint: limitResetHint(record, Date.now()),
|
|
7485
|
+
}) ?? null
|
|
7486
|
+
: null;
|
|
7317
7487
|
if (reroutePlan) {
|
|
7318
7488
|
void record.reroute.applyReroute(reroutePlan, {
|
|
7319
7489
|
getCurrentModelName: () => record.session.getCurrentModel()?.name,
|
|
@@ -7323,14 +7493,23 @@ function attachSessionListeners(record) {
|
|
|
7323
7493
|
},
|
|
7324
7494
|
});
|
|
7325
7495
|
}
|
|
7326
|
-
else if (
|
|
7496
|
+
else if (resumePlan) {
|
|
7497
|
+
// Charge the attempt budget so a limit that re-fires after the reset can
|
|
7498
|
+
// eventually exhaust (→ surface) instead of looping, then park the turn
|
|
7499
|
+
// as a scheduled resume rather than a dead error.
|
|
7500
|
+
record.reroute.noteResumeApplied();
|
|
7501
|
+
scheduleSessionResume(record, resumePlan);
|
|
7502
|
+
}
|
|
7503
|
+
else if (messageError) {
|
|
7504
|
+
// Only the server-owned (pi-ai) path surfaces here; a Claude Code error
|
|
7505
|
+
// the runtime already broadcast falls through to avoid a duplicate bubble.
|
|
7327
7506
|
record.lastFailureAt = Date.now();
|
|
7328
7507
|
metadata.touchSession(record.id, "failed");
|
|
7329
7508
|
scheduleAdvertise();
|
|
7330
|
-
broadcast({ type: "session.error", sessionId: record.id, error:
|
|
7509
|
+
broadcast({ type: "session.error", sessionId: record.id, error: messageError });
|
|
7331
7510
|
// If the terminal error is an auth failure (expired key/token → 4xx),
|
|
7332
7511
|
// also raise the sign-in sheet for the failing provider.
|
|
7333
|
-
maybeSignalAuthRequired(record,
|
|
7512
|
+
maybeSignalAuthRequired(record, messageError);
|
|
7334
7513
|
void sendNotificationHint({
|
|
7335
7514
|
kind: "session_error",
|
|
7336
7515
|
sessionId: record.id,
|
|
@@ -10655,6 +10834,14 @@ const server = app.listen(port, host, async () => {
|
|
|
10655
10834
|
// Recover interactive sessions a restart interrupted mid-turn (auto-continue, or
|
|
10656
10835
|
// flag for a one-tap manual Resume) per the node's sessionResumeMode setting.
|
|
10657
10836
|
void reconcileInterruptedSessions().catch((error) => console.warn("[resume] interrupted-session reconciliation failed", error));
|
|
10837
|
+
// Re-arm (or fire) durable auto-resume markers a limit-hit turn left behind,
|
|
10838
|
+
// so a session waiting out a usage/rate window still resumes after a restart.
|
|
10839
|
+
try {
|
|
10840
|
+
sessionResumeSweep();
|
|
10841
|
+
}
|
|
10842
|
+
catch (error) {
|
|
10843
|
+
console.warn("[resume] auto-resume sweep failed at boot", error);
|
|
10844
|
+
}
|
|
10658
10845
|
// Universal Agent Harness — network effect boundary (opt-in via
|
|
10659
10846
|
// BIVY_EGRESS_PROXY). Governs/logs outbound traffic of CLI agents, which
|
|
10660
10847
|
// inherit the proxy env from process.ts.
|
package/package.json
CHANGED