agent-dealer 1.3.2 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle/server/dist/adapters/agent-health.test.js +52 -0
- package/bundle/server/dist/capacity/service.js +66 -6
- package/bundle/server/dist/capacity/service.test.js +220 -0
- package/bundle/server/dist/db/index.js +8 -0
- package/bundle/server/dist/db/schema.sql +5 -1
- package/bundle/server/dist/repository/runtime-availability.js +29 -11
- package/bundle/server/dist/runners/usage-cap.js +66 -15
- package/bundle/server/dist/runners/usage-cap.test.js +96 -2
- package/bundle/server/package.json +2 -2
- package/bundle/server/static-ui/assets/index-B_pNUDRY.js +64 -0
- package/bundle/server/static-ui/index.html +1 -1
- package/bundle/shared/dist/runtime-capacity.d.ts +85 -0
- package/bundle/shared/dist/runtime-capacity.js +19 -0
- package/bundle/shared/package.json +1 -1
- package/package.json +1 -1
- package/bundle/server/static-ui/assets/index-DQ5ivcyg.js +0 -64
|
@@ -17,6 +17,8 @@ const { migrate, getDb } = await import("../db/index.js");
|
|
|
17
17
|
const { createAgent, getAgent } = await import("../repository/agents.js");
|
|
18
18
|
const { healthForAgent, runtimeIssuesUncached, githubIssuesUncached, githubIssuesSync, classifyGithubAuthStatus, clearAgentHealthCaches, setCursorProbeTimingForTests, setRunCommandForTests, } = await import("./agent-health.js");
|
|
19
19
|
const { setMuseCapabilityProbeForTests, settleMuseCapabilityCheckForTests } = await import("./muse-capability.js");
|
|
20
|
+
const { recordMuseUsageCap } = await import("../runners/usage-cap.js");
|
|
21
|
+
const { runtimeAvailability, clearAllRuntimeAvailability, } = await import("../repository/runtime-availability.js");
|
|
20
22
|
migrate();
|
|
21
23
|
// NOT-277: health never runs the real (billed) Muse capability probe in tests.
|
|
22
24
|
setMuseCapabilityProbeForTests(async () => ({ status: "capable" }));
|
|
@@ -441,6 +443,56 @@ The keychain item is stuck. Delete it and sign in again:
|
|
|
441
443
|
assert.match(issues[0].message, /panic: something else broke/);
|
|
442
444
|
assert.doesNotMatch(issues[0].message, /muse login/);
|
|
443
445
|
});
|
|
446
|
+
// ---------------------------------------------------------------------------
|
|
447
|
+
// NOT-439: agent health renders the exact provider boundary from persisted
|
|
448
|
+
// availability — the reproduced Muse message resolves to 11:59 AM in
|
|
449
|
+
// America/Los_Angeles, not the 11:46 AM of the old 30-minute fallback.
|
|
450
|
+
// ---------------------------------------------------------------------------
|
|
451
|
+
test("NOT-439: persisted muse provider reset renders the exact boundary, not the fallback", async () => {
|
|
452
|
+
clearAllRuntimeAvailability();
|
|
453
|
+
clearAgentHealthCaches();
|
|
454
|
+
try {
|
|
455
|
+
// Pinned incident replay: the provider said resets-at 18:59:01Z while the
|
|
456
|
+
// old fallback stored 18:46:20.154Z. Timezone-safe assertions on the
|
|
457
|
+
// stored instant itself.
|
|
458
|
+
const incidentNow = Date.parse("2026-10-12T18:16:20.154Z");
|
|
459
|
+
const cap = recordMuseUsageCap({
|
|
460
|
+
kind: "usage_cap",
|
|
461
|
+
message: "Subscription quota exhausted. Your usage window resets at 2026-10-12T18:59:01Z",
|
|
462
|
+
}, incidentNow);
|
|
463
|
+
assert.equal(cap?.unavailableUntil, "2026-10-12T18:59:01.000Z");
|
|
464
|
+
const stored = runtimeAvailability("muse_code", incidentNow);
|
|
465
|
+
assert.equal(stored.available, false);
|
|
466
|
+
if (!stored.available)
|
|
467
|
+
assert.equal(stored.until, "2026-10-12T18:59:01.000Z");
|
|
468
|
+
const la = { hour: "2-digit", minute: "2-digit", timeZone: "America/Los_Angeles" };
|
|
469
|
+
assert.equal(new Date("2026-10-12T18:59:01.000Z").toLocaleTimeString("en-US", la), "11:59 AM");
|
|
470
|
+
// The superseded fallback instant renders differently, proving the
|
|
471
|
+
// assertion distinguishes the two.
|
|
472
|
+
assert.equal(new Date("2026-10-12T18:46:20.154Z").toLocaleTimeString("en-US", la), "11:46 AM");
|
|
473
|
+
// Live surface: a guaranteed-future provider reset flows verbatim from
|
|
474
|
+
// persisted availability into the usage_capped health message.
|
|
475
|
+
const future = new Date(Date.now() + 2 * 3600_000);
|
|
476
|
+
future.setMilliseconds(0);
|
|
477
|
+
const resetIso = future.toISOString();
|
|
478
|
+
const live = recordMuseUsageCap({ kind: "usage_cap", message: `Subscription quota exhausted. Resets at ${resetIso}` });
|
|
479
|
+
assert.equal(live?.unavailableUntil, resetIso);
|
|
480
|
+
const stub = stubMuse(`cat ${JSON.stringify(path.join(FIXTURES, "muse-version.txt"))}`);
|
|
481
|
+
const agent = createAgent({ name: "muse-capped", runtime: "muse_code", deckId: randomUUID() });
|
|
482
|
+
const result = await withMuseEnv({ MUSE_CLI: stub.bin, META_API_KEY: "k", configHome: emptyConfigHome() }, () => healthForAgent(agent, true, undefined, true, null, NO_GITHUB));
|
|
483
|
+
const issue = result.issues.find((i) => i.code === "usage_capped");
|
|
484
|
+
assert.ok(issue, "expected a usage_capped issue from persisted availability");
|
|
485
|
+
const expectedRendering = new Date(resetIso).toLocaleTimeString([], {
|
|
486
|
+
hour: "2-digit",
|
|
487
|
+
minute: "2-digit",
|
|
488
|
+
});
|
|
489
|
+
assert.ok(issue.message.includes(expectedRendering), `expected health message ${JSON.stringify(issue.message)} to render ${expectedRendering}`);
|
|
490
|
+
}
|
|
491
|
+
finally {
|
|
492
|
+
clearAllRuntimeAvailability();
|
|
493
|
+
clearAgentHealthCaches();
|
|
494
|
+
}
|
|
495
|
+
});
|
|
444
496
|
test("a muse probe that hangs past the timeout is runtime_unknown", async () => {
|
|
445
497
|
const stub = stubMuse("sleep 30");
|
|
446
498
|
setCursorProbeTimingForTests({ timeoutMs: 150 });
|
|
@@ -5,12 +5,16 @@
|
|
|
5
5
|
// profiles sharing one account produce one entry — each carrying whatever
|
|
6
6
|
// windows the provider actually reported.
|
|
7
7
|
//
|
|
8
|
-
// Connection health
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
8
|
+
// Connection health and remaining capacity stay separate states: output never
|
|
9
|
+
// implies health. Only normalized snapshots leave this module — evidence refs
|
|
10
|
+
// and raw provider payloads stay server-side.
|
|
11
|
+
//
|
|
12
|
+
// NOT-440: an active hard cap (the same `runtime_availability` row that blocks
|
|
13
|
+
// admission) rides alongside the windows as `activeCap`; stored snapshots are
|
|
14
|
+
// never rewritten, and the overlay disappears with the row's lazy cleanup.
|
|
12
15
|
import { isWindowKnown, Runtime as RuntimeSchema } from "@agent-dealer/shared";
|
|
13
16
|
import { getDb } from "../db/index.js";
|
|
17
|
+
import { activeRuntimeCap } from "../repository/runtime-availability.js";
|
|
14
18
|
import { listAllCapacitySnapshots, recordCapacitySnapshots, } from "../repository/runtime-capacity.js";
|
|
15
19
|
import { normalizeAdapterWindow, normalizeUnavailableWindow, } from "./adapter.js";
|
|
16
20
|
/** Distinct runtimes across Agent profiles — the configured runtime accounts. */
|
|
@@ -83,6 +87,61 @@ function classifyWindow(w, nowMs) {
|
|
|
83
87
|
known: false,
|
|
84
88
|
};
|
|
85
89
|
}
|
|
90
|
+
const PAIR_RUNTIMES = new Set(["claude_code", "codex_local", "muse_code"]);
|
|
91
|
+
const CURSOR_BILLING_WINDOW_KEY = "billing_cycle";
|
|
92
|
+
/** Structured provider limit types that name a compact critical window. */
|
|
93
|
+
const LIMIT_TYPE_ROLE = {
|
|
94
|
+
five_hour: "five_hour",
|
|
95
|
+
seven_day: "weekly",
|
|
96
|
+
};
|
|
97
|
+
/** A cap reset and a snapshot reset within this distance name the same boundary. */
|
|
98
|
+
const RESET_MATCH_TOLERANCE_MS = 60_000;
|
|
99
|
+
/** The compact role a stored window presents under, or null when it never renders compactly. */
|
|
100
|
+
function compactRole(runtime, w) {
|
|
101
|
+
if (runtime === "cursor_local")
|
|
102
|
+
return w.windowKey === CURSOR_BILLING_WINDOW_KEY ? "billing_cycle" : null;
|
|
103
|
+
if (PAIR_RUNTIMES.has(runtime))
|
|
104
|
+
return w.criticalRole;
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Which compact window an active cap exhausts: structured limit metadata
|
|
109
|
+
* first (a reported non-compact limit stays unmapped), then a tightly bounded
|
|
110
|
+
* match between the provider reset and exactly one tagged snapshot reset.
|
|
111
|
+
* A fallback-cooldown cap (no provider reset) is never matched.
|
|
112
|
+
*/
|
|
113
|
+
export function resolveCapWindowRole(runtime, cap, windows) {
|
|
114
|
+
if (cap.limitType !== null) {
|
|
115
|
+
const role = LIMIT_TYPE_ROLE[cap.limitType] ?? null;
|
|
116
|
+
return role !== null && PAIR_RUNTIMES.has(runtime) ? role : null;
|
|
117
|
+
}
|
|
118
|
+
if (cap.providerResetAt === null)
|
|
119
|
+
return null;
|
|
120
|
+
const capResetMs = Date.parse(cap.providerResetAt);
|
|
121
|
+
if (!Number.isFinite(capResetMs))
|
|
122
|
+
return null;
|
|
123
|
+
const matched = new Set();
|
|
124
|
+
for (const w of windows) {
|
|
125
|
+
const role = compactRole(runtime, w);
|
|
126
|
+
if (role === null || w.resetAt === null)
|
|
127
|
+
continue;
|
|
128
|
+
const resetMs = Date.parse(w.resetAt);
|
|
129
|
+
if (Number.isFinite(resetMs) && Math.abs(resetMs - capResetMs) <= RESET_MATCH_TOLERANCE_MS) {
|
|
130
|
+
matched.add(role);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return matched.size === 1 ? [...matched][0] : null;
|
|
134
|
+
}
|
|
135
|
+
function activeCapFor(runtime, windows, nowMs) {
|
|
136
|
+
const cap = activeRuntimeCap(runtime, nowMs);
|
|
137
|
+
if (cap === null)
|
|
138
|
+
return null;
|
|
139
|
+
return {
|
|
140
|
+
unavailableUntil: cap.unavailableUntil,
|
|
141
|
+
reason: cap.reason,
|
|
142
|
+
windowRole: resolveCapWindowRole(runtime, cap, windows),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
86
145
|
/**
|
|
87
146
|
* Read model for `GET /api/runtime-capacity`: every configured runtime, its
|
|
88
147
|
* windows with freshness applied, and an explicit unavailable reason whenever
|
|
@@ -103,14 +162,15 @@ export function getRuntimeCapacitySnapshot(nowMs = Date.now()) {
|
|
|
103
162
|
const entries = [...runtimes]
|
|
104
163
|
.sort()
|
|
105
164
|
.map((runtime) => {
|
|
106
|
-
const
|
|
165
|
+
const raw = byRuntime.get(runtime) ?? [];
|
|
166
|
+
const windows = raw.map((w) => classifyWindow(w, nowMs).snapshot);
|
|
107
167
|
const knownCount = windows.filter((w) => w.unavailableReason === null).length;
|
|
108
168
|
let unavailableReason = null;
|
|
109
169
|
if (knownCount === 0) {
|
|
110
170
|
const first = windows.find((w) => w.unavailableReason)?.unavailableReason;
|
|
111
171
|
unavailableReason = first ?? "missing";
|
|
112
172
|
}
|
|
113
|
-
return { runtime, windows, unavailableReason };
|
|
173
|
+
return { runtime, windows, unavailableReason, activeCap: activeCapFor(runtime, raw, nowMs) };
|
|
114
174
|
});
|
|
115
175
|
return { runtimes: entries, generatedAt: new Date(nowMs).toISOString() };
|
|
116
176
|
}
|
|
@@ -219,3 +219,223 @@ test("a past reset time is never presented as current capacity", async () => {
|
|
|
219
219
|
assert.equal(muse.windows[0].unavailableReason, "expired");
|
|
220
220
|
assert.equal(muse.unavailableReason, "expired");
|
|
221
221
|
});
|
|
222
|
+
// NOT-440: active hard caps overlay the compact read model without rewriting
|
|
223
|
+
// stored provider observations.
|
|
224
|
+
const NOT440_NOW = Date.parse("2026-10-12T18:20:00.000Z");
|
|
225
|
+
const NOT440_RESET = "2026-10-12T18:59:01.000Z";
|
|
226
|
+
const NOT440_WEEKLY_RESET = "2026-10-16T02:00:00.000Z";
|
|
227
|
+
async function seedPair(runtime, windows) {
|
|
228
|
+
const { recordCapacitySnapshots } = await import("../repository/runtime-capacity.js");
|
|
229
|
+
recordCapacitySnapshots(runtime, windows.map((w) => ({
|
|
230
|
+
windowKey: w.windowKey,
|
|
231
|
+
providerBucket: "all_models",
|
|
232
|
+
durationMinutes: w.durationMinutes ?? (w.criticalRole === "weekly" ? 10080 : 300),
|
|
233
|
+
displayLabel: w.criticalRole === "weekly" ? "1W" : "5H",
|
|
234
|
+
usedValue: 100 - w.remainingPercent,
|
|
235
|
+
usedUnit: "percent",
|
|
236
|
+
remainingPercent: w.remainingPercent,
|
|
237
|
+
resetAt: w.resetAt,
|
|
238
|
+
observedAt: "2026-10-12T18:03:00.000Z",
|
|
239
|
+
freshUntil: "2026-10-12T19:03:00.000Z",
|
|
240
|
+
expiresAt: "2026-10-12T19:03:00.000Z",
|
|
241
|
+
source: "observed_event",
|
|
242
|
+
unavailableReason: null,
|
|
243
|
+
criticalRole: w.criticalRole,
|
|
244
|
+
})));
|
|
245
|
+
}
|
|
246
|
+
async function seedReproducedMuse() {
|
|
247
|
+
await seedPair("muse_code", [
|
|
248
|
+
{ windowKey: "five_hour", criticalRole: "five_hour", remainingPercent: 50, resetAt: NOT440_RESET },
|
|
249
|
+
{ windowKey: "weekly", criticalRole: "weekly", remainingPercent: 52, resetAt: NOT440_WEEKLY_RESET },
|
|
250
|
+
]);
|
|
251
|
+
}
|
|
252
|
+
function storedMuseRemaining() {
|
|
253
|
+
const rows = getDb()
|
|
254
|
+
.prepare("SELECT window_key, remaining_percent FROM runtime_capacity_snapshots WHERE runtime = 'muse_code'")
|
|
255
|
+
.all();
|
|
256
|
+
return Object.fromEntries(rows.map((r) => [r.window_key, r.remaining_percent]));
|
|
257
|
+
}
|
|
258
|
+
const { getDb } = await import("../db/index.js");
|
|
259
|
+
const { clearAllRuntimeAvailability, recordRuntimeAvailability, runtimeAvailability } = await import("../repository/runtime-availability.js");
|
|
260
|
+
const { recordMuseUsageCap } = await import("../runners/usage-cap.js");
|
|
261
|
+
const { resolveCapWindowRole } = await import("./service.js");
|
|
262
|
+
test("NOT-440: reproduced Muse cap maps to 5H without modifying stored snapshots", async () => {
|
|
263
|
+
clearAllCapacitySnapshots();
|
|
264
|
+
clearAllRuntimeAvailability();
|
|
265
|
+
await seedReproducedMuse();
|
|
266
|
+
assert.deepEqual(storedMuseRemaining(), { five_hour: 50, weekly: 52 });
|
|
267
|
+
recordMuseUsageCap({ kind: "usage_cap", message: `Subscription quota exhausted. Usage resets at 2026-10-12T18:59:01Z.` }, Date.parse("2026-10-12T18:16:00.000Z"));
|
|
268
|
+
const snap = getRuntimeCapacitySnapshot(NOT440_NOW);
|
|
269
|
+
const muse = snap.runtimes.find((r) => r.runtime === "muse_code");
|
|
270
|
+
assert.deepEqual(muse.activeCap, {
|
|
271
|
+
unavailableUntil: NOT440_RESET,
|
|
272
|
+
reason: muse.activeCap.reason,
|
|
273
|
+
windowRole: "five_hour",
|
|
274
|
+
});
|
|
275
|
+
assert.match(muse.activeCap.reason, /muse_code usage capped/);
|
|
276
|
+
// Windows remain the raw provider observation; the overlay is activeCap.
|
|
277
|
+
assert.equal(muse.windows.find((w) => w.criticalRole === "five_hour").remainingPercent, 50);
|
|
278
|
+
assert.equal(muse.windows.find((w) => w.criticalRole === "weekly").remainingPercent, 52);
|
|
279
|
+
assert.deepEqual(storedMuseRemaining(), { five_hour: 50, weekly: 52 });
|
|
280
|
+
// Admission/health read the same row.
|
|
281
|
+
assert.deepEqual(runtimeAvailability("muse_code", NOT440_NOW), {
|
|
282
|
+
available: false,
|
|
283
|
+
until: NOT440_RESET,
|
|
284
|
+
reason: muse.activeCap.reason,
|
|
285
|
+
});
|
|
286
|
+
});
|
|
287
|
+
test("NOT-440: overlay is absent at and after unavailableUntil; history is kept", async () => {
|
|
288
|
+
for (const offset of [0, 1]) {
|
|
289
|
+
clearAllCapacitySnapshots();
|
|
290
|
+
clearAllRuntimeAvailability();
|
|
291
|
+
await seedReproducedMuse();
|
|
292
|
+
recordMuseUsageCap({ kind: "usage_cap", message: `Subscription quota exhausted. Usage resets at 2026-10-12T18:59:01Z.` }, Date.parse("2026-10-12T18:16:00.000Z"));
|
|
293
|
+
const atMs = Date.parse(NOT440_RESET) + offset;
|
|
294
|
+
const snap = getRuntimeCapacitySnapshot(atMs);
|
|
295
|
+
const muse = snap.runtimes.find((r) => r.runtime === "muse_code");
|
|
296
|
+
assert.equal(muse.activeCap, null, `offset ${offset}ms`);
|
|
297
|
+
// Normal classification resumes: the 5H reset has passed, 1W is still current.
|
|
298
|
+
const fiveHour = muse.windows.find((w) => w.criticalRole === "five_hour");
|
|
299
|
+
assert.equal(fiveHour.remainingPercent, null);
|
|
300
|
+
assert.equal(fiveHour.unavailableReason, "expired");
|
|
301
|
+
assert.equal(muse.windows.find((w) => w.criticalRole === "weekly").remainingPercent, 52);
|
|
302
|
+
assert.deepEqual(storedMuseRemaining(), { five_hour: 50, weekly: 52 });
|
|
303
|
+
const capRows = getDb()
|
|
304
|
+
.prepare("SELECT COUNT(*) AS n FROM runtime_availability WHERE runtime = 'muse_code'")
|
|
305
|
+
.get();
|
|
306
|
+
assert.equal(capRows.n, 0, "lazy cleanup cleared the expired cap row");
|
|
307
|
+
assert.deepEqual(runtimeAvailability("muse_code", atMs), { available: true });
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
test("NOT-440: a cap with no provider reset or role stays unmapped (explicit capped)", async () => {
|
|
311
|
+
clearAllCapacitySnapshots();
|
|
312
|
+
clearAllRuntimeAvailability();
|
|
313
|
+
await seedReproducedMuse();
|
|
314
|
+
// Fallback cooldown that happens to land on the 5H reset is still not a provider boundary.
|
|
315
|
+
recordRuntimeAvailability({
|
|
316
|
+
runtime: "muse_code",
|
|
317
|
+
unavailableUntil: NOT440_RESET,
|
|
318
|
+
reason: "muse_code usage capped — usage cap",
|
|
319
|
+
});
|
|
320
|
+
const muse = getRuntimeCapacitySnapshot(NOT440_NOW).runtimes.find((r) => r.runtime === "muse_code");
|
|
321
|
+
assert.equal(muse.activeCap.windowRole, null);
|
|
322
|
+
assert.equal(muse.activeCap.unavailableUntil, NOT440_RESET);
|
|
323
|
+
assert.deepEqual(storedMuseRemaining(), { five_hour: 50, weekly: 52 });
|
|
324
|
+
});
|
|
325
|
+
const tagged = (windowKey, criticalRole, resetAt) => ({
|
|
326
|
+
windowKey,
|
|
327
|
+
providerBucket: "all_models",
|
|
328
|
+
durationMinutes: 300,
|
|
329
|
+
displayLabel: "5H",
|
|
330
|
+
usedValue: 50,
|
|
331
|
+
usedUnit: "percent",
|
|
332
|
+
remainingPercent: 50,
|
|
333
|
+
resetAt,
|
|
334
|
+
observedAt: "2026-10-12T18:03:00.000Z",
|
|
335
|
+
freshUntil: "2026-10-12T19:03:00.000Z",
|
|
336
|
+
expiresAt: "2026-10-12T19:03:00.000Z",
|
|
337
|
+
source: "observed_event",
|
|
338
|
+
unavailableReason: null,
|
|
339
|
+
criticalRole,
|
|
340
|
+
});
|
|
341
|
+
test("NOT-440: cap window role resolution per runtime (metadata first, then reset match)", () => {
|
|
342
|
+
const pair = [tagged("five_hour", "five_hour", NOT440_RESET), tagged("weekly", "weekly", NOT440_WEEKLY_RESET)];
|
|
343
|
+
const cases = [
|
|
344
|
+
{
|
|
345
|
+
name: "Claude five_hour metadata wins even without a reset match",
|
|
346
|
+
runtime: "claude_code",
|
|
347
|
+
cap: { providerResetAt: "2026-10-12T20:00:00.000Z", limitType: "five_hour" },
|
|
348
|
+
windows: pair,
|
|
349
|
+
expected: "five_hour",
|
|
350
|
+
},
|
|
351
|
+
{
|
|
352
|
+
name: "Claude seven_day metadata maps to weekly",
|
|
353
|
+
runtime: "claude_code",
|
|
354
|
+
cap: { providerResetAt: null, limitType: "seven_day" },
|
|
355
|
+
windows: pair,
|
|
356
|
+
expected: "weekly",
|
|
357
|
+
},
|
|
358
|
+
{
|
|
359
|
+
name: "Claude model-specific limit stays unmapped despite a matching weekly reset",
|
|
360
|
+
runtime: "claude_code",
|
|
361
|
+
cap: { providerResetAt: NOT440_WEEKLY_RESET, limitType: "seven_day_sonnet" },
|
|
362
|
+
windows: pair,
|
|
363
|
+
expected: null,
|
|
364
|
+
},
|
|
365
|
+
{
|
|
366
|
+
name: "Codex reset match on the weekly window",
|
|
367
|
+
runtime: "codex_local",
|
|
368
|
+
cap: { providerResetAt: NOT440_WEEKLY_RESET, limitType: null },
|
|
369
|
+
windows: pair,
|
|
370
|
+
expected: "weekly",
|
|
371
|
+
},
|
|
372
|
+
{
|
|
373
|
+
name: "Muse reset match on the 5H window (within tolerance)",
|
|
374
|
+
runtime: "muse_code",
|
|
375
|
+
cap: { providerResetAt: "2026-10-12T18:59:30.000Z", limitType: null },
|
|
376
|
+
windows: pair,
|
|
377
|
+
expected: "five_hour",
|
|
378
|
+
},
|
|
379
|
+
{
|
|
380
|
+
name: "Muse reset outside tolerance stays unmapped",
|
|
381
|
+
runtime: "muse_code",
|
|
382
|
+
cap: { providerResetAt: "2026-10-12T19:05:00.000Z", limitType: null },
|
|
383
|
+
windows: pair,
|
|
384
|
+
expected: null,
|
|
385
|
+
},
|
|
386
|
+
{
|
|
387
|
+
name: "Cursor reset match on the billing cycle",
|
|
388
|
+
runtime: "cursor_local",
|
|
389
|
+
cap: { providerResetAt: "2026-11-01T00:00:00.000Z", limitType: null },
|
|
390
|
+
windows: [tagged("billing_cycle", null, "2026-11-01T00:00:00.000Z")],
|
|
391
|
+
expected: "billing_cycle",
|
|
392
|
+
},
|
|
393
|
+
{
|
|
394
|
+
name: "Untagged diagnostic window with a matching reset never maps",
|
|
395
|
+
runtime: "muse_code",
|
|
396
|
+
cap: { providerResetAt: "2026-10-13T00:00:00.000Z", limitType: null },
|
|
397
|
+
windows: [...pair, tagged("muse_account_usage", null, "2026-10-13T00:00:00.000Z")],
|
|
398
|
+
expected: null,
|
|
399
|
+
},
|
|
400
|
+
{
|
|
401
|
+
name: "Fallback cooldown (no provider reset) never maps",
|
|
402
|
+
runtime: "codex_local",
|
|
403
|
+
cap: { providerResetAt: null, limitType: null },
|
|
404
|
+
windows: pair,
|
|
405
|
+
expected: null,
|
|
406
|
+
},
|
|
407
|
+
{
|
|
408
|
+
name: "Ambiguous reset matching both pair windows stays unmapped",
|
|
409
|
+
runtime: "codex_local",
|
|
410
|
+
cap: { providerResetAt: NOT440_RESET, limitType: null },
|
|
411
|
+
windows: [tagged("five_hour", "five_hour", NOT440_RESET), tagged("weekly", "weekly", NOT440_RESET)],
|
|
412
|
+
expected: null,
|
|
413
|
+
},
|
|
414
|
+
];
|
|
415
|
+
for (const c of cases) {
|
|
416
|
+
assert.equal(resolveCapWindowRole(c.runtime, c.cap, c.windows), c.expected, c.name);
|
|
417
|
+
}
|
|
418
|
+
});
|
|
419
|
+
test("NOT-440: Claude structured rate_limit_event persists limit type and maps through the read model", async () => {
|
|
420
|
+
clearAllCapacitySnapshots();
|
|
421
|
+
clearAllRuntimeAvailability();
|
|
422
|
+
await seedPair("claude_code", [
|
|
423
|
+
{ windowKey: "five_hour", criticalRole: "five_hour", remainingPercent: 40, resetAt: NOT440_RESET },
|
|
424
|
+
{ windowKey: "weekly", criticalRole: "weekly", remainingPercent: 70, resetAt: NOT440_WEEKLY_RESET },
|
|
425
|
+
]);
|
|
426
|
+
const { recordUsageCapFromEvents } = await import("../runners/usage-cap.js");
|
|
427
|
+
recordUsageCapFromEvents([
|
|
428
|
+
{
|
|
429
|
+
type: "rate_limit_event",
|
|
430
|
+
rate_limit_info: {
|
|
431
|
+
status: "rejected",
|
|
432
|
+
rateLimitType: "seven_day",
|
|
433
|
+
resetsAt: Math.floor(Date.parse(NOT440_WEEKLY_RESET) / 1000),
|
|
434
|
+
},
|
|
435
|
+
},
|
|
436
|
+
], "claude_code", NOT440_NOW);
|
|
437
|
+
const claude = getRuntimeCapacitySnapshot(NOT440_NOW).runtimes.find((r) => r.runtime === "claude_code");
|
|
438
|
+
assert.equal(claude.activeCap.windowRole, "weekly");
|
|
439
|
+
assert.equal(claude.activeCap.unavailableUntil, NOT440_WEEKLY_RESET);
|
|
440
|
+
clearAllRuntimeAvailability();
|
|
441
|
+
});
|
|
@@ -378,6 +378,14 @@ export function migrate() {
|
|
|
378
378
|
);
|
|
379
379
|
`);
|
|
380
380
|
}
|
|
381
|
+
// NOT-440: normalized cap metadata for capacity reconciliation.
|
|
382
|
+
const runtimeAvailCols = db.prepare("PRAGMA table_info(runtime_availability)").all();
|
|
383
|
+
if (!runtimeAvailCols.some((c) => c.name === "provider_reset_at")) {
|
|
384
|
+
db.exec("ALTER TABLE runtime_availability ADD COLUMN provider_reset_at TEXT");
|
|
385
|
+
}
|
|
386
|
+
if (!runtimeAvailCols.some((c) => c.name === "limit_type")) {
|
|
387
|
+
db.exec("ALTER TABLE runtime_availability ADD COLUMN limit_type TEXT");
|
|
388
|
+
}
|
|
381
389
|
// NOT-245: normalized capacity snapshots for databases created before
|
|
382
390
|
// schema.sql declared the table (same upgrade-path pattern as queue_entries).
|
|
383
391
|
const capacitySnapshots = db
|
|
@@ -473,7 +473,11 @@ CREATE TABLE IF NOT EXISTS runtime_availability (
|
|
|
473
473
|
unavailable_until TEXT NOT NULL,
|
|
474
474
|
reason TEXT NOT NULL,
|
|
475
475
|
evidence_json TEXT,
|
|
476
|
-
observed_at TEXT NOT NULL
|
|
476
|
+
observed_at TEXT NOT NULL,
|
|
477
|
+
-- NOT-440: provider-reported reset (null when unavailable_until is the
|
|
478
|
+
-- fallback cooldown) and structured provider limit type, if any.
|
|
479
|
+
provider_reset_at TEXT,
|
|
480
|
+
limit_type TEXT
|
|
477
481
|
);
|
|
478
482
|
|
|
479
483
|
-- NOT-171: append-only normalized failure-cause evidence. A derived view over
|
|
@@ -2,19 +2,23 @@
|
|
|
2
2
|
//
|
|
3
3
|
// NOT-111: durable per-runtime (account-level) usage-cap state. Runners write when they
|
|
4
4
|
// observe a hard cap; coordinator and queue admission read via runtimeAvailability().
|
|
5
|
+
// NOT-440: the capacity read model reads the same row via activeRuntimeCap().
|
|
5
6
|
import { getDb } from "../db/index.js";
|
|
6
7
|
/** Upsert the latest cap observation for a runtime (one row per runtime). */
|
|
7
8
|
export function recordRuntimeAvailability(input) {
|
|
8
9
|
const now = new Date().toISOString();
|
|
9
10
|
getDb()
|
|
10
11
|
.prepare(`
|
|
11
|
-
INSERT INTO runtime_availability
|
|
12
|
-
|
|
12
|
+
INSERT INTO runtime_availability
|
|
13
|
+
(runtime, unavailable_until, reason, evidence_json, observed_at, provider_reset_at, limit_type)
|
|
14
|
+
VALUES (@runtime, @unavailable_until, @reason, @evidence_json, @observed_at, @provider_reset_at, @limit_type)
|
|
13
15
|
ON CONFLICT(runtime) DO UPDATE SET
|
|
14
16
|
unavailable_until = excluded.unavailable_until,
|
|
15
17
|
reason = excluded.reason,
|
|
16
18
|
evidence_json = excluded.evidence_json,
|
|
17
|
-
observed_at = excluded.observed_at
|
|
19
|
+
observed_at = excluded.observed_at,
|
|
20
|
+
provider_reset_at = excluded.provider_reset_at,
|
|
21
|
+
limit_type = excluded.limit_type
|
|
18
22
|
`)
|
|
19
23
|
.run({
|
|
20
24
|
runtime: input.runtime,
|
|
@@ -22,6 +26,8 @@ export function recordRuntimeAvailability(input) {
|
|
|
22
26
|
reason: input.reason,
|
|
23
27
|
evidence_json: input.evidence !== undefined ? JSON.stringify(input.evidence) : null,
|
|
24
28
|
observed_at: now,
|
|
29
|
+
provider_reset_at: input.providerResetAt ?? null,
|
|
30
|
+
limit_type: input.limitType ?? null,
|
|
25
31
|
});
|
|
26
32
|
}
|
|
27
33
|
/** Clear a runtime row once its unavailable_until has passed (lazy cleanup on read). */
|
|
@@ -30,23 +36,35 @@ function clearExpired(runtime, nowMs = Date.now()) {
|
|
|
30
36
|
.prepare("DELETE FROM runtime_availability WHERE runtime = ? AND unavailable_until <= ?")
|
|
31
37
|
.run(runtime, new Date(nowMs).toISOString());
|
|
32
38
|
}
|
|
33
|
-
/**
|
|
34
|
-
|
|
35
|
-
* Returns `{ available: true }` when no active cap is recorded.
|
|
36
|
-
*/
|
|
37
|
-
export function runtimeAvailability(runtime, nowMs = Date.now()) {
|
|
39
|
+
/** The active cap row for a runtime, or null once it has expired (lazily cleared). */
|
|
40
|
+
export function activeRuntimeCap(runtime, nowMs = Date.now()) {
|
|
38
41
|
clearExpired(runtime, nowMs);
|
|
39
42
|
const row = getDb()
|
|
40
43
|
.prepare("SELECT * FROM runtime_availability WHERE runtime = ?")
|
|
41
44
|
.get(runtime);
|
|
42
45
|
if (!row)
|
|
43
|
-
return
|
|
46
|
+
return null;
|
|
44
47
|
const untilMs = Date.parse(row.unavailable_until);
|
|
45
48
|
if (!Number.isFinite(untilMs) || untilMs <= nowMs) {
|
|
46
49
|
getDb().prepare("DELETE FROM runtime_availability WHERE runtime = ?").run(runtime);
|
|
47
|
-
return
|
|
50
|
+
return null;
|
|
48
51
|
}
|
|
49
|
-
return {
|
|
52
|
+
return {
|
|
53
|
+
unavailableUntil: row.unavailable_until,
|
|
54
|
+
reason: row.reason,
|
|
55
|
+
providerResetAt: row.provider_reset_at,
|
|
56
|
+
limitType: row.limit_type,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Single read function for coordinator deferral and NOT-103 queue admission.
|
|
61
|
+
* Returns `{ available: true }` when no active cap is recorded.
|
|
62
|
+
*/
|
|
63
|
+
export function runtimeAvailability(runtime, nowMs = Date.now()) {
|
|
64
|
+
const cap = activeRuntimeCap(runtime, nowMs);
|
|
65
|
+
if (!cap)
|
|
66
|
+
return { available: true };
|
|
67
|
+
return { available: false, until: cap.unavailableUntil, reason: cap.reason };
|
|
50
68
|
}
|
|
51
69
|
/** Test helper — wipe all cap rows. */
|
|
52
70
|
export function clearAllRuntimeAvailability() {
|
|
@@ -23,9 +23,46 @@ function parseResetsAt(raw) {
|
|
|
23
23
|
const d = new Date(ms);
|
|
24
24
|
return Number.isFinite(d.getTime()) ? d : null;
|
|
25
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* Provider reset timestamps embedded in already-classified cap-error text
|
|
28
|
+
* (NOT-439). ISO-8601 only (`2026-10-12T18:59:01Z`, optional fractional
|
|
29
|
+
* seconds or numeric offset), and only when introduced by reset phrasing
|
|
30
|
+
* ("resets at", "reset_at:", "until", "retry after") so unrelated stamps in
|
|
31
|
+
* the same stderr/log text are ignored. Never consulted to decide whether a
|
|
32
|
+
* cap exists, only to place the boundary of one the cap classifier accepted.
|
|
33
|
+
*/
|
|
34
|
+
const RESET_TIMESTAMP_RE = /\b(?:resets?(?:_at)?|until|retry\s+after)\b[^\d\n]{0,20}(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2}))/gi;
|
|
35
|
+
/**
|
|
36
|
+
* Shared reset extractor for every hard-cap path (NOT-439): structured
|
|
37
|
+
* numeric seconds/milliseconds first, then provider reset timestamps in
|
|
38
|
+
* already-classified cap-error text. Returns the normalized ISO-8601 instant
|
|
39
|
+
* of the first finite future reset, or null when no trustworthy future reset
|
|
40
|
+
* is present (absent, past, or malformed). Never creates a cap by itself —
|
|
41
|
+
* callers invoke it only after their existing cap classifier has matched.
|
|
42
|
+
*/
|
|
43
|
+
export function extractUsageCapReset(input, nowMs = Date.now()) {
|
|
44
|
+
for (const raw of input.resets ?? []) {
|
|
45
|
+
const d = parseResetsAt(raw);
|
|
46
|
+
if (d && d.getTime() > nowMs)
|
|
47
|
+
return d.toISOString();
|
|
48
|
+
}
|
|
49
|
+
if (typeof input.text === "string" && input.text) {
|
|
50
|
+
for (const [, candidate] of input.text.matchAll(RESET_TIMESTAMP_RE)) {
|
|
51
|
+
const ms = Date.parse(candidate);
|
|
52
|
+
if (Number.isFinite(ms) && ms > nowMs)
|
|
53
|
+
return new Date(ms).toISOString();
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
26
58
|
function fallbackUntil(nowMs = Date.now()) {
|
|
27
59
|
return new Date(nowMs + usageCapFallbackCooldownMs()).toISOString();
|
|
28
60
|
}
|
|
61
|
+
/** Provider reset when trustworthy, otherwise the configured fallback cooldown. */
|
|
62
|
+
function resolveUnavailableUntil(input, nowMs) {
|
|
63
|
+
const providerResetAt = extractUsageCapReset(input, nowMs);
|
|
64
|
+
return { unavailableUntil: providerResetAt ?? fallbackUntil(nowMs), providerResetAt };
|
|
65
|
+
}
|
|
29
66
|
function capReason(runtime, detail) {
|
|
30
67
|
return `${runtime} usage capped — ${detail}`;
|
|
31
68
|
}
|
|
@@ -38,16 +75,16 @@ function isHardCapRateLimitInfo(info) {
|
|
|
38
75
|
function signalFromRateLimitInfo(runtime, info, evidence, nowMs) {
|
|
39
76
|
if (!isHardCapRateLimitInfo(info))
|
|
40
77
|
return null;
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
const
|
|
46
|
-
const limitType = String(info.rateLimitType ?? info.rate_limit_type ?? "plan");
|
|
78
|
+
const until = resolveUnavailableUntil({
|
|
79
|
+
resets: [info.resetsAt, info.overageResetsAt, info.overage_resets_at, info.resets_at],
|
|
80
|
+
}, nowMs);
|
|
81
|
+
const rawLimitType = info.rateLimitType ?? info.rate_limit_type;
|
|
82
|
+
const limitType = typeof rawLimitType === "string" && rawLimitType ? rawLimitType : null;
|
|
47
83
|
return {
|
|
48
|
-
|
|
49
|
-
reason: capReason(runtime, `${limitType} limit rejected`),
|
|
84
|
+
...until,
|
|
85
|
+
reason: capReason(runtime, `${limitType ?? "plan"} limit rejected`),
|
|
50
86
|
evidence: { ...evidence, rate_limit_info: info },
|
|
87
|
+
limitType,
|
|
51
88
|
};
|
|
52
89
|
}
|
|
53
90
|
function signalFromApiRetry(e, runtime, nowMs) {
|
|
@@ -64,6 +101,8 @@ function signalFromApiRetry(e, runtime, nowMs) {
|
|
|
64
101
|
unavailableUntil: fallbackUntil(nowMs),
|
|
65
102
|
reason: capReason(runtime, err.replace("_", " ")),
|
|
66
103
|
evidence: { event: e },
|
|
104
|
+
providerResetAt: null,
|
|
105
|
+
limitType: null,
|
|
67
106
|
};
|
|
68
107
|
}
|
|
69
108
|
function signalFromResultError(e, runtime, nowMs) {
|
|
@@ -73,7 +112,8 @@ function signalFromResultError(e, runtime, nowMs) {
|
|
|
73
112
|
if (!CAP_ERROR_RE.test(text))
|
|
74
113
|
return null;
|
|
75
114
|
return {
|
|
76
|
-
|
|
115
|
+
...resolveUnavailableUntil({ text }, nowMs),
|
|
116
|
+
limitType: null,
|
|
77
117
|
reason: capReason(runtime, "session ended with cap error"),
|
|
78
118
|
evidence: { result: text.slice(0, 500) },
|
|
79
119
|
};
|
|
@@ -113,7 +153,8 @@ function codexCapFromEvents(events, runtime, nowMs) {
|
|
|
113
153
|
}
|
|
114
154
|
if (CAP_ERROR_RE.test(msg)) {
|
|
115
155
|
return {
|
|
116
|
-
|
|
156
|
+
...resolveUnavailableUntil({ text: msg }, nowMs),
|
|
157
|
+
limitType: null,
|
|
117
158
|
reason: capReason(runtime, msg.slice(0, 120) || "turn failed"),
|
|
118
159
|
evidence: { turn_failed: err },
|
|
119
160
|
};
|
|
@@ -121,7 +162,8 @@ function codexCapFromEvents(events, runtime, nowMs) {
|
|
|
121
162
|
}
|
|
122
163
|
if (e.type === "error" && typeof e.message === "string" && CAP_ERROR_RE.test(e.message)) {
|
|
123
164
|
return {
|
|
124
|
-
|
|
165
|
+
...resolveUnavailableUntil({ text: e.message }, nowMs),
|
|
166
|
+
limitType: null,
|
|
125
167
|
reason: capReason(runtime, e.message.slice(0, 120)),
|
|
126
168
|
evidence: { error: e.message },
|
|
127
169
|
};
|
|
@@ -198,7 +240,8 @@ export function detectUsageCapFromRawLog(raw, runtime, nowMs = Date.now()) {
|
|
|
198
240
|
const haystack = textFallbackHaystack(raw, events);
|
|
199
241
|
if (CAP_FALLBACK_RE.test(haystack)) {
|
|
200
242
|
return {
|
|
201
|
-
|
|
243
|
+
...resolveUnavailableUntil({ text: haystack }, nowMs),
|
|
244
|
+
limitType: null,
|
|
202
245
|
reason: capReason(runtime, "stderr/log matched usage-cap pattern"),
|
|
203
246
|
evidence: { matched: haystack.slice(0, 300) },
|
|
204
247
|
};
|
|
@@ -221,6 +264,8 @@ export function recordUsageCapFromLog(logPath, runtime, nowMs = Date.now()) {
|
|
|
221
264
|
unavailableUntil: cap.unavailableUntil,
|
|
222
265
|
reason: cap.reason,
|
|
223
266
|
evidence: cap.evidence,
|
|
267
|
+
providerResetAt: cap.providerResetAt,
|
|
268
|
+
limitType: cap.limitType,
|
|
224
269
|
});
|
|
225
270
|
return cap;
|
|
226
271
|
}
|
|
@@ -236,19 +281,23 @@ export function recordUsageCapFromEvents(events, runtime, nowMs = Date.now()) {
|
|
|
236
281
|
unavailableUntil: cap.unavailableUntil,
|
|
237
282
|
reason: cap.reason,
|
|
238
283
|
evidence: cap.evidence,
|
|
284
|
+
providerResetAt: cap.providerResetAt,
|
|
285
|
+
limitType: cap.limitType,
|
|
239
286
|
});
|
|
240
287
|
return cap;
|
|
241
288
|
}
|
|
242
289
|
/**
|
|
243
290
|
* NOT-181: Muse's usage-cap signal is the parsed failure kind (runners/muse-code-jsonl.ts), not a
|
|
244
|
-
* Claude-shaped log event.
|
|
245
|
-
*
|
|
291
|
+
* Claude-shaped log event. NOT-439: the 2026-10-12 incident proved Muse does report a reset
|
|
292
|
+
* timestamp in the failure message (`... resets at 2026-10-12T18:59:01Z`), so the deferral
|
|
293
|
+
* preserves it and uses the fallback cooldown only when no trustworthy future reset is present.
|
|
246
294
|
*/
|
|
247
295
|
export function recordMuseUsageCap(failure, nowMs = Date.now()) {
|
|
248
296
|
if (failure?.kind !== "usage_cap")
|
|
249
297
|
return null;
|
|
250
298
|
const cap = {
|
|
251
|
-
|
|
299
|
+
...resolveUnavailableUntil({ text: failure.message }, nowMs),
|
|
300
|
+
limitType: null,
|
|
252
301
|
reason: capReason("muse_code", failure.message.slice(0, 120) || "usage cap"),
|
|
253
302
|
evidence: { muse_failure: failure },
|
|
254
303
|
};
|
|
@@ -257,6 +306,8 @@ export function recordMuseUsageCap(failure, nowMs = Date.now()) {
|
|
|
257
306
|
unavailableUntil: cap.unavailableUntil,
|
|
258
307
|
reason: cap.reason,
|
|
259
308
|
evidence: cap.evidence,
|
|
309
|
+
providerResetAt: cap.providerResetAt,
|
|
310
|
+
limitType: cap.limitType,
|
|
260
311
|
});
|
|
261
312
|
return cap;
|
|
262
313
|
}
|