limn-review 0.1.172 → 0.1.174
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai/providers/codex.js +14 -3
- package/dist/ai/usage-lockout.js +108 -0
- package/dist/api/routes/me.js +4 -0
- package/dist/db/erase-account.js +4 -0
- package/dist/db/migrations/0159_team_claims_default_on.sql +21 -0
- package/dist/db/migrations/meta/_journal.json +7 -0
- package/dist/db/schema.sqlite.js +22 -0
- package/dist/db/team-claim-lines-delete.js +3 -0
- package/dist/review/budget-gate.js +9 -1
- package/dist/review/claude-review/auto-retry.js +10 -0
- package/dist/review/claude-review/auto-settings.js +6 -5
- package/dist/review/claude-review/auto.js +11 -1
- package/dist/review/claude-review/manager.js +121 -20
- package/dist/review/claude-review/persist.js +25 -0
- package/dist/review/claude-review/routes.js +4 -0
- package/dist/review/claude-review/run-watchdog.js +50 -0
- package/dist/review/claude-review/team-claim-release.js +22 -5
- package/dist/review/claude-review/team-claim-seen.js +76 -0
- package/dist/review/claude-review/team-claim.js +63 -28
- package/package.json +1 -1
- package/public/assets/{AiFixTab-dmX-usAV.js → AiFixTab-B7AIpa7v.js} +1 -1
- package/public/assets/ClaudeReviewTab-BWIpd_Ft.js +20 -0
- package/public/assets/{PrBotBehaviourTab-CRKe6flL.js → PrBotBehaviourTab-EgZ8mnoW.js} +1 -1
- package/public/assets/index-AIhwoS_J.css +1 -0
- package/public/assets/index-Bcjt3Gbz.js +87 -0
- package/public/index.html +2 -2
- package/public/assets/ClaudeReviewTab-C2Aisxkx.js +0 -20
- package/public/assets/index-CFH1_ouL.js +0 -87
- package/public/assets/index-Cwdx3tuB.css +0 -1
|
@@ -57,6 +57,7 @@ import { confirmCodexAccount, recheckCodexAccount } from '../codex/app-server.js
|
|
|
57
57
|
import { CODEX_API_KEY_NOT_AVAILABLE_MESSAGE, CODEX_SIGN_IN_UNCHECKED_MESSAGE, codexUsageSource, detectCodex, } from '../codex/detect.js';
|
|
58
58
|
import { addUsage, cleanCodexError, codexTokensAsAgentTokens, progressFromEvent, stepFromEvent, stderrRefusals, stderrTail, usageDelta, } from '../codex/events.js';
|
|
59
59
|
import { pathGuardHook, verifyHookTrusted } from '../codex/hook.js';
|
|
60
|
+
import { noteProviderFailure } from '../usage-lockout.js';
|
|
60
61
|
import { codexBinary, codexHome, ensureCodexHomeConfig } from '../codex/home.js';
|
|
61
62
|
import { armCodexThreadSweeps, queueCodexThreadCleanup } from '../codex/threads.js';
|
|
62
63
|
import { runCodexExec } from '../codex/run.js';
|
|
@@ -150,6 +151,16 @@ export function codexExecEnv(credential, hook, base = process.env) {
|
|
|
150
151
|
// ---- shared run plumbing ------------------------------------------------------------------------
|
|
151
152
|
const DEFAULT_AGENT_TIMEOUT_MS = 45 * 60_000;
|
|
152
153
|
const DEFAULT_COMPLETE_TIMEOUT_MS = 5 * 60_000;
|
|
154
|
+
/**
|
|
155
|
+
* `execFailure`, and a usage limit noted as a LOCKOUT (ai/usage-lockout.ts): every automatic run on
|
|
156
|
+
* Codex stands down until the reset the error names. Every run kind's failure passes through here.
|
|
157
|
+
*/
|
|
158
|
+
function failureOf(res) {
|
|
159
|
+
const failure = execFailure(res);
|
|
160
|
+
if (failure)
|
|
161
|
+
noteProviderFailure('codex', failure);
|
|
162
|
+
return failure;
|
|
163
|
+
}
|
|
153
164
|
/** Why a finished `codex exec` did not produce an answer, as one sentence; null when it did. */
|
|
154
165
|
export function execFailure(res) {
|
|
155
166
|
if (res.cancelled)
|
|
@@ -295,7 +306,7 @@ async function complete(req) {
|
|
|
295
306
|
finally {
|
|
296
307
|
queueCodexThreadCleanup(home, res.threadId);
|
|
297
308
|
}
|
|
298
|
-
const failure =
|
|
309
|
+
const failure = failureOf(res);
|
|
299
310
|
if (failure)
|
|
300
311
|
throw new CodexRunError(failure, res.cancelled);
|
|
301
312
|
const text = res.finalResponse.trim();
|
|
@@ -547,7 +558,7 @@ export async function runAgentUnchecked(spec) {
|
|
|
547
558
|
first = await runCodexExec(args, { cwd, env, signal, onEvent, stdin: spec.prompt, timeoutMs });
|
|
548
559
|
final = first;
|
|
549
560
|
usage = first.usage;
|
|
550
|
-
failure =
|
|
561
|
+
failure = failureOf(first);
|
|
551
562
|
if (!failure && resultTool && toolSchema) {
|
|
552
563
|
let verdict = await judge(z, resultTool, toolSchema, first.finalResponse);
|
|
553
564
|
if (!verdict.ok) {
|
|
@@ -569,7 +580,7 @@ export async function runAgentUnchecked(spec) {
|
|
|
569
580
|
if (retry.usage)
|
|
570
581
|
usage = addUsage(usage, usageDelta(retry.usage, first.usage));
|
|
571
582
|
final = retry;
|
|
572
|
-
failure =
|
|
583
|
+
failure = failureOf(retry);
|
|
573
584
|
if (!failure) {
|
|
574
585
|
verdict = await judge(z, resultTool, toolSchema, retry.finalResponse);
|
|
575
586
|
if (!verdict.ok)
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { parseRetryAtMs } from '../review/claude-review/auto-retry.js';
|
|
2
|
+
/** How long a usage limit locks a provider out when its error names no reset time. */
|
|
3
|
+
export const LOCKOUT_DEFAULT_MS = 60 * 60 * 1000;
|
|
4
|
+
/** A lock never runs longer than this (a misread date must not lock a provider out for a month). */
|
|
5
|
+
export const LOCKOUT_MAX_MS = 8 * 24 * 60 * 60 * 1000;
|
|
6
|
+
// ⚠ NARROWER than auto-retry's USAGE_LIMIT: a plain rate limit, a 429 or "too many requests" is a
|
|
7
|
+
// short throttle, not a lockout. Only the account's own usage allowance being used up locks it out.
|
|
8
|
+
// Linear (no nested quantifiers): it runs over provider text.
|
|
9
|
+
const LOCKOUT = /usage limit|hit your (?!rate)[\w'’ ]{0,30}limit|out of (?:usage )?credits|quota exceeded/i;
|
|
10
|
+
/** Does this error say the provider's usage allowance is used up? */
|
|
11
|
+
export function isUsageLockoutError(error) {
|
|
12
|
+
const e = (error ?? '').slice(0, 2000);
|
|
13
|
+
return e.trim() !== '' && LOCKOUT.test(e);
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* When the lockout this error describes ends (ms), or null when the error is not a usage limit.
|
|
17
|
+
* The reset the error names when it names one (never before the failure), else the default.
|
|
18
|
+
*/
|
|
19
|
+
export function lockoutUntilFromError(error, failedAtMs) {
|
|
20
|
+
if (!isUsageLockoutError(error))
|
|
21
|
+
return null;
|
|
22
|
+
const named = parseRetryAtMs(error, failedAtMs);
|
|
23
|
+
const until = named != null && named > failedAtMs ? named : failedAtMs + LOCKOUT_DEFAULT_MS;
|
|
24
|
+
return Math.min(until, failedAtMs + LOCKOUT_MAX_MS);
|
|
25
|
+
}
|
|
26
|
+
const locks = new Map();
|
|
27
|
+
/**
|
|
28
|
+
* Record a failure of `provider` when it is a usage limit (any other error is ignored). A later
|
|
29
|
+
* reset replaces an earlier one; an earlier one never shortens a lock already held.
|
|
30
|
+
*/
|
|
31
|
+
export function noteProviderFailure(provider, error, atMs = Date.now()) {
|
|
32
|
+
const until = lockoutUntilFromError(error, atMs);
|
|
33
|
+
if (until == null)
|
|
34
|
+
return;
|
|
35
|
+
const held = locks.get(provider);
|
|
36
|
+
if (!held || held.untilMs < until)
|
|
37
|
+
locks.set(provider, { untilMs: until });
|
|
38
|
+
}
|
|
39
|
+
/** The provider's lockout now, or null (an ended lock is forgotten). */
|
|
40
|
+
export function usageLockout(provider, nowMs = Date.now()) {
|
|
41
|
+
const held = locks.get(provider);
|
|
42
|
+
if (!held)
|
|
43
|
+
return null;
|
|
44
|
+
if (held.untilMs <= nowMs) {
|
|
45
|
+
locks.delete(provider);
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
return held;
|
|
49
|
+
}
|
|
50
|
+
/** Is any provider locked out now? (cheap: lets a caller skip resolving the default provider). */
|
|
51
|
+
export function anyUsageLockout(nowMs = Date.now()) {
|
|
52
|
+
for (const p of [...locks.keys()])
|
|
53
|
+
if (usageLockout(p, nowMs))
|
|
54
|
+
return true;
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
export function isLockedOut(provider, nowMs = Date.now()) {
|
|
58
|
+
return usageLockout(provider, nowMs) != null;
|
|
59
|
+
}
|
|
60
|
+
/** Forget a lock (a new sign-in) — and the test hook (no provider ⇒ all). */
|
|
61
|
+
export function clearUsageLockout(provider) {
|
|
62
|
+
if (provider)
|
|
63
|
+
locks.delete(provider);
|
|
64
|
+
else
|
|
65
|
+
locks.clear();
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* WHICH PROVIDERS AN AUTOMATIC RUN USES (pure). `pair`: the workspace asks for Claude + Codex.
|
|
69
|
+
* • a pair ⇒ every provider in `order` that is ready and not locked out (two ⇒ a pair, one ⇒ alone);
|
|
70
|
+
* • a single run ⇒ the default, unless it is LOCKED OUT: then the other provider when it is ready
|
|
71
|
+
* and not locked, else none (skipped). A default that is merely not ready is the caller's
|
|
72
|
+
* existing wait (it is not moved).
|
|
73
|
+
* Empty ⇒ do not run now.
|
|
74
|
+
*/
|
|
75
|
+
export function automaticRunProviders(opts) {
|
|
76
|
+
const { defaultProvider, pair, order, ready, locked } = opts;
|
|
77
|
+
if (pair)
|
|
78
|
+
return order.filter((p) => ready(p) && !locked(p));
|
|
79
|
+
if (!locked(defaultProvider))
|
|
80
|
+
return [defaultProvider];
|
|
81
|
+
const other = order.find((p) => p !== defaultProvider && ready(p) && !locked(p));
|
|
82
|
+
return other ? [other] : [];
|
|
83
|
+
}
|
|
84
|
+
/** The SPA's one line (`MeResponse.ai.providers.<p>.usageLockedUntil`), as ISO-8601, or null. */
|
|
85
|
+
export function usageLockedUntilIso(provider, nowMs = Date.now()) {
|
|
86
|
+
const l = usageLockout(provider, nowMs);
|
|
87
|
+
return l ? new Date(l.untilMs).toISOString() : null;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* THE BOOT'S RE-DERIVATION: the newest failed review row per provider, from the last
|
|
91
|
+
* LOCKOUT_MAX_MS. Each is noted at its finish time (a lock already over is not kept). Never throws.
|
|
92
|
+
*/
|
|
93
|
+
export async function restoreUsageLockouts(read) {
|
|
94
|
+
try {
|
|
95
|
+
for (const r of await read()) {
|
|
96
|
+
if (r.provider !== 'claude' && r.provider !== 'codex')
|
|
97
|
+
continue;
|
|
98
|
+
if (r.finishedAtMs == null)
|
|
99
|
+
continue;
|
|
100
|
+
noteProviderFailure(r.provider, r.error, r.finishedAtMs);
|
|
101
|
+
usageLockout(r.provider); // drops it at once when it has already ended
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
/* a lock is an optimisation: never fails a boot */
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=usage-lockout.js.map
|
package/dist/api/routes/me.js
CHANGED
|
@@ -12,6 +12,7 @@ import { getAiRuntimeStatus } from '../../ai/runtime.js';
|
|
|
12
12
|
import { codexKindStatus, detectClaudeAuth, NO_CLAUDE_AUTH_MESSAGE } from '../../review/auth.js';
|
|
13
13
|
import { resolveAiCredential } from '../../ai/credentials.js';
|
|
14
14
|
import { codexDetailCached } from '../../ai/codex/detect.js';
|
|
15
|
+
import { usageLockedUntilIso } from '../../ai/usage-lockout.js';
|
|
15
16
|
import { aiEffortSettingStatus } from '../../ai/effort.js';
|
|
16
17
|
import { aiProviderSettingStatus, ALL_RUN_KINDS, CODEX_NOT_AVAILABLE_MESSAGE, CODEX_RUNNABLE, getActiveProvider, } from '../../ai/provider.js';
|
|
17
18
|
import { getMyTurn } from '../../db/queries.js';
|
|
@@ -226,6 +227,8 @@ function agenticAiStatus() {
|
|
|
226
227
|
// models only when it is ready and signed in (the per-run choice, ai/run-choice.ts).
|
|
227
228
|
runtime: claudeRt.runtime,
|
|
228
229
|
runtimeMessage: claudeRt.runtimeMessage,
|
|
230
|
+
// A usage limit's lockout (ai/usage-lockout.ts): the Run area's one line.
|
|
231
|
+
usageLockedUntil: usageLockedUntilIso('claude'),
|
|
229
232
|
},
|
|
230
233
|
codex: {
|
|
231
234
|
runtime: codex.runtime,
|
|
@@ -245,6 +248,7 @@ function agenticAiStatus() {
|
|
|
245
248
|
const r = codexKindStatus(kind);
|
|
246
249
|
return r.status === 'ok' ? { auth: 'ok', message: null } : { auth: 'none', message: r.message };
|
|
247
250
|
}),
|
|
251
|
+
usageLockedUntil: usageLockedUntilIso('codex'),
|
|
248
252
|
},
|
|
249
253
|
},
|
|
250
254
|
};
|
package/dist/db/erase-account.js
CHANGED
|
@@ -175,6 +175,8 @@ export async function eraseAccountData(accountId) {
|
|
|
175
175
|
await tx.delete(schema.sonarSnapshots).where(eq(schema.sonarSnapshots.accountId, accountId)).execute();
|
|
176
176
|
// The team-claim ledger (migration 0138): GitHub logins and heads. No FK, so this is the whole guarantee.
|
|
177
177
|
await tx.delete(schema.teamClaimLines).where(eq(schema.teamClaimLines.accountId, accountId)).execute();
|
|
178
|
+
// The teammate lines the AI Review tab names (migration 0159): GitHub logins. No FK.
|
|
179
|
+
await tx.delete(schema.teamClaimSeen).where(eq(schema.teamClaimSeen.accountId, accountId)).execute();
|
|
178
180
|
// The worktree ledger (migration 0146): which PR / ticket a checkout on disk was for. No FK. The
|
|
179
181
|
// directories themselves become unledgered orphans the worktree sweep collects by age.
|
|
180
182
|
await tx.delete(schema.worktreeLedger).where(eq(schema.worktreeLedger.accountId, accountId)).execute();
|
|
@@ -366,6 +368,8 @@ export function accountScopedTables() {
|
|
|
366
368
|
{ name: 'sonarSnapshots', col: schema.sonarSnapshots.accountId, table: schema.sonarSnapshots },
|
|
367
369
|
// The team-claim ledger (migration 0138), erased explicitly above.
|
|
368
370
|
{ name: 'teamClaimLines', col: schema.teamClaimLines.accountId, table: schema.teamClaimLines },
|
|
371
|
+
// The teammate lines (migration 0159), erased explicitly above.
|
|
372
|
+
{ name: 'teamClaimSeen', col: schema.teamClaimSeen.accountId, table: schema.teamClaimSeen },
|
|
369
373
|
// The worktree ledger (migration 0146), erased explicitly above.
|
|
370
374
|
{ name: 'worktreeLedger', col: schema.worktreeLedger.accountId, table: schema.worktreeLedger },
|
|
371
375
|
{ name: 'workspaceSonar', col: schema.workspaceSonar.accountId, table: schema.workspaceSonar },
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
-- TEAM CLAIMS ON BY DEFAULT (docs/CLAUDE-REVIEW.md § Team claims, 2026-11-02). The owner: "Just on
|
|
2
|
+
-- by default on all workspaces". Every existing workspace is switched ON (also one switched off
|
|
3
|
+
-- before); a new workspace's NULL now reads as ON (only a stored false is off).
|
|
4
|
+
UPDATE `workspaces` SET `team_claims_enabled` = 1;
|
|
5
|
+
--> statement-breakpoint
|
|
6
|
+
-- A TEAMMATE'S LINE THIS LIMN STOOD DOWN FOR: one row per (account, PR, head, provider), shown on the
|
|
7
|
+
-- AI Review tab ("Robin Dunn reviewed this commit with Claude · no comments").
|
|
8
|
+
CREATE TABLE `team_claim_seen` (
|
|
9
|
+
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
|
10
|
+
`account_id` integer NOT NULL,
|
|
11
|
+
`pr_id` integer NOT NULL,
|
|
12
|
+
`head_sha` text NOT NULL,
|
|
13
|
+
`provider` text NOT NULL,
|
|
14
|
+
`kind` text NOT NULL,
|
|
15
|
+
`login` text NOT NULL,
|
|
16
|
+
`description` text,
|
|
17
|
+
`line_at` integer NOT NULL,
|
|
18
|
+
`updated_at` integer DEFAULT (unixepoch()) NOT NULL
|
|
19
|
+
);
|
|
20
|
+
--> statement-breakpoint
|
|
21
|
+
CREATE UNIQUE INDEX `team_claim_seen_account_pr_head_provider` ON `team_claim_seen` (`account_id`,`pr_id`,`head_sha`,`provider`);
|
|
@@ -1114,6 +1114,13 @@
|
|
|
1114
1114
|
"when": 1792224061016,
|
|
1115
1115
|
"tag": "0158_repo_docs_unreadable",
|
|
1116
1116
|
"breakpoints": true
|
|
1117
|
+
},
|
|
1118
|
+
{
|
|
1119
|
+
"idx": 159,
|
|
1120
|
+
"version": "6",
|
|
1121
|
+
"when": 1792224062016,
|
|
1122
|
+
"tag": "0159_team_claims_default_on",
|
|
1123
|
+
"breakpoints": true
|
|
1117
1124
|
}
|
|
1118
1125
|
]
|
|
1119
1126
|
}
|
package/dist/db/schema.sqlite.js
CHANGED
|
@@ -3915,6 +3915,28 @@ export const teamClaimLines = sqliteTable('team_claim_lines', {
|
|
|
3915
3915
|
// One head's rows (supersede, Clear, the tab's "Clear" offer).
|
|
3916
3916
|
prHeadIdx: index('team_claim_lines_account_pr_head').on(t.accountId, t.prId, t.headSha),
|
|
3917
3917
|
}));
|
|
3918
|
+
// 0159. A TEAMMATE'S LINE THIS LIMN STOOD DOWN FOR (docs/CLAUDE-REVIEW.md § Team claims → The
|
|
3919
|
+
// teammate's line on the tab): one row per (account, PR, head, provider) — the last `limn/review-<p>`
|
|
3920
|
+
// line by someone else that made this machine skip its auto review there. The AI Review tab says
|
|
3921
|
+
// "Robin Dunn reviewed this commit with Claude · no comments" from it, after a reload or a restart
|
|
3922
|
+
// (the skip memory itself is in process). `kind` 'reviewing' (their pending line) | 'reviewed'
|
|
3923
|
+
// (their success line); `description` = their line as GitHub shows it. Written by team-claim.ts on a
|
|
3924
|
+
// skip and by the reconciler when it re-reads the head. No FK on pr_id: BOTH delete paths delete its
|
|
3925
|
+
// rows (db/team-claim-lines-delete.ts), the erasure deletes everything.
|
|
3926
|
+
export const teamClaimSeen = sqliteTable('team_claim_seen', {
|
|
3927
|
+
id: integer('id').primaryKey({ autoIncrement: true }),
|
|
3928
|
+
accountId: integer('account_id').notNull(),
|
|
3929
|
+
prId: integer('pr_id').notNull(),
|
|
3930
|
+
headSha: text('head_sha').notNull(),
|
|
3931
|
+
provider: text('provider').notNull(),
|
|
3932
|
+
kind: text('kind', { enum: ['reviewing', 'reviewed'] }).notNull(),
|
|
3933
|
+
login: text('login').notNull(),
|
|
3934
|
+
description: text('description'),
|
|
3935
|
+
lineAt: integer('line_at', { mode: 'timestamp' }).notNull(),
|
|
3936
|
+
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql `(unixepoch())`),
|
|
3937
|
+
}, (t) => ({
|
|
3938
|
+
prHeadProviderUx: uniqueIndex('team_claim_seen_account_pr_head_provider').on(t.accountId, t.prId, t.headSha, t.provider),
|
|
3939
|
+
}));
|
|
3918
3940
|
// 0139. THIS DATABASE'S OWN FACTS (key → value) — never an account's, so not account-scoped and not
|
|
3919
3941
|
// erased with one. `instance_id`: a random id written once (db/instance-id.ts) that names this
|
|
3920
3942
|
// database file wherever it is moved; the reference-docs janitor's ownership marker.
|
|
@@ -14,5 +14,8 @@ export async function deleteSettledClaimLinesForPrs(tx, prIds) {
|
|
|
14
14
|
.delete(tl)
|
|
15
15
|
.where(and(inArray(tl.prId, prIds), not(and(eq(tl.whose, 'mine'), inArray(tl.state, ['claiming', 'pending'])))))
|
|
16
16
|
.execute();
|
|
17
|
+
// The teammate lines the AI Review tab names (migration 0159): nothing on GitHub to release, so
|
|
18
|
+
// every row of these PRs goes.
|
|
19
|
+
await tx.delete(schema.teamClaimSeen).where(inArray(schema.teamClaimSeen.prId, prIds)).execute();
|
|
17
20
|
}
|
|
18
21
|
//# sourceMappingURL=team-claim-lines-delete.js.map
|
|
@@ -1,8 +1,16 @@
|
|
|
1
|
+
import { activeProviderId } from '../ai/provider-setting.js';
|
|
2
|
+
import { anyUsageLockout, isLockedOut } from '../ai/usage-lockout.js';
|
|
1
3
|
// THE ONE WAY AN AUTOMATIC AGENTIC RUN ASKS THE CLAUDE SPEND BUDGET (ai/budget.ts). False ⇒ skip
|
|
2
4
|
// this run, do not queue it: every caller is a sweep or a hook that re-derives what is due, so the
|
|
3
5
|
// work comes back by itself once the budget is under its limit again. A context without the gate (a
|
|
4
6
|
// test's fake) is always allowed. A gate that throws allows — the budget never wedges a sweep.
|
|
5
|
-
|
|
7
|
+
// ⚠ A PROVIDER LOCKED OUT BY ITS USAGE LIMIT (ai/usage-lockout.ts) is refused here too, the same
|
|
8
|
+
// way: skipped, not queued. `provider`: the provider the run will use (absent ⇒ the default; null ⇒
|
|
9
|
+
// the caller picks the provider later and asks the lock itself — the auto-review sweep, whose items
|
|
10
|
+
// move to the other provider when the default is locked out).
|
|
11
|
+
export async function automaticRunAllowed(ctx, accountId, what, provider) {
|
|
12
|
+
if (provider !== null && anyUsageLockout() && isLockedOut(provider ?? activeProviderId()))
|
|
13
|
+
return false;
|
|
6
14
|
if (!ctx.aiBudget)
|
|
7
15
|
return true;
|
|
8
16
|
try {
|
|
@@ -24,6 +24,9 @@ const TRANSIENT = new RegExp([
|
|
|
24
24
|
'service unavailable',
|
|
25
25
|
'temporarily unavailable',
|
|
26
26
|
'interrupted by restart',
|
|
27
|
+
// The run watchdog (run-watchdog.ts): a sleep, or a run that stopped making progress.
|
|
28
|
+
'interrupted: the computer slept',
|
|
29
|
+
'interrupted: the run stopped making progress',
|
|
27
30
|
'not signed in',
|
|
28
31
|
'not logged in',
|
|
29
32
|
'please run /login',
|
|
@@ -69,6 +72,13 @@ export function parseRetryAtMs(error, failedAtMs) {
|
|
|
69
72
|
if (Number.isFinite(ms) && ms > 0)
|
|
70
73
|
return ms;
|
|
71
74
|
}
|
|
75
|
+
// An ISO-8601 time after the reset words: "try again at 2026-11-02T20:54:00Z" (no zone ⇒ local).
|
|
76
|
+
const iso = /(?:try again|retry|resets?|available again|until)(?: at| after| on)?:?\s{1,3}(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d{1,3})?)?(?:Z|[+-]\d{2}:?\d{2})?)/i.exec(e);
|
|
77
|
+
if (iso) {
|
|
78
|
+
const ms = Date.parse(iso[1].replace(' ', 'T'));
|
|
79
|
+
if (Number.isFinite(ms))
|
|
80
|
+
return ms;
|
|
81
|
+
}
|
|
72
82
|
// "try again in 2 hours 5 minutes" / "resets in 45m".
|
|
73
83
|
const inAt = /(?:try again|retry|resets?|available again) in\b/i.exec(e);
|
|
74
84
|
if (inAt) {
|
|
@@ -50,7 +50,8 @@ import { onOff, writeSettingEvent } from '../../db/setting-event-write.js';
|
|
|
50
50
|
//
|
|
51
51
|
// TEAM CLAIMS ("Mark auto reviews on GitHub") ride the same route as one more column,
|
|
52
52
|
// `team_claims_enabled` (migration 0134): may an AUTO review post a status line on the PR's head so
|
|
53
|
-
// teammates' Limn skips it (team-claim.ts)?
|
|
53
|
+
// teammates' Limn skips it (team-claim.ts)? ⚠ ON BY DEFAULT since 2026-11-02 (migration 0159 switched
|
|
54
|
+
// every workspace on) — only a stored false is off, in every reader and in the writer.
|
|
54
55
|
//
|
|
55
56
|
// AUTOMATIC DESIGN DIAGRAMS ride the same route as one more column, `auto_diagrams_enabled`
|
|
56
57
|
// (migration 0126): may a succeeded review (MANUAL OR AUTO) queue its design diagrams
|
|
@@ -174,8 +175,8 @@ export function autoReviewOf(row) {
|
|
|
174
175
|
pairStrictness: resolvePairStrictness(row?.pairStrictness),
|
|
175
176
|
autoPost: resolveAutoPostSettings(row?.autoPostEnabled, row?.autoPostSettings, row?.approveLeftovers),
|
|
176
177
|
autoFix: resolveAutoFixSettings(row?.autoFixSettings),
|
|
177
|
-
//
|
|
178
|
-
teamClaimsEnabled: row?.teamClaimsEnabled
|
|
178
|
+
// ON unless explicitly switched off (0159).
|
|
179
|
+
teamClaimsEnabled: row?.teamClaimsEnabled !== false,
|
|
179
180
|
};
|
|
180
181
|
}
|
|
181
182
|
/**
|
|
@@ -323,8 +324,8 @@ export async function setWorkspaceAutoReview(ctx, accountId, workspaceId, change
|
|
|
323
324
|
// Only a stored false is off (ON by default, 0126).
|
|
324
325
|
const diagramsWas = existing.autoDiagramsEnabled !== false;
|
|
325
326
|
const diagrams = autoDiagramsEnabled ?? diagramsWas;
|
|
326
|
-
// Only a stored
|
|
327
|
-
const claimsWas = existing.teamClaimsEnabled
|
|
327
|
+
// Only a stored false is off (ON by default, 0159).
|
|
328
|
+
const claimsWas = existing.teamClaimsEnabled !== false;
|
|
328
329
|
const claims = teamClaimsEnabled ?? claimsWas;
|
|
329
330
|
const capOverride = dailyCap === undefined ? existing.dailyCap : dailyCap === AUTO_REVIEW_DAILY_CAP ? null : dailyCap;
|
|
330
331
|
// Auto-posting: each field optional; a kind left out keeps its stored value.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { config } from '../../config.js';
|
|
2
2
|
import { agenticRunReady } from './ai-ready.js';
|
|
3
3
|
import { AUTO_RETRY_BACKOFF_FIRST_MS, AUTO_RETRY_BACKOFF_LATER_MS, AUTO_RETRY_MAX_ATTEMPTS, classifyAutoReviewFailure, parseRetryAtMs, } from './auto-retry.js';
|
|
4
|
+
import { usageLockout } from '../../ai/usage-lockout.js';
|
|
4
5
|
import { automaticRunAllowed } from '../budget-gate.js';
|
|
5
6
|
import { listAutoReviewWorkspaces } from './auto-settings.js';
|
|
6
7
|
import { teamClaimHeldHead } from './team-claim.js';
|
|
@@ -65,6 +66,12 @@ export function autoReviewDue(input) {
|
|
|
65
66
|
return { due: false, reason: 'not_retryable' };
|
|
66
67
|
if (retry.attempts >= AUTO_RETRY_MAX_ATTEMPTS)
|
|
67
68
|
return { due: false, reason: 'attempts_used' };
|
|
69
|
+
// ⚠ A USAGE LIMIT IS A LOCKOUT: while the provider that hit it is still locked out, the failure is
|
|
70
|
+
// not retried — not even when the default provider changed (a pair would run the locked half
|
|
71
|
+
// again; 2026-11-02, PR 199965: three failures a minute apart).
|
|
72
|
+
if (kind === 'usage_limit' && retry.lockedUntilMs != null && retry.lockedUntilMs > nowMs) {
|
|
73
|
+
return { due: false, reason: 'retry_wait', retryAtMs: retry.lockedUntilMs };
|
|
74
|
+
}
|
|
68
75
|
if (retry.failedProvider !== retry.currentProvider)
|
|
69
76
|
return { due: true };
|
|
70
77
|
const resetAtMs = kind === 'usage_limit' ? (parseRetryAtMs(retry.error, retry.failedAtMs) ?? retry.usageResetAtMs ?? null) : null;
|
|
@@ -209,7 +216,9 @@ export async function runAutoReviewSweep(ctx, nowMs = Date.now(), press = null)
|
|
|
209
216
|
// A PRESS does not ask: the reader chose the run (the SPA's two-press confirm ran when over).
|
|
210
217
|
const budgetOk = new Map();
|
|
211
218
|
for (const accountId of pressed ? [] : new Set(roster.map((w) => w.accountId))) {
|
|
212
|
-
|
|
219
|
+
// `null`: the budget only — the usage-limit lockout is asked per item, when its provider is
|
|
220
|
+
// chosen (manager.ts startAutoItem moves a locked default to the other provider).
|
|
221
|
+
const ok = await automaticRunAllowed(ctx, accountId, 'auto review', null);
|
|
213
222
|
budgetOk.set(accountId, ok);
|
|
214
223
|
if (!ok)
|
|
215
224
|
result.overBudget = true;
|
|
@@ -315,6 +324,7 @@ export async function runAutoReviewSweep(ctx, nowMs = Date.now(), press = null)
|
|
|
315
324
|
failedProvider: cand.provider,
|
|
316
325
|
currentProvider: providerNow,
|
|
317
326
|
usageResetAtMs,
|
|
327
|
+
lockedUntilMs: usageLockout(cand.provider, nowMs)?.untilMs ?? null,
|
|
318
328
|
},
|
|
319
329
|
pressed,
|
|
320
330
|
});
|
|
@@ -4,10 +4,12 @@ import { config } from '../../config.js';
|
|
|
4
4
|
import { getTicketSource } from '../../tracker/ticket-source.js';
|
|
5
5
|
import { agenticRunReady } from './ai-ready.js';
|
|
6
6
|
import { automaticRunAllowed } from '../budget-gate.js';
|
|
7
|
+
import { detectWake, FORCE_RELEASE_MS, runsToInterrupt, WATCHDOG_TICK_MS } from './run-watchdog.js';
|
|
8
|
+
import { automaticRunProviders, isLockedOut, LOCKOUT_MAX_MS, noteProviderFailure, restoreUsageLockouts } from '../../ai/usage-lockout.js';
|
|
7
9
|
import { decideReviewMode } from './routing.js';
|
|
8
10
|
import { offeredSpecialists } from './specialists.js';
|
|
9
11
|
import { buildUserPrompt, peerRef, pickReviewNonce, skipSummary, sonarHasContent, systemPromptForMode, untrustedTexts, } from './prompts.js';
|
|
10
|
-
import { pairProvidersReady, readAutoReviewProvidersForPr } from './pair.js';
|
|
12
|
+
import { PAIR_ORDER, pairProvidersReady, readAutoReviewProvidersForPr } from './pair.js';
|
|
11
13
|
import { consolidatePair, notStartedHalf } from './pair-consolidate.js';
|
|
12
14
|
import { markPairBusy } from './pair-view.js';
|
|
13
15
|
import { preflightFor } from '../../ai/run-choice.js';
|
|
@@ -17,7 +19,7 @@ import { currentThreadAssessmentsForPrs, priorAssessmentsForProvider } from './t
|
|
|
17
19
|
import { activeProviderId } from '../../ai/provider-setting.js';
|
|
18
20
|
import { dropSettledReraises, similarTitles } from './settled-by-reply.js';
|
|
19
21
|
import { attachFindingThreads } from './finding-replies.js';
|
|
20
|
-
import { getClaudeReviewById, getLatestClaudeReview, getReviewPeerContexts, getReviewPrContext, insertQueuedReview, markReviewRanOn, markReviewPeersRead, isAutoReReviewSettled, isAutoRetryStillDue, loadPriorReviewForFollowUp, loadSettledByReplyFindings, loadUnjudgedReplyFindings, loadPriorRunForCarry, markReviewCommentsSeen, markReviewCancelled, markReviewFailed, markReviewRouted, reconcileOrphanedReviewRuns, saveReviewSuccess, } from './persist.js';
|
|
22
|
+
import { getClaudeReviewById, getLatestClaudeReview, getReviewPeerContexts, getReviewPrContext, insertQueuedReview, markReviewRanOn, markReviewPeersRead, isAutoReReviewSettled, isAutoRetryStillDue, loadPriorReviewForFollowUp, loadSettledByReplyFindings, loadUnjudgedReplyFindings, loadPriorRunForCarry, markReviewCommentsSeen, markReviewCancelled, markReviewFailed, markReviewRouted, reconcileOrphanedReviewRuns, failedRunsSinceLastSuccess, saveReviewSuccess, } from './persist.js';
|
|
21
23
|
import { emitClaudeReviewCompleted } from '../../platform/event-hooks.js';
|
|
22
24
|
import { claimBeforeAutoRun, finishTeamClaim, postedReviewUrl, saveTeamClaimRecord, startTeamClaimRenew, stopTeamClaimRenew, } from './team-claim.js';
|
|
23
25
|
import { ensureSonarSnapshot } from '../../sonar/snapshot.js';
|
|
@@ -338,10 +340,40 @@ async function startAutoItem(a) {
|
|
|
338
340
|
// exists, or the PR's one automatic review is spent on a failure (ai-ready.ts).
|
|
339
341
|
if (!agenticRunReady(ctx, 'review'))
|
|
340
342
|
return;
|
|
343
|
+
// An AUTOMATIC run goes to the DEFAULT provider (Settings → AI), read now — unless the workspace
|
|
344
|
+
// asks for Claude + Codex (`auto_review_providers = 'both'`): then a PAIR when both can review,
|
|
345
|
+
// else the one that can. ⚠ A provider LOCKED OUT by its usage limit (ai/usage-lockout.ts) never
|
|
346
|
+
// starts an automatic run: a pair runs its other half alone; a single run moves to the other
|
|
347
|
+
// provider when that one is ready, else the item is dropped with no row (a skip, not a failure).
|
|
348
|
+
// A READER'S PRESS is the reader's choice: it is not moved.
|
|
349
|
+
const detect = preflightFor(ctx);
|
|
350
|
+
const nowMs = Date.now();
|
|
351
|
+
const wantsPair = (await readAutoReviewProvidersForPr(ctx, accountId, prId)) === 'both';
|
|
352
|
+
// ⚠ A PAIR drops a locked half EVEN ON A PRESS while the other half can run: that half would
|
|
353
|
+
// fail at once (2026-11-03: six pressed Codex halves over the limit, nothing reviewed). Only when
|
|
354
|
+
// every provider is locked does a press run anyway (the reader's choice).
|
|
355
|
+
const pick = (lockedOk) => automaticRunProviders({
|
|
356
|
+
defaultProvider: activeProviderId(),
|
|
357
|
+
pair: wantsPair,
|
|
358
|
+
order: PAIR_ORDER,
|
|
359
|
+
ready: (p) => detect(p, 'review').status === 'ok',
|
|
360
|
+
locked: (p) => !lockedOk && isLockedOut(p, nowMs),
|
|
361
|
+
});
|
|
362
|
+
let providers = pick(a.pressed === true && !wantsPair);
|
|
363
|
+
if (providers.length === 0 && a.pressed)
|
|
364
|
+
providers = pick(true);
|
|
365
|
+
if (providers.length === 0) {
|
|
366
|
+
ctx.log.info(`auto claude review pr ${prId}: skipped — no provider outside its usage limit`);
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
let provider = providers[0];
|
|
370
|
+
let pair = providers.length === 2
|
|
371
|
+
? { role: 'first', pairId: 0, secondProvider: providers[1], ...(a.pressed ? { pressed: true } : {}) }
|
|
372
|
+
: undefined;
|
|
341
373
|
// THE CLAUDE SPEND BUDGET (ai/budget.ts): over it, drop the item before its row exists — the
|
|
342
374
|
// sweeper re-derives it once the budget is under its limit again.
|
|
343
375
|
// A READER'S PRESS is not asked again: they chose it (the SPA's over-budget confirm ran).
|
|
344
|
-
if (!a.pressed && !(await automaticRunAllowed(ctx, accountId, 'auto review')))
|
|
376
|
+
if (!a.pressed && !(await automaticRunAllowed(ctx, accountId, 'auto review', provider)))
|
|
345
377
|
return;
|
|
346
378
|
const prCtx = await getReviewPrContext(ctx, prId, accountId);
|
|
347
379
|
if (!prCtx?.headSha)
|
|
@@ -356,19 +388,6 @@ async function startAutoItem(a) {
|
|
|
356
388
|
else if (await isAutoReReviewSettled(ctx, prId, accountId, prCtx.headSha, a.commentsAtMs ?? null))
|
|
357
389
|
return;
|
|
358
390
|
const model = DEFAULT_CLAUDE_REVIEW_MODEL;
|
|
359
|
-
// An AUTOMATIC run goes to the DEFAULT provider (Settings → AI), read now — unless the workspace
|
|
360
|
-
// asks for Claude + Codex (`auto_review_providers = 'both'`): then a PAIR when both can review,
|
|
361
|
-
// else the one that can.
|
|
362
|
-
let provider = activeProviderId();
|
|
363
|
-
let pair;
|
|
364
|
-
if ((await readAutoReviewProvidersForPr(ctx, accountId, prId)) === 'both') {
|
|
365
|
-
const ready = pairProvidersReady(preflightFor(ctx));
|
|
366
|
-
if (ready.length > 0)
|
|
367
|
-
provider = ready[0];
|
|
368
|
-
if (ready.length === 2) {
|
|
369
|
-
pair = { role: 'first', pairId: 0, secondProvider: ready[1], ...(a.pressed ? { pressed: true } : {}) };
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
391
|
// TEAM CLAIMS ("Mark auto reviews on GitHub", team-claim.ts): a teammate's Limn already holds
|
|
373
392
|
// this head for this provider ⇒ drop the item with NO row (a skip is not a failure: no retry
|
|
374
393
|
// count, no daily-cap use; the tab shows it from memory). Otherwise claim it. Never throws.
|
|
@@ -440,7 +459,16 @@ function launch(item) {
|
|
|
440
459
|
}
|
|
441
460
|
const controller = new AbortController();
|
|
442
461
|
controllers.set(reviewId, controller);
|
|
462
|
+
item.lastProgressMs = Date.now();
|
|
463
|
+
item.wokeAtMs = null;
|
|
464
|
+
startWatchdog();
|
|
465
|
+
// A run that ignores its abort is released anyway (the watchdog's FORCE_RELEASE_MS): the pipeline
|
|
466
|
+
// races this, so the end and the seat's release below still run.
|
|
467
|
+
const hardStop = new Promise((resolve) => {
|
|
468
|
+
item.forceStop = () => resolve(false);
|
|
469
|
+
});
|
|
443
470
|
const emit = (progress) => {
|
|
471
|
+
item.lastProgressMs = Date.now();
|
|
444
472
|
progressByReview.set(reviewId, progress);
|
|
445
473
|
// The PR's stream follows the half in the per-PR seat; a partner's progress is kept for when
|
|
446
474
|
// it moves there.
|
|
@@ -460,7 +488,7 @@ function launch(item) {
|
|
|
460
488
|
}
|
|
461
489
|
// Did this run end the whole job (an ordinary run, or the LAST half of a pair)?
|
|
462
490
|
let ends = true;
|
|
463
|
-
void runPipeline(item, controller, emit)
|
|
491
|
+
void Promise.race([runPipeline(item, controller, emit), hardStop])
|
|
464
492
|
.then((succeeded) => {
|
|
465
493
|
item.succeeded = succeeded;
|
|
466
494
|
})
|
|
@@ -684,7 +712,7 @@ async function startPairSecond(first) {
|
|
|
684
712
|
// Why the second half did not start (the pair's one review names it).
|
|
685
713
|
let why = 'error';
|
|
686
714
|
try {
|
|
687
|
-
if (first.trigger === 'auto' && !pair.pressed && !(await automaticRunAllowed(ctx, accountId, 'auto review (second half)'))) {
|
|
715
|
+
if (first.trigger === 'auto' && !pair.pressed && !(await automaticRunAllowed(ctx, accountId, 'auto review (second half)', provider))) {
|
|
688
716
|
why = 'budget';
|
|
689
717
|
return;
|
|
690
718
|
}
|
|
@@ -946,7 +974,7 @@ async function runPipeline(item, controller, emit) {
|
|
|
946
974
|
// on Codex, where Limn runs each specialist as its own run — the spend budget asked before each.
|
|
947
975
|
usage: { accountId: item.accountId, prId: item.prId },
|
|
948
976
|
...(item.trigger === 'auto'
|
|
949
|
-
? { allowSpecialistStart: () => automaticRunAllowed(ctx, item.accountId, 'auto review specialist') }
|
|
977
|
+
? { allowSpecialistStart: () => automaticRunAllowed(ctx, item.accountId, 'auto review specialist', item.provider) }
|
|
950
978
|
: {}),
|
|
951
979
|
applyAuthEnv: APPLY_AUTH_ENV,
|
|
952
980
|
abortController: controller,
|
|
@@ -976,11 +1004,19 @@ async function runPipeline(item, controller, emit) {
|
|
|
976
1004
|
diffCapped: prep.diffCapped,
|
|
977
1005
|
...(res.usageRecorded ? { usageRecorded: true } : {}),
|
|
978
1006
|
};
|
|
979
|
-
if (res.aborted) {
|
|
1007
|
+
if (res.aborted && item.interrupt) {
|
|
1008
|
+
// THE WATCHDOG stopped it (a sleep, or no progress): FAILED with a TRANSIENT reason, so the
|
|
1009
|
+
// auto-retry rule re-runs it when still required — never "cancelled", which is never retried.
|
|
1010
|
+
await markReviewFailed(ctx, reviewId, item.interrupt.reason, { ...telemetry, excludedFiles: prep.excludedFiles });
|
|
1011
|
+
return false;
|
|
1012
|
+
}
|
|
1013
|
+
else if (res.aborted) {
|
|
980
1014
|
await markReviewCancelled(ctx, reviewId);
|
|
981
1015
|
return false;
|
|
982
1016
|
}
|
|
983
1017
|
else if (!res.submitted) {
|
|
1018
|
+
// A usage limit locks the provider out of automatic runs until its reset (ai/usage-lockout.ts).
|
|
1019
|
+
noteProviderFailure(item.provider, res.failureReason);
|
|
984
1020
|
await markReviewFailed(ctx, reviewId, res.failureReason ?? 'review failed', {
|
|
985
1021
|
...telemetry,
|
|
986
1022
|
scope: res.scope,
|
|
@@ -1226,6 +1262,68 @@ function pump() {
|
|
|
1226
1262
|
void startAutoItem(a);
|
|
1227
1263
|
}
|
|
1228
1264
|
}
|
|
1265
|
+
// ---- THE RUN WATCHDOG (run-watchdog.ts; docs/CLAUDE-REVIEW.md § Auto review → Sleep and stalled runs) ----
|
|
1266
|
+
let watchdogTimer = null;
|
|
1267
|
+
let lastWatchdogTickMs = null;
|
|
1268
|
+
/** Every run in flight in this process (each half of a pair). */
|
|
1269
|
+
function inFlightRuns() {
|
|
1270
|
+
return [...runningItems.values(), ...pairPartners.values()].filter((it) => it.done !== true);
|
|
1271
|
+
}
|
|
1272
|
+
function startWatchdog() {
|
|
1273
|
+
if (watchdogTimer)
|
|
1274
|
+
return;
|
|
1275
|
+
lastWatchdogTickMs = Date.now();
|
|
1276
|
+
watchdogTimer = setInterval(() => {
|
|
1277
|
+
void watchdogTick(Date.now());
|
|
1278
|
+
}, WATCHDOG_TICK_MS);
|
|
1279
|
+
watchdogTimer.unref?.();
|
|
1280
|
+
}
|
|
1281
|
+
/**
|
|
1282
|
+
* One watchdog tick (exported for the test). A late tick ⇒ the machine slept: every run in flight is
|
|
1283
|
+
* marked woken. Then each run `runsToInterrupt` names is aborted with its reason (failed, not
|
|
1284
|
+
* cancelled); one still not ended FORCE_RELEASE_MS after its abort is failed in the DB and released.
|
|
1285
|
+
* Stops its timer when nothing is in flight.
|
|
1286
|
+
*/
|
|
1287
|
+
export async function watchdogTick(nowMs) {
|
|
1288
|
+
const woke = detectWake(lastWatchdogTickMs, nowMs);
|
|
1289
|
+
lastWatchdogTickMs = nowMs;
|
|
1290
|
+
const runs = inFlightRuns();
|
|
1291
|
+
if (woke != null)
|
|
1292
|
+
for (const it of runs)
|
|
1293
|
+
it.wokeAtMs = woke;
|
|
1294
|
+
const byId = new Map(runs.map((it) => [it.reviewId, it]));
|
|
1295
|
+
for (const it of runs) {
|
|
1296
|
+
if (it.interrupt && nowMs - it.interrupt.atMs >= FORCE_RELEASE_MS) {
|
|
1297
|
+
const why = it.interrupt.reason;
|
|
1298
|
+
it.ctx.log.warn(`claude review pr ${it.prId}: run ${it.reviewId} did not stop after "${why}"; releasing it`);
|
|
1299
|
+
await markReviewFailed(it.ctx, it.reviewId, why).catch(() => { });
|
|
1300
|
+
it.forceStop?.();
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
const watched = runs
|
|
1304
|
+
.filter((it) => !it.interrupt)
|
|
1305
|
+
.map((it) => ({ reviewId: it.reviewId, lastProgressMs: it.lastProgressMs ?? nowMs, wokeAtMs: it.wokeAtMs ?? null }));
|
|
1306
|
+
for (const { reviewId, reason } of runsToInterrupt(watched, nowMs)) {
|
|
1307
|
+
const it = byId.get(reviewId);
|
|
1308
|
+
if (!it)
|
|
1309
|
+
continue;
|
|
1310
|
+
it.ctx.log.warn(`claude review pr ${it.prId}: run ${reviewId} interrupted — ${reason}`);
|
|
1311
|
+
it.interrupt = { reason, atMs: nowMs };
|
|
1312
|
+
controllers.get(reviewId)?.abort();
|
|
1313
|
+
}
|
|
1314
|
+
if (runs.length === 0 && watchdogTimer) {
|
|
1315
|
+
clearInterval(watchdogTimer);
|
|
1316
|
+
watchdogTimer = null;
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
/** Test seam: the in-flight runs' watchdog view. */
|
|
1320
|
+
export function _watchdogRunsForTest() {
|
|
1321
|
+
return inFlightRuns().map((it) => ({ reviewId: it.reviewId, wokeAtMs: it.wokeAtMs ?? null, interrupted: it.interrupt?.reason ?? null }));
|
|
1322
|
+
}
|
|
1323
|
+
/** Test seam: pretend the previous tick ran at `ms` (null = no tick yet). */
|
|
1324
|
+
export function _setWatchdogLastTickForTest(ms) {
|
|
1325
|
+
lastWatchdogTickMs = ms;
|
|
1326
|
+
}
|
|
1229
1327
|
/**
|
|
1230
1328
|
* Cancel the review for a PR. `accountId` is REQUIRED and enforced against the queue entry's
|
|
1231
1329
|
* own owner: without it, one tenant could abort another tenant's in-flight (billed) agentic run
|
|
@@ -1387,6 +1485,9 @@ export async function listActiveReviews(accountId) {
|
|
|
1387
1485
|
*/
|
|
1388
1486
|
export async function reconcileReviewsOnStartup(ctx) {
|
|
1389
1487
|
const runs = await reconcileOrphanedReviewRuns(ctx);
|
|
1488
|
+
// A provider still inside a usage limit a run hit before the restart stays locked out of
|
|
1489
|
+
// automatic runs (ai/usage-lockout.ts). Cheap: one bounded read.
|
|
1490
|
+
await restoreUsageLockouts(() => failedRunsSinceLastSuccess(ctx, Date.now() - LOCKOUT_MAX_MS));
|
|
1390
1491
|
if (runs.length > 0)
|
|
1391
1492
|
ctx.log.info(`reconciled ${runs.length} orphaned claude review(s) -> failed`);
|
|
1392
1493
|
const pairsFinished = finishInterruptedPairs(ctx, runs).catch((err) => {
|