@quickchatai/quickreview 1.3.1 → 1.3.2

package/src/state.ts

@@ -2,7 +2,10 @@
 // Core tables: pr_state, rounds, bot_comments (registry), deliveries
 // (webhook dedupe), events (append log).
 import { findingRowFields } from "@quickchatai/pi-quickreview/src/finding-row.js";
-import { roundStageDeadlines } from "@quickchatai/pi-quickreview/src/round-stage-deadlines.js";
+import {
+  discoveryRowsOf,
+  roundStageDeadlines,
+} from "@quickchatai/pi-quickreview/src/round-stage-deadlines.js";
 import Database from "better-sqlite3";
 import { chmodSync, mkdirSync, renameSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
@@ -550,8 +553,9 @@ export type DrainState =
       /** When the hold lapses; null = until lifted (operator only). */
       expires_at: string | null;
       /** Why a SYSTEM engaged it (`worker_failures`, the crash-loop breaker); null
-       * for a hand-set drain. A drain with a reason is lifted only by an explicit
-       * `drain off`, never by a boot's first reconcile pass. */
+       * for a hand-set drain. A drain with a reason is never lifted by a boot's
+       * first reconcile pass: the breaker's lapses at its `expires_at`, or an
+       * explicit `drain off` lifts it sooner. */
       reason: string | null;
       /** The operator's no-auto-lift HOLD (`qcr drain on --hold`): no
        * boot's first reconcile lifts it, only an explicit operator `drain off`.
@@ -1050,7 +1054,8 @@ CREATE TABLE IF NOT EXISTS drain_state (
 -- with parked_at set gets no worker until the operator's rerun-round or
 -- restart-session clears the row. worker_breaker_global.epoch ticks at every
 -- worker success (and at the lift of the breaker's drain); a failure counts
--- toward the global drain only while it carries the current epoch.
+-- toward the global drain only while it carries the current epoch. trips
+-- counts the breaker's drains since then: each waits twice as long to lapse.
 CREATE TABLE IF NOT EXISTS worker_breaker (
   pr              INTEGER PRIMARY KEY,
   failures        INTEGER NOT NULL DEFAULT 0,
@@ -1064,6 +1069,7 @@ CREATE TABLE IF NOT EXISTS worker_breaker (
 CREATE TABLE IF NOT EXISTS worker_breaker_global (
   id         INTEGER PRIMARY KEY CHECK (id = 1),
   epoch      INTEGER NOT NULL DEFAULT 0,
+  trips      INTEGER NOT NULL DEFAULT 0,
   updated_at TEXT NOT NULL
 );
 -- The host lease: who holds a PR's session while the operator works
@@ -1828,6 +1834,7 @@ export class StateStore {
     this.db.exec("CREATE INDEX IF NOT EXISTS bot_comments_id ON bot_comments(comment_id)");
     this.migrateRecheckReceipts();
     this.migrateDrainHold();
+    this.migrateBreakerHeal();
     this.migrateSessionCommandCancelledStatus();
     this.runDataMigrations();
   }
@@ -1932,6 +1939,37 @@ export class StateStore {
     })();
   }
 
+  /**
+   * The breaker's drain lapses on its own (src/workerbreaker.ts). One an older
+   * daemon set stands with no expiry, and only `drain off` would lift it: the
+   * boot that adds the `trips` column gives it an expiry of now, so the first
+   * sweep lifts it and the queue retries, with the failures that tripped it
+   * spent. Only the breaker's drain is carried
+   * (its reason, no hold); a hand-set drain stands as it was set.
+   */
+  private migrateBreakerHeal(): void {
+    this.db.transaction(() => {
+      if (
+        !this.addColumn(
+          "ALTER TABLE worker_breaker_global ADD COLUMN trips INTEGER NOT NULL DEFAULT 0",
+        )
+      )
+        return;
+      const info = this.db
+        .prepare(
+          `UPDATE drain_state SET expires_at = ?
+            WHERE id = 1 AND draining = 1 AND reason = ? AND expires_at IS NULL AND hold = 0`,
+        )
+        .run(this.ts(), WORKER_FAILURES_DRAIN_REASON);
+      if (info.changes > 0) {
+        // The failures that tripped it are spent, as at a new trip: a retry
+        // failing once more must not re-drain with them.
+        this.resetWorkerFailureWindow();
+        this.logEvent("breaker_drain_heal_carried", null, {});
+      }
+    })();
+  }
+
   /** Existing databases carry a CHECK constraint that predates operator
    * cancellation. Rebuild this one table in-place; every column and index is
    * copied, and the transaction makes the old or new shape visible atomically. */
@@ -6696,18 +6734,31 @@ export class StateStore {
    * One more consecutive worker failure for `pr`. The PR is parked when the count
    * reaches `parkAfter`; `parked` says whether it stands parked now and
    * `newlyParked` whether THIS failure parked it (the caller raises the anomaly
-   * once).
+   * once). With `drain: false` it does not count toward the global drain: its
+   * epoch is one no window ever carries, so `failedWorkerPrsSince` never lists
+   * it. With `park: false` it counts but does not park the PR by itself.
    */
   recordWorkerFailure(
     pr: number,
-    o: { commandId: string; error: string; at: string; parkAfter: number },
+    o: {
+      commandId: string;
+      error: string;
+      at: string;
+      parkAfter: number;
+      drain?: boolean;
+      park?: boolean;
+    },
   ): { failures: number; parked: boolean; newlyParked: boolean } {
     return this.db
       .transaction(() => {
         const row = this.workerBreaker(pr);
         const failures = (row?.failures ?? 0) + 1;
         const wasParked = row?.parked_at != null;
-        const parkedAt = wasParked ? row.parked_at : failures >= o.parkAfter ? o.at : null;
+        const parkedAt = wasParked
+          ? row.parked_at
+          : o.park !== false && failures >= o.parkAfter
+            ? o.at
+            : null;
         this.db
           .prepare(
             `INSERT INTO worker_breaker (pr, failures, parked_at, last_error, last_failure_at, last_command_id, failure_epoch, updated_at)
@@ -6722,7 +6773,7 @@ export class StateStore {
             error: o.error,
             at: o.at,
             command: o.commandId,
-            epoch: this.workerEpoch(),
+            epoch: o.drain === false ? -1 : this.workerEpoch(),
           });
         return {
           failures,
@@ -6770,12 +6821,31 @@ export class StateStore {
     return row?.epoch ?? 0;
   }
 
-  /** Start the global failure window afresh: earlier failures stop counting. */
+  /** Start the global failure window afresh: earlier failures stop counting,
+   * and the breaker's next drain waits the shortest time again. */
   resetWorkerFailureWindow(): void {
     this.db
       .prepare(
-        `INSERT INTO worker_breaker_global (id, epoch, updated_at) VALUES (1, 1, @ts)
-         ON CONFLICT(id) DO UPDATE SET epoch = epoch + 1, updated_at = @ts`,
+        `INSERT INTO worker_breaker_global (id, epoch, trips, updated_at) VALUES (1, 1, 0, @ts)
+         ON CONFLICT(id) DO UPDATE SET epoch = epoch + 1, trips = 0, updated_at = @ts`,
+      )
+      .run({ ts: this.ts() });
+  }
+
+  /** The breaker's drains since the last worker success or operator lift. */
+  workerBreakerTrips(): number {
+    const row = this.db.prepare("SELECT trips FROM worker_breaker_global WHERE id = 1").get() as
+      { trips: number } | undefined;
+    return row?.trips ?? 0;
+  }
+
+  /** The breaker drained once more: the failures that tripped it are spent (a
+   * new epoch), and its next drain waits longer. */
+  recordWorkerBreakerTrip(): void {
+    this.db
+      .prepare(
+        `INSERT INTO worker_breaker_global (id, epoch, trips, updated_at) VALUES (1, 1, 1, @ts)
+         ON CONFLICT(id) DO UPDATE SET epoch = epoch + 1, trips = trips + 1, updated_at = @ts`,
       )
       .run({ ts: this.ts() });
   }
@@ -7005,6 +7075,25 @@ export class StateStore {
     return someMarkHasKey(rows, key);
   }
 
+  /** Has a check after this (pr, key)'s spent nudge already been recorded? */
+  ledgerRepeatSeen(pr: number, key: string): boolean {
+    const rows = this.db
+      .prepare("SELECT payload FROM events WHERE kind = 'ledger_stale_repeat' AND pr = ?")
+      .all(pr) as Array<{ payload: string | null }>;
+    return someMarkHasKey(rows, key);
+  }
+
+  /** The heads a recheck upheld on `pr` at or after `since` (every one when null). */
+  recheckUpheldHeads(pr: number, since: string | null): string[] {
+    const rows = this.db
+      .prepare("SELECT payload FROM events WHERE kind = 'recheck_upheld' AND pr = ? AND ts >= ?")
+      .all(pr, since ?? "") as Array<{ payload: string | null }>;
+    // An unreadable mark names no head.
+    return rows
+      .map((r) => readStringField(r.payload, "head_sha"))
+      .filter((head): head is string => head !== undefined && head !== "");
+  }
+
   /**
    * What the event log says about the migration-collision COMMENT with this
    * fingerprint on this PR (the collision comment's dedupe). Scans every such event for the
@@ -8178,6 +8267,7 @@ export function buildStatusSnapshot(
                   started,
                   roundStageBounds,
                   round.sweep_owed === 1,
+                  discoveryRowsOf(round),
                 );
                 return {
                   ...identity,

package/src/tui/alerts.ts

@@ -30,6 +30,7 @@ import { zoneDateString } from "../eod.js";
 import { curationNeedsAttention } from "../learner/status.js";
 import { memoryStatusLine, type MemoryStatus } from "../memory/health.js";
 import type { StatusSnapshot } from "../state.js";
+import { DEFAULT_BREAKER_LIMITS } from "../workerbreaker.js";
 
 export type AlertTier = "notice" | "alert";
 
@@ -616,7 +617,11 @@ export interface WorkerCrashAlertMemory {
 export function maybeWriteWorkerCrashAlerts(
   alertsDir: string,
   snapshot:
-    Pick<StatusSnapshot, "prs" | "draining" | "draining_reason" | "draining_since"> | undefined,
+    | Pick<
+        StatusSnapshot,
+        "prs" | "draining" | "draining_reason" | "draining_since" | "draining_until"
+      >
+    | undefined,
   fresh: boolean,
   mem: WorkerCrashAlertMemory,
   now: () => Date = () => new Date(),
@@ -660,7 +665,9 @@ export function maybeWriteWorkerCrashAlerts(
       "worker-failures-drain",
       null,
       "workers failing on several PRs: the bot is draining",
-      "Workers failed on three PRs within 15 minutes with no success, so the bot drains: no round, answer or debug worker starts. Fix the cause, then run drain off.",
+      snapshot.draining_until
+        ? `Workers failed on three PRs within 15 minutes with no success, so the bot drains: no round, answer or debug worker starts. It lifts itself at ${snapshot.draining_until} and the queue retries; if workers fail again it drains again, each time waiting twice as long (up to ${DEFAULT_BREAKER_LIMITS.healMaxMs / 3_600_000} h). Run drain off to retry now.`
+        : "Workers failed on three PRs within 15 minutes with no success, so the bot drains: no round, answer or debug worker starts. Fix the cause, then run drain off.",
     );
   return names;
 }

package/src/tui/board.ts

@@ -534,7 +534,7 @@ function barSegments(view: BoardView, alerts: boolean): string[] {
   // guessed at. The operator's HOLD says that no boot lifts it; an
   // operator drain from a daemon that reports no hold marker says "hold ?".
   const drainSeg = s?.draining
-    ? `${FG.anomaly}⛔ DRAINING — ${s.running_rounds} round${s.running_rounds === 1 ? "" : "s"} finishing · ${s.drain_failed_loud ?? 0} failed loud · ${s.drained_deliveries ?? 0} dropped · ${leverHolderLabel(s.draining_by, s.draining_until)}${drainHoldLabel(s)}${s.draining_reason === "worker_failures" ? " · workers keep failing, drain off lifts it" : ""}${RESET}`
+    ? `${FG.anomaly}⛔ DRAINING — ${s.running_rounds} round${s.running_rounds === 1 ? "" : "s"} finishing · ${s.drain_failed_loud ?? 0} failed loud · ${s.drained_deliveries ?? 0} dropped · ${leverHolderLabel(s.draining_by, s.draining_until)}${drainHoldLabel(s)}${s.draining_reason === "worker_failures" ? (s.draining_until ? " · workers kept failing, it retries when it lapses; drain off lifts it now" : " · workers keep failing, drain off lifts it") : ""}${RESET}`
     : "";
   const td = s?.thread_decisions;
   const breakerSeg =

package/src/verdicts.ts

@@ -80,6 +80,7 @@ import {
 import { estimateUsd, type ModelPricing } from "./usage.js";
 import { TranscriptIndex } from "./transcriptindex.js";
 import { withinBudget } from "@quickchatai/pi-quickreview/src/budget.js";
+import { VALIDATE_PASS } from "@quickchatai/pi-quickreview/src/include.js";
 import type { GateResult } from "@quickchatai/pi-quickreview/src/gate.js";
 import { applyEvent, restoreStandingLifecycle } from "./statemachine.js";
 import type { VerdictAnnouncer } from "./slack/announce.js";
@@ -505,6 +506,12 @@ function plannedTierMismatches(passes: VerdictPassEntry[], rows: PlanRow[]): str
   return wrong;
 }
 
+/** Every lens of the stage reported its candidate count, and every count is zero. */
+function nothingToValidate(passes: VerdictPassEntry[]): boolean {
+  const lenses = passes.filter((p) => p.name !== VALIDATE_PASS);
+  return lenses.length > 0 && lenses.every((p) => p.findings === 0);
+}
+
 function comparePlan(
   passes: VerdictPassEntry[],
   plan: PlanRow[] | null,
@@ -3003,6 +3010,12 @@ export class VerdictWatcher {
       const deltaPasses = followUp ? v.passes.filter((p) => p.phase !== "sweep") : v.passes;
       const sweepPasses = followUp ? v.passes.filter((p) => p.phase === "sweep") : [];
       const main = comparePlan(deltaPasses, plan, addable, legacyPlanned, true);
+      // A follow-up's delta stage dispatches no validator when its lenses
+      // found nothing (prompts/round-follow-up.md, "Nothing to validate ⇒ no
+      // validator"): that absence keeps the rule. Only a count every lens
+      // reported as zero says so; an unreported one still owes the validator.
+      if (followUp && nothingToValidate(deltaPasses))
+        main.missing = main.missing.filter((name) => name !== VALIDATE_PASS);
       let sweep: PlanComparison | null = null;
       if (sweepPasses.length > 0 || sweepStatus === "ran") {
         const sweepPlan = round.sweep_plan ? (JSON.parse(round.sweep_plan) as PlanRow[]) : [];

package/src/wire.ts

@@ -1836,8 +1836,9 @@ export function createProductionDaemon(parts: ProductionParts = buildParts()): D
         // pump starts nothing, recheck/ping-pong/EOD hold — in-flight rounds
         // finish naturally. Off is the manual lift; the automatic one is the
         // next boot's first healthy reconcile pass (daemon.ts) — except for
-        // the operator's HOLD, which only an explicit off lifts. Never touches
-        // GitHub, never goes near the gate.
+        // the operator's HOLD, which only an explicit off lifts, and the worker
+        // breaker's drain, which lapses on its own (src/workerbreaker.ts).
+        // Never touches GitHub, never goes near the gate.
         drain(on, issuer, forMin, deliveryKey, hold) {
           return drainLever(
             { ...leverDeps, deliveryKey },

package/src/workerbreaker.ts

@@ -4,14 +4,22 @@
 //
 //   - PER PR. A worker's failure is a non-zero exit, a `failed` event over the IPC,
 //     or a deadline kill (wall clock, stage or silence). After `parkAfter`
-//     consecutive ones the PR is PARKED: no further spawn, a `worker_crash_loop`
+//     consecutive ones the PR is PARKED (a provider network drop counts but
+//     never parks by itself: it says nothing about the PR): no further spawn, a `worker_crash_loop`
 //     anomaly (to the companion inbox), the last error in status.json and a TUI
 //     alert. A worker that served its round resets the PR's count. The operator
 //     unparks with `rerun-round` or `restart-session` (src/operator-recovery.ts).
 //   - GLOBAL. `drainAfterPrs` PRs failing within `windowMs`, with no worker success
 //     in between, set the existing drain valve with the reason `worker_failures`.
-//     Only an explicit `drain off` lifts it (a boot's first healthy reconcile pass
-//     leaves a reasoned drain standing).
+//     The drain heals itself: it lapses after `healAfterMs`, the queue retries,
+//     and if workers fail again it drains again, each time waiting twice as long
+//     up to `healMaxMs`. A served round (or the operator's `drain off`, which
+//     lifts it at once) starts the wait from `healAfterMs` again. A boot's first
+//     healthy reconcile pass does not lift it early, and an operator's own
+//     `drain on` over it stands until the operator lifts it. A PR's FIRST
+//     failure that is the provider's network dropping (`fetch failed`, a reset
+//     socket) does not feed the drain: one blip fails every concurrent worker at
+//     the same instant, and each of them is retried. A retry that fails again counts.
 //   - EVERY HOST. `gate` is the one question each host asks before it starts a
 //     round worker, an answer worker or a debug round: the bot
 //     is drained, or the PR is parked, and the answer is no. A takeover
@@ -29,12 +37,18 @@ export interface BreakerLimits {
   drainAfterPrs: number;
   /** The window those PRs' latest failures must fall in. */
   windowMs: number;
+  /** How long the breaker's first drain stands before it lapses and the queue retries. */
+  healAfterMs: number;
+  /** The longest a drain waits, however often it has tripped. */
+  healMaxMs: number;
 }
 
 export const DEFAULT_BREAKER_LIMITS: BreakerLimits = {
   parkAfter: 2,
   drainAfterPrs: 3,
   windowMs: 15 * 60_000,
+  healAfterMs: 15 * 60_000,
+  healMaxMs: 4 * 60 * 60_000,
 };
 
 export type SpawnVerdict = { ok: true } | { ok: false; why: "draining" | "parked"; reason: string };
@@ -48,6 +62,15 @@ export interface WorkerFailure {
 
 const ERROR_CHARS = 300;
 
+/** The provider's transport dropped the turn (src/botworker.ts words it), as
+ * opposed to the provider answering with an error or refusing the login. */
+const PROVIDER_TRANSPORT =
+  /the provider failed the turn: .*\b(fetch failed|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|socket hang up)\b/i;
+
+function isProviderTransportFailure(error: string): boolean {
+  return PROVIDER_TRANSPORT.test(error);
+}
+
 export class WorkerBreaker {
   readonly limits: BreakerLimits;
   private readonly now: () => number;
@@ -96,11 +119,15 @@ export class WorkerBreaker {
   failure(pr: number, commandId: string, error: string): WorkerFailure {
     const at = this.iso();
     const last = error.replace(/\s+/g, " ").trim().slice(0, ERROR_CHARS) || "no reason given";
+    const firstFailure = (this.store.workerBreaker(pr)?.failures ?? 0) === 0;
+    const transport = isProviderTransportFailure(last);
     const res = this.store.recordWorkerFailure(pr, {
       commandId,
       error: last,
       at,
       parkAfter: this.limits.parkAfter,
+      drain: !(firstFailure && transport),
+      park: !transport,
     });
     if (res.newlyParked) {
       const detail = `PR #${pr} is parked after ${res.failures} consecutive worker failures; last: ${last}`;
@@ -126,12 +153,20 @@ export class WorkerBreaker {
     const since = new Date(this.now() - this.limits.windowMs).toISOString();
     const prs = this.store.failedWorkerPrsSince(since);
     if (prs.length < this.limits.drainAfterPrs) return false;
+    // A standing drain, the breaker's or a hand-set one, is never extended here.
+    if (this.store.drainState().draining) return false;
+    const waitMs = Math.min(
+      this.limits.healAfterMs * 2 ** this.store.workerBreakerTrips(),
+      this.limits.healMaxMs,
+    );
+    const lapses = new Date(this.now() + waitMs).toISOString();
     const res = this.store.engageDrain(
-      { issuer: "operator", expires_at: null },
+      { issuer: "operator", expires_at: lapses },
       WORKER_FAILURES_DRAIN_REASON,
     );
     if (res.outcome === "unchanged") return false;
-    const detail = `workers failed for ${prs.length} PRs (${prs.map((n) => `#${n}`).join(", ")}) within ${Math.round(this.limits.windowMs / 60_000)} min with no success; the bot is draining (${WORKER_FAILURES_DRAIN_REASON}) until an operator runs drain off`;
+    this.store.recordWorkerBreakerTrip();
+    const detail = `workers failed for ${prs.length} PRs (${prs.map((n) => `#${n}`).join(", ")}) within ${Math.round(this.limits.windowMs / 60_000)} min with no success; the bot is draining (${WORKER_FAILURES_DRAIN_REASON}) and retries at ${lapses} (in ${Math.round(waitMs / 60_000)} min); drain off lifts it now`;
     this.log(`worker_failures drain — ${detail}`);
     this.store.logEvent("drain_on", null, {
       by: "system",
@@ -139,6 +174,7 @@ export class WorkerBreaker {
       outcome: res.outcome,
       prs,
       at,
+      expires_at: lapses,
     });
     this.store.logAnomaly(
       "worker_crash_loop",