@tablation/crew 0.79.10 → 0.80.0

package/dist/cli.js

@@ -2981,9 +2981,11 @@ function describeRelease(o, route) {
       level: "fail",
       headline: `${route}: release blocked \u2014 tests failed`,
       detail: withHookTail(
-        o.composition ? `the target stays on the previous release
+        (o.composition ? `the target stays on the previous release
 
-${describeComposition(o.composition)}` : "the target stays on the previous release",
+${describeComposition(o.composition)}` : "the target stays on the previous release") + (o.isolation ? `
+
+${o.isolation}` : ""),
         o.hookFailure
       ),
       ...o.composition ? { head: o.composition.head } : {}
@@ -3013,7 +3015,9 @@ ${describeComposition(o.composition)}` : "the target stays on the previous relea
     return {
       level: "fail",
       headline: `${route}: build FAILED`,
-      detail: withHookTail("the target stays on the previous release \u2014 nothing was deployed", o.hookFailure)
+      detail: withHookTail(`the target stays on the previous release \u2014 nothing was deployed${o.isolation ? `
+
+${o.isolation}` : ""}`, o.hookFailure)
     };
   }
   if (o.blockedCycles && o.blockedCycles >= 2) {
@@ -11311,7 +11315,7 @@ function planBuildFailureBounce(cwd, c, base, remote) {
   return { ...c, base, remote, worktree: worktreeForBranch(cwd, c.branch) };
 }
 function buildFailureComment(b) {
-  const how = b.how === "files" ? `The failing output names files that only \`${b.branch}\` changed among the branches in that release.` : `Built alone on the previous \`${b.base}\`, \`${b.branch}\` fails the same hook.`;
+  const how = b.how === "pair" && b.sibling ? `Each branch passes the hook alone; \`${b.branch}\` together with ${b.sibling.key} (\`${b.sibling.branch}\`) fails it. ${b.sibling.key} was verified first, so the fix belongs on this branch: \`git merge ${b.remote}/${b.sibling.branch}\` first, then fix what disagrees. Fix it on this branch, never a fresh one.` : b.how === "files" ? `The failing output names files that only \`${b.branch}\` changed among the branches in that release.` : `Built alone on the previous \`${b.base}\`, \`${b.branch}\` fails the same hook.`;
   return `${BUILD_FAILURE_MARKER}
 
 **Back with the dev lane \u2014 this branch builds on its own but not on current \`${b.base}\`.**
@@ -11326,8 +11330,46 @@ ${b.tail.trimEnd()}
 
 Nothing was written to \`${b.base}\` for this ticket; the rest of the release shipped without it.`;
 }
-async function applyBuildFailureBounce(writer, b, contract, memberId, log, dryRun) {
+function priorBuildFailures(ticket, events) {
+  const stamp = (e) => e.occurred_at ?? e.created_at ?? "";
+  const mine = events.filter((e) => e.record_id === ticket.id);
+  const pushes = mine.filter((e) => e.event_type === "pushed").map(stamp);
+  return mine.filter((e) => e.event_type === "build_failed" && pushes.some((p) => p > stamp(e))).length;
+}
+function buildFailureEscalation(b) {
+  return `${BUILD_FAILURE_MARKER}
+
+**Failed the release gate again after a hand-back \u2014 stopping rather than looping.**
+
+\`${b.branch}\` was handed back for breaking the \`${b.hook}\` hook, pushed, verified again, and breaks it once more. That is a call for a person, not another lap through the lanes.
+
+\`\`\`
+${b.tail.trimEnd()}
+\`\`\``;
+}
+async function applyBuildFailureBounce(writer, b, contract, memberId, log, dryRun, prior = { count: 0 }) {
   const t = { ticket: b.ticket.issue_id, step: "release" };
+  if (prior.count > 0) {
+    if (dryRun) {
+      log.emit(`would escalate to ${contract.statuses.needsHuman} \u2014 handed back for the ${b.hook} hook before`, t);
+      return { kind: "escalated" };
+    }
+    try {
+      await writer.updateTicket(
+        b.ticket.id,
+        { status: contract.statuses.needsHuman, ...prior.operatorId ? { assignee_id: prior.operatorId } : {} },
+        b.ticket.updated_at
+      );
+      await writer.postEvent(b.ticket.id, buildFailureEscalation(b), memberId, "build_failed_escalated").catch(() => {
+        log.warn("escalated, but the note failed to post", t);
+      });
+      log.warn(`breaks the ${b.hook} hook again after a hand-back \u2014 escalated to ${contract.statuses.needsHuman}`, t);
+      return { kind: "escalated" };
+    } catch (e) {
+      if (e instanceof StaleWriteError8) return { kind: "stale" };
+      return { kind: "failed", why: e.message };
+    }
+  }
   if (dryRun) {
     log.emit(`would hand ${b.ticket.issue_id} back to ${contract.statuses.building} \u2014 its branch breaks the ${b.hook} hook`, t);
     return { kind: "handed-back" };
@@ -11341,6 +11383,13 @@ async function applyBuildFailureBounce(writer, b, contract, memberId, log, dryRu
     await writer.postEvent(b.ticket.id, buildFailureComment(b), memberId, "build_failed").catch(() => {
       log.warn("handed back, but the note failed to post", t);
     });
+    await writer.postEvent(
+      b.ticket.id,
+      `${b.ticket.issue_tag ?? b.ticket.issue_id} left out of the release: ${b.hook} hook, found by ${b.how}${b.sibling ? `, together with ${b.sibling.key} (${b.sibling.branch})` : ""}`,
+      memberId,
+      "release_culprit"
+    ).catch(() => {
+    });
     log.emit(`handed back to ${contract.statuses.building} \u2014 breaks the ${b.hook} hook once merged with the rest`, t);
     return { kind: "handed-back" };
   } catch (e) {
@@ -12672,6 +12721,25 @@ async function runReleaseCycle(o, movedPast2, localRefSkips) {
     }
   }
 }
+var MAX_ISOLATE_PAIRS = 10;
+function pairOrder(n, max = MAX_ISOLATE_PAIRS) {
+  const pairs = [];
+  let truncated = false;
+  outer: for (let j = 1; j < n; j++) {
+    for (let i = 0; i < j; i++) {
+      if (pairs.length >= max) {
+        truncated = true;
+        break outer;
+      }
+      pairs.push([i, j]);
+    }
+  }
+  return { pairs, truncated };
+}
+function selfRepairLine(maxCandidates) {
+  const cap = maxCandidates ?? DEFAULT_ISOLATE_MAX_CANDIDATES;
+  return cap === 0 ? "culprit isolation is off (release.isolateFailures.maxCandidates: 0); a failed gate files an alert" : `culprit isolation up to ${cap} merged branches, ${MAX_ISOLATE_PAIRS} pairs before bisecting; a ticket handed back twice after a push escalates with an alert`;
+}
 var isolateCap = (o) => o.repo.release.isolateFailures?.maxCandidates ?? DEFAULT_ISOLATE_MAX_CANDIDATES;
 async function isolateAndRetry(o, name, output, merged, preMerge, decision, fresh, releaseBranch) {
   const applied = merged.filter((c) => c.sha && c.branch);
@@ -12696,13 +12764,13 @@ async function isolateAndRetry(o, name, output, merged, preMerge, decision, fres
     if (baseRun && baseRun.code !== 0) {
       o.emit.warn(`${name} fails on the base without ${only.branch} \u2014 not blaming the branch`, { ticket: only.ticket.issue_id });
       gitOk(o.cwd, ["reset", "--hard", head]);
-      return null;
+      return { tried: `Isolation found no culprit: the ${name} hook also fails on the base alone, without ${only.branch}, so there is no branch to hand back.` };
     }
     o.emit.warn(`${name} failed with only ${only.branch} merged and passes without it \u2014 treating it as the culprit`, { ticket: only.ticket.issue_id });
     culprits.push({ c: only, how: "solo", tail: lastLines(output, HOOK_FAILURE_TAIL_LINES) });
   } else if (applied.length > cap) {
     o.emit.warn(`${name} failed with ${applied.length} branches merged, over release.isolateFailures.maxCandidates (${cap}) \u2014 not isolating`);
-    return null;
+    return { tried: `Isolation was skipped: ${applied.length} branches were merged, over release.isolateFailures.maxCandidates (${cap}).` };
   } else {
     o.emit.warn(`${name} failed with ${applied.length} branches merged \u2014 building each alone to find the culprit`);
     for (const c of applied) {
@@ -12715,12 +12783,51 @@ async function isolateAndRetry(o, name, output, merged, preMerge, decision, fres
         culprits.push({ c, how: "solo", tail: lastLines(r.output, HOOK_FAILURE_TAIL_LINES) });
       }
     }
+    if (culprits.length === 0) {
+      const { pairs, truncated } = pairOrder(applied.length);
+      for (const [i, j] of pairs) {
+        gitOk(o.cwd, ["reset", "--hard", preMerge]);
+        const a = mergeOne(o, applied[i]);
+        const b = typeof a === "object" && "conflict" in a ? a : mergeOne(o, applied[j]);
+        if (typeof b === "object" && "conflict" in b) continue;
+        const r = await hook(o, name);
+        if (r && r.code !== 0) {
+          o.emit.warn(`${applied[j].branch} fails the ${name} hook together with ${applied[i].branch}`, { ticket: applied[j].ticket.issue_id });
+          culprits.push({ c: applied[j], how: "pair", sibling: applied[i], tail: lastLines(r.output, HOOK_FAILURE_TAIL_LINES) });
+          break;
+        }
+      }
+      if (culprits.length === 0 && truncated) {
+        let lo = 2, hi = applied.length;
+        let tail = lastLines(output, HOOK_FAILURE_TAIL_LINES);
+        while (lo < hi) {
+          const mid = Math.floor((lo + hi) / 2);
+          gitOk(o.cwd, ["reset", "--hard", preMerge]);
+          let clean = true;
+          for (const c of applied.slice(0, mid)) {
+            const m = mergeOne(o, c);
+            if (typeof m === "object" && "conflict" in m) {
+              clean = false;
+              break;
+            }
+          }
+          const r = clean ? await hook(o, name) : null;
+          if (r && r.code !== 0) {
+            hi = mid;
+            tail = lastLines(r.output, HOOK_FAILURE_TAIL_LINES);
+          } else lo = mid + 1;
+        }
+        const owner = applied[hi - 1];
+        o.emit.warn(`${owner.branch} is the first branch whose prefix fails the ${name} hook (bisected over ${applied.length})`, { ticket: owner.ticket.issue_id });
+        culprits.push({ c: owner, how: "pair", sibling: applied[hi - 2], tail });
+      }
+    }
   }
   gitOk(o.cwd, ["reset", "--hard", preMerge]);
   if (culprits.length === 0) {
     o.emit.warn("no single branch could be blamed \u2014 failing the release as a whole");
     gitOk(o.cwd, ["reset", "--hard", head]);
-    return null;
+    return { tried: `Isolation found no culprit: each of the ${applied.length} merged branches passes the ${name} hook alone and no pair among them fails it.` };
   }
   const out = new Set(culprits.map((x) => x.c.ticket.id));
   const rest = decision.merges.filter((c) => !out.has(c.ticket.id));
@@ -12732,7 +12839,8 @@ async function isolateAndRetry(o, name, output, merged, preMerge, decision, fres
       branch: x.c.branch,
       hook: name,
       tail: x.tail,
-      how: x.how
+      how: x.how,
+      ...x.sibling ? { sibling: { key: x.sibling.ticket.issue_tag ?? x.sibling.ticket.issue_id, branch: x.sibling.branch } } : {}
     }))
   };
 }
@@ -12743,6 +12851,7 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
   const conflicts = [];
   const unbuildable = [];
   let applied = 0;
+  let isolation;
   for (const c of decision.merges) {
     if (!c.branch || c.mergedSha) {
       if (c.skipReason === "already-released") {
@@ -12855,7 +12964,8 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
       }
       if (r && r.code !== 0 && !isolated) {
         const out = await isolateAndRetry(o, "test", r.output, merged, preMerge, decision, fresh, releaseBranch);
-        if (out) return out;
+        if (out && "tried" in out) isolation = out.tried;
+        else if (out) return out;
       }
       if (r && r.code !== 0) {
         o.emit.error(`test gate FAILED \u2014 not deploying; the target stays on the previous release`);
@@ -12876,7 +12986,7 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
           label: hookLabel(o.repo, "test"),
           tail: failureSummary(r.output, HOOK_FAILURE_TAIL_LINES)
         };
-        return { merged, conflicts, unbuildable, deployed: false, stopped: "tests failed", decision, gateRed, hookFailure: hookFailure2, composition };
+        return { merged, conflicts, unbuildable, deployed: false, stopped: "tests failed", decision, gateRed, hookFailure: hookFailure2, composition, ...isolation ? { isolation } : {} };
       }
       o.state?.clearTestGateFailed?.();
     }
@@ -12915,7 +13025,8 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
       }
       if (r && r.code !== 0 && !isolated) {
         const out = await isolateAndRetry(o, "build", r.output, merged, preMerge, decision, fresh, releaseBranch);
-        if (out) return out;
+        if (out && "tried" in out) isolation = out.tried;
+        else if (out) return out;
       }
       if (r && r.code !== 0) {
         o.emit.error("build failed \u2014 not deploying");
@@ -12923,7 +13034,7 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
           git(o.cwd, ["reset", "--hard", preBump]);
           o.emit.warn(`dropped the unreleased version commit for ${version} (reset ${o.repo.branch.base} to ${preBump.slice(0, 8)}) so a retry starts clean`);
         }
-        return { merged, conflicts, unbuildable, version, deployed: false, stopped: "build failed", decision, hookFailure: hookFailure(o, "build", r.output) };
+        return { merged, conflicts, unbuildable, version, deployed: false, stopped: "build failed", decision, hookFailure: hookFailure(o, "build", r.output), ...isolation ? { isolation } : {} };
       }
     }
   }
@@ -18294,21 +18405,25 @@ ${changed2.map((c) => `  - ${c}
           dryRun
         );
       }
+      const alertPlacement = async () => {
+        const placement = {};
+        try {
+          const repoId = repoIdForName(c, target.name);
+          if (repoId) {
+            placement.repo_id = repoId;
+            const projectId = (await tracker.repoRow(repoId))?.project_id;
+            if (projectId) placement.project_id = projectId;
+          }
+        } catch {
+        }
+        return placement;
+      };
       const news = describeRelease(outcome, scope);
       if (news) {
         const notified = await notify(c, cfg.ship, news, remit, dryRun);
         if (!notified && news.level === "fail") {
           const memberId = releaseSeat(c);
-          const placement = {};
-          try {
-            const repoId = repoIdForName(c, target.name);
-            if (repoId) {
-              placement.repo_id = repoId;
-              const projectId = (await tracker.repoRow(repoId))?.project_id;
-              if (projectId) placement.project_id = projectId;
-            }
-          } catch {
-          }
+          const placement = await alertPlacement();
           await applyFailureAlert(tracker, news, scope, all, memberId, remit, dryRun, placement).catch((e) => {
             remit.warn(`could not file/update a failure ticket: ${e.message}`, { step: "release" });
           });
@@ -18332,13 +18447,36 @@ ${changed2.map((c) => `  - ${c}
       }
       if (outcome.buildCulprits?.length) {
         const seat = releaseSeat(c);
+        const prior = [...await tracker.events("build_failed"), ...await tracker.events("pushed")];
         for (const cu of outcome.buildCulprits) {
           try {
             const bounce = planBuildFailureBounce(target.dir, cu, repo.branch.base, repo.branch.remote);
-            const r = await applyBuildFailureBounce(tracker, bounce, tracker.contract, seat, remit, dryRun);
+            const r = await applyBuildFailureBounce(
+              tracker,
+              bounce,
+              tracker.contract,
+              seat,
+              remit,
+              dryRun,
+              { count: priorBuildFailures(cu.ticket, prior), operatorId: c.resolved?.operator }
+            );
             if (r.kind === "failed") {
               remit.warn(`could not hand back ${cu.ticket.issue_id}: ${r.why}`, { step: "release" });
             }
+            if (r.kind === "escalated" && !dryRun) {
+              const key = cu.ticket.issue_tag ?? cu.ticket.issue_id;
+              await applyFailureAlert(tracker, {
+                level: "fail",
+                headline: `${scope}: release blocked \u2014 ${key} broke the ${cu.hook} gate twice`,
+                detail: `${key} (\`${cu.branch}\`) was handed back after a failed ${cu.hook} gate, pushed and re-verified, and failed it again. Isolation found it both times (${cu.how}), so it is escalated to the operator instead of another lap.
+
+\`\`\`
+${cu.tail.trimEnd()}
+\`\`\``
+              }, scope, all, seat, remit, dryRun, await alertPlacement()).catch((e) => {
+                remit.warn(`could not file/update a failure ticket: ${e.message}`, { step: "release" });
+              });
+            }
           } catch (e) {
             remit.warn(`could not hand back ${cu.ticket.issue_id}: ${e.message}`, { step: "release" });
           }
@@ -21546,6 +21684,8 @@ routes:            ${cfg.routes.map((c) => c.route).join(", ")}
           process.stdout.write(`                   - release readiness: ${p2}
 `);
         }
+        process.stdout.write(`                   release self-repair: ${selfRepairLine(config.release.isolateFailures?.maxCandidates)}
+`);
         const dh = doctorBaseReport(r, config.branch.base, baseHealth);
         process.stdout.write(dh.text);
         if (dh.failed) process.exitCode = 1;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tablation/crew",
-  "version": "0.79.10",
+  "version": "0.80.0",
   "description": "A standing team of headless agents that picks work off a Tablation board, does it on a machine you control, and reports back.",
   "type": "module",
   "license": "MIT",