@tablation/crew 0.79.9 → 0.80.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/cli.js +219 -49
  2. package/package.json +1 -1
package/dist/cli.js CHANGED
@@ -2981,9 +2981,11 @@ function describeRelease(o, route) {
2981
2981
  level: "fail",
2982
2982
  headline: `${route}: release blocked \u2014 tests failed`,
2983
2983
  detail: withHookTail(
2984
- o.composition ? `the target stays on the previous release
2984
+ (o.composition ? `the target stays on the previous release
2985
2985
 
2986
- ${describeComposition(o.composition)}` : "the target stays on the previous release",
2986
+ ${describeComposition(o.composition)}` : "the target stays on the previous release") + (o.isolation ? `
2987
+
2988
+ ${o.isolation}` : ""),
2987
2989
  o.hookFailure
2988
2990
  ),
2989
2991
  ...o.composition ? { head: o.composition.head } : {}
@@ -3013,7 +3015,9 @@ ${describeComposition(o.composition)}` : "the target stays on the previous relea
3013
3015
  return {
3014
3016
  level: "fail",
3015
3017
  headline: `${route}: build FAILED`,
3016
- detail: withHookTail("the target stays on the previous release \u2014 nothing was deployed", o.hookFailure)
3018
+ detail: withHookTail(`the target stays on the previous release \u2014 nothing was deployed${o.isolation ? `
3019
+
3020
+ ${o.isolation}` : ""}`, o.hookFailure)
3017
3021
  };
3018
3022
  }
3019
3023
  if (o.blockedCycles && o.blockedCycles >= 2) {
@@ -4326,7 +4330,7 @@ function reconcileHeldWorktrees(state, repo, log, stillRefused) {
4326
4330
  }
4327
4331
  }
4328
4332
  function openShipAttention(state, nowMs = Date.now()) {
4329
- const stale = staleRelease(state.releaseHeartbeat().at(), nowMs);
4333
+ const stale = state.releaseInProgress() ? null : staleRelease(state.releaseHeartbeat().at(), nowMs);
4330
4334
  const stored = state.shipAttention().previous();
4331
4335
  return stale ? [...stored, stale] : stored;
4332
4336
  }
@@ -5465,7 +5469,7 @@ var HandoffTranscriptTracker = class {
5465
5469
  };
5466
5470
 
5467
5471
  // src/pending-handoff.ts
5468
- import { existsSync as existsSync15, mkdirSync as mkdirSync8, readdirSync as readdirSync5, readFileSync as readFileSync13, rmSync as rmSync8, writeFileSync as writeFileSync9 } from "fs";
5472
+ import { existsSync as existsSync15, mkdirSync as mkdirSync8, readdirSync as readdirSync6, readFileSync as readFileSync13, rmSync as rmSync8, writeFileSync as writeFileSync9 } from "fs";
5469
5473
  import { join as join16 } from "path";
5470
5474
 
5471
5475
  // src/ticket-branch.ts
@@ -5476,7 +5480,7 @@ import { hostname } from "os";
5476
5480
  import { TablationClient } from "@tablation/client";
5477
5481
 
5478
5482
  // src/state.ts
5479
- import { readFileSync as readFileSync12, writeFileSync as writeFileSync8, existsSync as existsSync12, mkdirSync as mkdirSync7, rmSync as rmSync7, statSync as statSync4 } from "fs";
5483
+ import { readFileSync as readFileSync12, writeFileSync as writeFileSync8, existsSync as existsSync12, mkdirSync as mkdirSync7, rmSync as rmSync7, statSync as statSync4, readdirSync as readdirSync5 } from "fs";
5480
5484
  import { join as join13 } from "path";
5481
5485
  var EPOCH = "1970-01-01T00:00:00.000Z";
5482
5486
  var State = class _State {
@@ -5605,6 +5609,35 @@ var State = class _State {
5605
5609
  }
5606
5610
  };
5607
5611
  }
5612
+ /**
5613
+ * Whether a release is running right now: any `.release-<scope>.lock` in the
5614
+ * state directory naming a live process (the locks `releasePhase` takes per
5615
+ * scope). A lock naming a dead pid is ignored, as `acquire` ignores it.
5616
+ */
5617
+ releaseInProgress() {
5618
+ let names;
5619
+ try {
5620
+ names = readdirSync5(this.dir);
5621
+ } catch {
5622
+ return false;
5623
+ }
5624
+ for (const n of names) {
5625
+ if (!/^\.release-.+\.lock$/.test(n)) continue;
5626
+ let pid = 0;
5627
+ try {
5628
+ pid = Number.parseInt(readFileSync12(this.path(n), "utf8").trim(), 10);
5629
+ } catch {
5630
+ continue;
5631
+ }
5632
+ if (!Number.isFinite(pid) || pid <= 0) continue;
5633
+ try {
5634
+ process.kill(pid, 0);
5635
+ return true;
5636
+ } catch {
5637
+ }
5638
+ }
5639
+ return false;
5640
+ }
5608
5641
  /**
5609
5642
  * Like `acquire`, but for a resource with `max` interchangeable slots
5610
5643
  * rather than one exclusive holder — this machine's overall concurrency
@@ -7519,7 +7552,7 @@ function listPendingHandoffs(stateDir, route) {
7519
7552
  const dir = pendingDirFor(stateDir);
7520
7553
  if (!existsSync15(dir)) return [];
7521
7554
  const out = [];
7522
- for (const f of readdirSync5(dir).filter((n) => n.endsWith(".json")).sort()) {
7555
+ for (const f of readdirSync6(dir).filter((n) => n.endsWith(".json")).sort()) {
7523
7556
  try {
7524
7557
  const rec = JSON.parse(readFileSync13(join16(dir, f), "utf8"));
7525
7558
  if (rec.version === 1 && rec.ticketRecordId && (!route || rec.route === route)) out.push(rec);
@@ -7639,7 +7672,7 @@ function expandSkillMarkers(text, lookup, opts = {}) {
7639
7672
  }
7640
7673
 
7641
7674
  // src/skills.ts
7642
- import { readFileSync as readFileSync15, readdirSync as readdirSync6, existsSync as existsSync17 } from "fs";
7675
+ import { readFileSync as readFileSync15, readdirSync as readdirSync7, existsSync as existsSync17 } from "fs";
7643
7676
  import { join as join18 } from "path";
7644
7677
  import { parse as parseYaml } from "yaml";
7645
7678
  import { TablationClient as TablationClient3, StaleWriteError as StaleWriteError3, TablationApiError as TablationApiError2 } from "@tablation/client";
@@ -8138,7 +8171,7 @@ function parseSkillFile(path) {
8138
8171
  function listLocalSkills(promptsDir) {
8139
8172
  const dir = join18(promptsDir, "skills");
8140
8173
  if (!existsSync17(dir)) return [];
8141
- return readdirSync6(dir).filter((f) => f.endsWith(".md")).map((f) => parseSkillFile(join18(dir, f)));
8174
+ return readdirSync7(dir).filter((f) => f.endsWith(".md")).map((f) => parseSkillFile(join18(dir, f)));
8142
8175
  }
8143
8176
  async function syncSkills(route, opts) {
8144
8177
  if (!route.resolved) {
@@ -9725,7 +9758,7 @@ async function signOutSessions(o) {
9725
9758
  }
9726
9759
 
9727
9760
  // src/reset.ts
9728
- import { existsSync as existsSync20, mkdirSync as mkdirSync12, readFileSync as readFileSync19, readdirSync as readdirSync7, rmSync as rmSync11, statSync as statSync6, writeFileSync as writeFileSync13 } from "fs";
9761
+ import { existsSync as existsSync20, mkdirSync as mkdirSync12, readFileSync as readFileSync19, readdirSync as readdirSync8, rmSync as rmSync11, statSync as statSync6, writeFileSync as writeFileSync13 } from "fs";
9729
9762
  import { dirname as dirname9, join as join22 } from "path";
9730
9763
  import { hostFromUrl as hostFromUrl3 } from "@tablation/client";
9731
9764
  function clonedCheckoutsFile(stateDir) {
@@ -9820,7 +9853,7 @@ function removeState(stateDir, dryRun) {
9820
9853
  const out = [];
9821
9854
  const targets = [join22(stateDir, "resolved"), join22(stateDir, "keys"), clonedCheckoutsFile(stateDir), join22(stateDir, "connect-session.json"), join22(stateDir, "reload-requested"), join22(stateDir, "daemon.started")];
9822
9855
  try {
9823
- for (const name of readdirSync7(stateDir)) {
9856
+ for (const name of readdirSync8(stateDir)) {
9824
9857
  if (name.startsWith(".") && statSync6(join22(stateDir, name)).isFile()) targets.push(join22(stateDir, name));
9825
9858
  }
9826
9859
  } catch {
@@ -11282,7 +11315,7 @@ function planBuildFailureBounce(cwd, c, base, remote) {
11282
11315
  return { ...c, base, remote, worktree: worktreeForBranch(cwd, c.branch) };
11283
11316
  }
11284
11317
  function buildFailureComment(b) {
11285
- const how = b.how === "files" ? `The failing output names files that only \`${b.branch}\` changed among the branches in that release.` : `Built alone on the previous \`${b.base}\`, \`${b.branch}\` fails the same hook.`;
11318
+ const how = b.how === "pair" && b.sibling ? `Each branch passes the hook alone; \`${b.branch}\` together with ${b.sibling.key} (\`${b.sibling.branch}\`) fails it. ${b.sibling.key} was verified first, so the fix belongs on this branch: \`git merge ${b.remote}/${b.sibling.branch}\` first, then fix what disagrees. Fix it on this branch, never a fresh one.` : b.how === "files" ? `The failing output names files that only \`${b.branch}\` changed among the branches in that release.` : `Built alone on the previous \`${b.base}\`, \`${b.branch}\` fails the same hook.`;
11286
11319
  return `${BUILD_FAILURE_MARKER}
11287
11320
 
11288
11321
  **Back with the dev lane \u2014 this branch builds on its own but not on current \`${b.base}\`.**
@@ -11297,8 +11330,46 @@ ${b.tail.trimEnd()}
11297
11330
 
11298
11331
  Nothing was written to \`${b.base}\` for this ticket; the rest of the release shipped without it.`;
11299
11332
  }
11300
- async function applyBuildFailureBounce(writer, b, contract, memberId, log, dryRun) {
11333
+ function priorBuildFailures(ticket, events) {
11334
+ const stamp = (e) => e.occurred_at ?? e.created_at ?? "";
11335
+ const mine = events.filter((e) => e.record_id === ticket.id);
11336
+ const pushes = mine.filter((e) => e.event_type === "pushed").map(stamp);
11337
+ return mine.filter((e) => e.event_type === "build_failed" && pushes.some((p) => p > stamp(e))).length;
11338
+ }
11339
+ function buildFailureEscalation(b) {
11340
+ return `${BUILD_FAILURE_MARKER}
11341
+
11342
+ **Failed the release gate again after a hand-back \u2014 stopping rather than looping.**
11343
+
11344
+ \`${b.branch}\` was handed back for breaking the \`${b.hook}\` hook, pushed, verified again, and breaks it once more. That is a call for a person, not another lap through the lanes.
11345
+
11346
+ \`\`\`
11347
+ ${b.tail.trimEnd()}
11348
+ \`\`\``;
11349
+ }
11350
+ async function applyBuildFailureBounce(writer, b, contract, memberId, log, dryRun, prior = { count: 0 }) {
11301
11351
  const t = { ticket: b.ticket.issue_id, step: "release" };
11352
+ if (prior.count > 0) {
11353
+ if (dryRun) {
11354
+ log.emit(`would escalate to ${contract.statuses.needsHuman} \u2014 handed back for the ${b.hook} hook before`, t);
11355
+ return { kind: "escalated" };
11356
+ }
11357
+ try {
11358
+ await writer.updateTicket(
11359
+ b.ticket.id,
11360
+ { status: contract.statuses.needsHuman, ...prior.operatorId ? { assignee_id: prior.operatorId } : {} },
11361
+ b.ticket.updated_at
11362
+ );
11363
+ await writer.postEvent(b.ticket.id, buildFailureEscalation(b), memberId, "build_failed_escalated").catch(() => {
11364
+ log.warn("escalated, but the note failed to post", t);
11365
+ });
11366
+ log.warn(`breaks the ${b.hook} hook again after a hand-back \u2014 escalated to ${contract.statuses.needsHuman}`, t);
11367
+ return { kind: "escalated" };
11368
+ } catch (e) {
11369
+ if (e instanceof StaleWriteError8) return { kind: "stale" };
11370
+ return { kind: "failed", why: e.message };
11371
+ }
11372
+ }
11302
11373
  if (dryRun) {
11303
11374
  log.emit(`would hand ${b.ticket.issue_id} back to ${contract.statuses.building} \u2014 its branch breaks the ${b.hook} hook`, t);
11304
11375
  return { kind: "handed-back" };
@@ -11312,6 +11383,13 @@ async function applyBuildFailureBounce(writer, b, contract, memberId, log, dryRu
11312
11383
  await writer.postEvent(b.ticket.id, buildFailureComment(b), memberId, "build_failed").catch(() => {
11313
11384
  log.warn("handed back, but the note failed to post", t);
11314
11385
  });
11386
+ await writer.postEvent(
11387
+ b.ticket.id,
11388
+ `${b.ticket.issue_tag ?? b.ticket.issue_id} left out of the release: ${b.hook} hook, found by ${b.how}${b.sibling ? `, together with ${b.sibling.key} (${b.sibling.branch})` : ""}`,
11389
+ memberId,
11390
+ "release_culprit"
11391
+ ).catch(() => {
11392
+ });
11315
11393
  log.emit(`handed back to ${contract.statuses.building} \u2014 breaks the ${b.hook} hook once merged with the rest`, t);
11316
11394
  return { kind: "handed-back" };
11317
11395
  } catch (e) {
@@ -12643,6 +12721,25 @@ async function runReleaseCycle(o, movedPast2, localRefSkips) {
12643
12721
  }
12644
12722
  }
12645
12723
  }
12724
+ var MAX_ISOLATE_PAIRS = 10;
12725
+ function pairOrder(n, max = MAX_ISOLATE_PAIRS) {
12726
+ const pairs = [];
12727
+ let truncated = false;
12728
+ outer: for (let j = 1; j < n; j++) {
12729
+ for (let i = 0; i < j; i++) {
12730
+ if (pairs.length >= max) {
12731
+ truncated = true;
12732
+ break outer;
12733
+ }
12734
+ pairs.push([i, j]);
12735
+ }
12736
+ }
12737
+ return { pairs, truncated };
12738
+ }
12739
+ function selfRepairLine(maxCandidates) {
12740
+ const cap = maxCandidates ?? DEFAULT_ISOLATE_MAX_CANDIDATES;
12741
+ return cap === 0 ? "culprit isolation is off (release.isolateFailures.maxCandidates: 0); a failed gate files an alert" : `culprit isolation up to ${cap} merged branches, ${MAX_ISOLATE_PAIRS} pairs before bisecting; a ticket handed back twice after a push escalates with an alert`;
12742
+ }
12646
12743
  var isolateCap = (o) => o.repo.release.isolateFailures?.maxCandidates ?? DEFAULT_ISOLATE_MAX_CANDIDATES;
12647
12744
  async function isolateAndRetry(o, name, output, merged, preMerge, decision, fresh, releaseBranch) {
12648
12745
  const applied = merged.filter((c) => c.sha && c.branch);
@@ -12667,13 +12764,13 @@ async function isolateAndRetry(o, name, output, merged, preMerge, decision, fres
12667
12764
  if (baseRun && baseRun.code !== 0) {
12668
12765
  o.emit.warn(`${name} fails on the base without ${only.branch} \u2014 not blaming the branch`, { ticket: only.ticket.issue_id });
12669
12766
  gitOk(o.cwd, ["reset", "--hard", head]);
12670
- return null;
12767
+ return { tried: `Isolation found no culprit: the ${name} hook also fails on the base alone, without ${only.branch}, so there is no branch to hand back.` };
12671
12768
  }
12672
12769
  o.emit.warn(`${name} failed with only ${only.branch} merged and passes without it \u2014 treating it as the culprit`, { ticket: only.ticket.issue_id });
12673
12770
  culprits.push({ c: only, how: "solo", tail: lastLines(output, HOOK_FAILURE_TAIL_LINES) });
12674
12771
  } else if (applied.length > cap) {
12675
12772
  o.emit.warn(`${name} failed with ${applied.length} branches merged, over release.isolateFailures.maxCandidates (${cap}) \u2014 not isolating`);
12676
- return null;
12773
+ return { tried: `Isolation was skipped: ${applied.length} branches were merged, over release.isolateFailures.maxCandidates (${cap}).` };
12677
12774
  } else {
12678
12775
  o.emit.warn(`${name} failed with ${applied.length} branches merged \u2014 building each alone to find the culprit`);
12679
12776
  for (const c of applied) {
@@ -12686,12 +12783,51 @@ async function isolateAndRetry(o, name, output, merged, preMerge, decision, fres
12686
12783
  culprits.push({ c, how: "solo", tail: lastLines(r.output, HOOK_FAILURE_TAIL_LINES) });
12687
12784
  }
12688
12785
  }
12786
+ if (culprits.length === 0) {
12787
+ const { pairs, truncated } = pairOrder(applied.length);
12788
+ for (const [i, j] of pairs) {
12789
+ gitOk(o.cwd, ["reset", "--hard", preMerge]);
12790
+ const a = mergeOne(o, applied[i]);
12791
+ const b = typeof a === "object" && "conflict" in a ? a : mergeOne(o, applied[j]);
12792
+ if (typeof b === "object" && "conflict" in b) continue;
12793
+ const r = await hook(o, name);
12794
+ if (r && r.code !== 0) {
12795
+ o.emit.warn(`${applied[j].branch} fails the ${name} hook together with ${applied[i].branch}`, { ticket: applied[j].ticket.issue_id });
12796
+ culprits.push({ c: applied[j], how: "pair", sibling: applied[i], tail: lastLines(r.output, HOOK_FAILURE_TAIL_LINES) });
12797
+ break;
12798
+ }
12799
+ }
12800
+ if (culprits.length === 0 && truncated) {
12801
+ let lo = 2, hi = applied.length;
12802
+ let tail = lastLines(output, HOOK_FAILURE_TAIL_LINES);
12803
+ while (lo < hi) {
12804
+ const mid = Math.floor((lo + hi) / 2);
12805
+ gitOk(o.cwd, ["reset", "--hard", preMerge]);
12806
+ let clean = true;
12807
+ for (const c of applied.slice(0, mid)) {
12808
+ const m = mergeOne(o, c);
12809
+ if (typeof m === "object" && "conflict" in m) {
12810
+ clean = false;
12811
+ break;
12812
+ }
12813
+ }
12814
+ const r = clean ? await hook(o, name) : null;
12815
+ if (r && r.code !== 0) {
12816
+ hi = mid;
12817
+ tail = lastLines(r.output, HOOK_FAILURE_TAIL_LINES);
12818
+ } else lo = mid + 1;
12819
+ }
12820
+ const owner = applied[hi - 1];
12821
+ o.emit.warn(`${owner.branch} is the first branch whose prefix fails the ${name} hook (bisected over ${applied.length})`, { ticket: owner.ticket.issue_id });
12822
+ culprits.push({ c: owner, how: "pair", sibling: applied[hi - 2], tail });
12823
+ }
12824
+ }
12689
12825
  }
12690
12826
  gitOk(o.cwd, ["reset", "--hard", preMerge]);
12691
12827
  if (culprits.length === 0) {
12692
12828
  o.emit.warn("no single branch could be blamed \u2014 failing the release as a whole");
12693
12829
  gitOk(o.cwd, ["reset", "--hard", head]);
12694
- return null;
12830
+ return { tried: `Isolation found no culprit: each of the ${applied.length} merged branches passes the ${name} hook alone and no pair among them fails it.` };
12695
12831
  }
12696
12832
  const out = new Set(culprits.map((x) => x.c.ticket.id));
12697
12833
  const rest = decision.merges.filter((c) => !out.has(c.ticket.id));
@@ -12703,7 +12839,8 @@ async function isolateAndRetry(o, name, output, merged, preMerge, decision, fres
12703
12839
  branch: x.c.branch,
12704
12840
  hook: name,
12705
12841
  tail: x.tail,
12706
- how: x.how
12842
+ how: x.how,
12843
+ ...x.sibling ? { sibling: { key: x.sibling.ticket.issue_tag ?? x.sibling.ticket.issue_id, branch: x.sibling.branch } } : {}
12707
12844
  }))
12708
12845
  };
12709
12846
  }
@@ -12714,6 +12851,7 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
12714
12851
  const conflicts = [];
12715
12852
  const unbuildable = [];
12716
12853
  let applied = 0;
12854
+ let isolation;
12717
12855
  for (const c of decision.merges) {
12718
12856
  if (!c.branch || c.mergedSha) {
12719
12857
  if (c.skipReason === "already-released") {
@@ -12826,7 +12964,8 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
12826
12964
  }
12827
12965
  if (r && r.code !== 0 && !isolated) {
12828
12966
  const out = await isolateAndRetry(o, "test", r.output, merged, preMerge, decision, fresh, releaseBranch);
12829
- if (out) return out;
12967
+ if (out && "tried" in out) isolation = out.tried;
12968
+ else if (out) return out;
12830
12969
  }
12831
12970
  if (r && r.code !== 0) {
12832
12971
  o.emit.error(`test gate FAILED \u2014 not deploying; the target stays on the previous release`);
@@ -12847,7 +12986,7 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
12847
12986
  label: hookLabel(o.repo, "test"),
12848
12987
  tail: failureSummary(r.output, HOOK_FAILURE_TAIL_LINES)
12849
12988
  };
12850
- return { merged, conflicts, unbuildable, deployed: false, stopped: "tests failed", decision, gateRed, hookFailure: hookFailure2, composition };
12989
+ return { merged, conflicts, unbuildable, deployed: false, stopped: "tests failed", decision, gateRed, hookFailure: hookFailure2, composition, ...isolation ? { isolation } : {} };
12851
12990
  }
12852
12991
  o.state?.clearTestGateFailed?.();
12853
12992
  }
@@ -12886,7 +13025,8 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
12886
13025
  }
12887
13026
  if (r && r.code !== 0 && !isolated) {
12888
13027
  const out = await isolateAndRetry(o, "build", r.output, merged, preMerge, decision, fresh, releaseBranch);
12889
- if (out) return out;
13028
+ if (out && "tried" in out) isolation = out.tried;
13029
+ else if (out) return out;
12890
13030
  }
12891
13031
  if (r && r.code !== 0) {
12892
13032
  o.emit.error("build failed \u2014 not deploying");
@@ -12894,7 +13034,7 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
12894
13034
  git(o.cwd, ["reset", "--hard", preBump]);
12895
13035
  o.emit.warn(`dropped the unreleased version commit for ${version} (reset ${o.repo.branch.base} to ${preBump.slice(0, 8)}) so a retry starts clean`);
12896
13036
  }
12897
- return { merged, conflicts, unbuildable, version, deployed: false, stopped: "build failed", decision, hookFailure: hookFailure(o, "build", r.output) };
13037
+ return { merged, conflicts, unbuildable, version, deployed: false, stopped: "build failed", decision, hookFailure: hookFailure(o, "build", r.output), ...isolation ? { isolation } : {} };
12898
13038
  }
12899
13039
  }
12900
13040
  }
@@ -15506,7 +15646,7 @@ function schedulerInstalled(plan, exists, crontab) {
15506
15646
  }
15507
15647
 
15508
15648
  // src/worktree-sweep.ts
15509
- import { existsSync as existsSync27, readFileSync as readFileSync25, readdirSync as readdirSync8 } from "fs";
15649
+ import { existsSync as existsSync27, readFileSync as readFileSync25, readdirSync as readdirSync9 } from "fs";
15510
15650
  import { basename as basename11, dirname as dirname18, join as join30, resolve as resolve4 } from "path";
15511
15651
  function planWorktreeSweep(target, tickets, contract) {
15512
15652
  const terminal = new Set(contract.statuses.resolved.filter((s) => s !== contract.statuses.verified));
@@ -15566,7 +15706,7 @@ function siblingWorktreeDirs(target) {
15566
15706
  const registered = new Set(worktrees(target.dir).map((w) => resolve4(w.path)));
15567
15707
  let names;
15568
15708
  try {
15569
- names = readdirSync8(parent);
15709
+ names = readdirSync9(parent);
15570
15710
  } catch {
15571
15711
  return [];
15572
15712
  }
@@ -15782,14 +15922,14 @@ function applyRemoteBranchCleanup(cwd, remote, branches2, dryRun, log) {
15782
15922
  }
15783
15923
 
15784
15924
  // src/stream-sweep.ts
15785
- import { existsSync as existsSync28, readdirSync as readdirSync9, rmSync as rmSync13, statSync as statSync8 } from "fs";
15925
+ import { existsSync as existsSync28, readdirSync as readdirSync10, rmSync as rmSync13, statSync as statSync8 } from "fs";
15786
15926
  import { join as join31 } from "path";
15787
15927
  var DAY_MS2 = 24 * 60 * 60 * 1e3;
15788
15928
  function planStreamSweep(stateDir, retentionDays, now = Date.now()) {
15789
15929
  const dir = join31(stateDir, "streams");
15790
15930
  if (!existsSync28(dir)) return [];
15791
15931
  const cutoff = now - retentionDays * DAY_MS2;
15792
- return readdirSync9(dir).map((name) => join31(dir, name)).filter((path) => {
15932
+ return readdirSync10(dir).map((name) => join31(dir, name)).filter((path) => {
15793
15933
  try {
15794
15934
  return statSync8(path).mtimeMs < cutoff;
15795
15935
  } catch {
@@ -15817,7 +15957,7 @@ function applyStreamSweep(paths, dryRun, log) {
15817
15957
  function planEventsSweep(stateDir, retentionDays, now = Date.now()) {
15818
15958
  if (!existsSync28(stateDir)) return [];
15819
15959
  const cutoff = now - retentionDays * DAY_MS2;
15820
- return readdirSync9(stateDir).filter((name) => ROTATED_EVENTS_RE.test(name)).map((name) => join31(stateDir, name)).filter((path) => {
15960
+ return readdirSync10(stateDir).filter((name) => ROTATED_EVENTS_RE.test(name)).map((name) => join31(stateDir, name)).filter((path) => {
15821
15961
  try {
15822
15962
  return statSync8(path).mtimeMs < cutoff;
15823
15963
  } catch {
@@ -15827,7 +15967,7 @@ function planEventsSweep(stateDir, retentionDays, now = Date.now()) {
15827
15967
  }
15828
15968
 
15829
15969
  // src/cli.ts
15830
- import { realpathSync as realpathSync3, readFileSync as readFileSync31, existsSync as existsSync34, mkdirSync as mkdirSync22, writeFileSync as writeFileSync23, rmSync as rmSync15, unlinkSync as unlinkSync5, copyFileSync as copyFileSync2, renameSync as renameSync3, statSync as statSync10, readdirSync as readdirSync14 } from "fs";
15970
+ import { realpathSync as realpathSync3, readFileSync as readFileSync31, existsSync as existsSync34, mkdirSync as mkdirSync22, writeFileSync as writeFileSync23, rmSync as rmSync15, unlinkSync as unlinkSync5, copyFileSync as copyFileSync2, renameSync as renameSync3, statSync as statSync10, readdirSync as readdirSync15 } from "fs";
15831
15971
  import { createInterface } from "readline/promises";
15832
15972
  import { dirname as dirOf, resolve as resolvePath } from "path";
15833
15973
 
@@ -16084,7 +16224,7 @@ function syncAllPassengerCheckouts(cfg) {
16084
16224
 
16085
16225
  // src/tunnel.ts
16086
16226
  import { spawn as spawn5 } from "child_process";
16087
- import { existsSync as existsSync30, mkdirSync as mkdirSync19, readFileSync as readFileSync27, writeFileSync as writeFileSync20, unlinkSync as unlinkSync3, readdirSync as readdirSync10 } from "fs";
16227
+ import { existsSync as existsSync30, mkdirSync as mkdirSync19, readFileSync as readFileSync27, writeFileSync as writeFileSync20, unlinkSync as unlinkSync3, readdirSync as readdirSync11 } from "fs";
16088
16228
  import { randomInt } from "crypto";
16089
16229
  import { join as join33 } from "path";
16090
16230
  var DEFAULT_SPAWN_OPTIONS = { stdio: "ignore", detached: true };
@@ -16266,7 +16406,7 @@ function syncPassengerTunnels(plans, relayHost, relayPort, privateKeyPath, state
16266
16406
  function readAllPersistedWorkspaceIds(stateDir) {
16267
16407
  const dir = join33(stateDir, "passengers");
16268
16408
  if (!existsSync30(dir)) return [];
16269
- return readdirSync10(dir).filter((f) => f.endsWith(".json") && !f.endsWith("-sync.json")).map((f) => f.slice(0, -".json".length));
16409
+ return readdirSync11(dir).filter((f) => f.endsWith(".json") && !f.endsWith("-sync.json")).map((f) => f.slice(0, -".json".length));
16270
16410
  }
16271
16411
 
16272
16412
  // src/mcp-endpoint-registry.ts
@@ -16386,7 +16526,7 @@ async function unregisterEndpoint(workspaceId, info, d) {
16386
16526
  import { createServer as createHttpServer } from "http";
16387
16527
  import { spawn as spawn6 } from "child_process";
16388
16528
  import { randomBytes } from "crypto";
16389
- import { existsSync as existsSync31, mkdirSync as mkdirSync20, readFileSync as readFileSync28, writeFileSync as writeFileSync21, unlinkSync as unlinkSync4, readdirSync as readdirSync11 } from "fs";
16529
+ import { existsSync as existsSync31, mkdirSync as mkdirSync20, readFileSync as readFileSync28, writeFileSync as writeFileSync21, unlinkSync as unlinkSync4, readdirSync as readdirSync12 } from "fs";
16390
16530
  import { join as join34 } from "path";
16391
16531
 
16392
16532
  // src/secret-compare.ts
@@ -16513,7 +16653,7 @@ function readAllPersistedSyncWorkspaceIds(stateDir) {
16513
16653
  const dir = join34(stateDir, "passengers");
16514
16654
  if (!existsSync31(dir)) return [];
16515
16655
  const suffix = "-sync.json";
16516
- return readdirSync11(dir).filter((f) => f.endsWith(suffix)).map((f) => f.slice(0, -suffix.length));
16656
+ return readdirSync12(dir).filter((f) => f.endsWith(suffix)).map((f) => f.slice(0, -suffix.length));
16517
16657
  }
16518
16658
  function syncPassengerSyncDaemons(plans, cfg, stateDir, deps = {}) {
16519
16659
  const alive = deps.isPidAlive ?? isPidAlive;
@@ -16599,11 +16739,11 @@ function describeCause(err) {
16599
16739
  }
16600
16740
 
16601
16741
  // src/daemon.ts
16602
- import { existsSync as existsSync33, mkdirSync as mkdirSync21, readFileSync as readFileSync30, readdirSync as readdirSync13, rmSync as rmSync14, statSync as statSync9, writeFileSync as writeFileSync22 } from "fs";
16742
+ import { existsSync as existsSync33, mkdirSync as mkdirSync21, readFileSync as readFileSync30, readdirSync as readdirSync14, rmSync as rmSync14, statSync as statSync9, writeFileSync as writeFileSync22 } from "fs";
16603
16743
  import { join as join36 } from "path";
16604
16744
 
16605
16745
  // src/interrupted.ts
16606
- import { existsSync as existsSync32, readdirSync as readdirSync12, readFileSync as readFileSync29 } from "fs";
16746
+ import { existsSync as existsSync32, readdirSync as readdirSync13, readFileSync as readFileSync29 } from "fs";
16607
16747
  import { join as join35 } from "path";
16608
16748
  var INTERRUPTED_MARKER = calloutHeader("NOTE", "Interrupted");
16609
16749
  function interruptedComment(b) {
@@ -16635,7 +16775,7 @@ function orphanedStreams(o) {
16635
16775
  if (!existsSync32(dir)) return [];
16636
16776
  const ownerAlive = o.isOwnerAlive ?? pidAlive;
16637
16777
  const out = [];
16638
- for (const name of readdirSync12(dir)) {
16778
+ for (const name of readdirSync13(dir)) {
16639
16779
  if (!name.endsWith(".jsonl") || name.endsWith(".events.jsonl")) continue;
16640
16780
  const streamPath = join35(dir, name);
16641
16781
  if (runFinished(streamPath)) continue;
@@ -16756,7 +16896,7 @@ function latestMtime(path) {
16756
16896
  if (stat.isFile()) return stat.mtimeMs;
16757
16897
  if (!stat.isDirectory()) return 0;
16758
16898
  let latest = stat.mtimeMs;
16759
- for (const entry of readdirSync13(path, { withFileTypes: true })) {
16899
+ for (const entry of readdirSync14(path, { withFileTypes: true })) {
16760
16900
  latest = Math.max(latest, latestMtime(join36(path, entry.name)));
16761
16901
  }
16762
16902
  return latest;
@@ -17531,6 +17671,7 @@ if (command === "passenger-sync-daemon") {
17531
17671
  }, releaseRefused = function() {
17532
17672
  const why = state.releaseSkipReason();
17533
17673
  if (!why) return false;
17674
+ if (!dryRun) state.releaseHeartbeat().touch();
17534
17675
  emit.emit(`${why} \u2014 release skipped`, { step: "release" });
17535
17676
  return true;
17536
17677
  }, doctorMachineId = function(stateDir) {
@@ -18264,21 +18405,25 @@ ${changed2.map((c) => ` - ${c}
18264
18405
  dryRun
18265
18406
  );
18266
18407
  }
18408
+ const alertPlacement = async () => {
18409
+ const placement = {};
18410
+ try {
18411
+ const repoId = repoIdForName(c, target.name);
18412
+ if (repoId) {
18413
+ placement.repo_id = repoId;
18414
+ const projectId = (await tracker.repoRow(repoId))?.project_id;
18415
+ if (projectId) placement.project_id = projectId;
18416
+ }
18417
+ } catch {
18418
+ }
18419
+ return placement;
18420
+ };
18267
18421
  const news = describeRelease(outcome, scope);
18268
18422
  if (news) {
18269
18423
  const notified = await notify(c, cfg.ship, news, remit, dryRun);
18270
18424
  if (!notified && news.level === "fail") {
18271
18425
  const memberId = releaseSeat(c);
18272
- const placement = {};
18273
- try {
18274
- const repoId = repoIdForName(c, target.name);
18275
- if (repoId) {
18276
- placement.repo_id = repoId;
18277
- const projectId = (await tracker.repoRow(repoId))?.project_id;
18278
- if (projectId) placement.project_id = projectId;
18279
- }
18280
- } catch {
18281
- }
18426
+ const placement = await alertPlacement();
18282
18427
  await applyFailureAlert(tracker, news, scope, all, memberId, remit, dryRun, placement).catch((e) => {
18283
18428
  remit.warn(`could not file/update a failure ticket: ${e.message}`, { step: "release" });
18284
18429
  });
@@ -18302,13 +18447,36 @@ ${changed2.map((c) => ` - ${c}
18302
18447
  }
18303
18448
  if (outcome.buildCulprits?.length) {
18304
18449
  const seat = releaseSeat(c);
18450
+ const prior = [...await tracker.events("build_failed"), ...await tracker.events("pushed")];
18305
18451
  for (const cu of outcome.buildCulprits) {
18306
18452
  try {
18307
18453
  const bounce = planBuildFailureBounce(target.dir, cu, repo.branch.base, repo.branch.remote);
18308
- const r = await applyBuildFailureBounce(tracker, bounce, tracker.contract, seat, remit, dryRun);
18454
+ const r = await applyBuildFailureBounce(
18455
+ tracker,
18456
+ bounce,
18457
+ tracker.contract,
18458
+ seat,
18459
+ remit,
18460
+ dryRun,
18461
+ { count: priorBuildFailures(cu.ticket, prior), operatorId: c.resolved?.operator }
18462
+ );
18309
18463
  if (r.kind === "failed") {
18310
18464
  remit.warn(`could not hand back ${cu.ticket.issue_id}: ${r.why}`, { step: "release" });
18311
18465
  }
18466
+ if (r.kind === "escalated" && !dryRun) {
18467
+ const key = cu.ticket.issue_tag ?? cu.ticket.issue_id;
18468
+ await applyFailureAlert(tracker, {
18469
+ level: "fail",
18470
+ headline: `${scope}: release blocked \u2014 ${key} broke the ${cu.hook} gate twice`,
18471
+ detail: `${key} (\`${cu.branch}\`) was handed back after a failed ${cu.hook} gate, pushed and re-verified, and failed it again. Isolation found it both times (${cu.how}), so it is escalated to the operator instead of another lap.
18472
+
18473
+ \`\`\`
18474
+ ${cu.tail.trimEnd()}
18475
+ \`\`\``
18476
+ }, scope, all, seat, remit, dryRun, await alertPlacement()).catch((e) => {
18477
+ remit.warn(`could not file/update a failure ticket: ${e.message}`, { step: "release" });
18478
+ });
18479
+ }
18312
18480
  } catch (e) {
18313
18481
  remit.warn(`could not hand back ${cu.ticket.issue_id}: ${e.message}`, { step: "release" });
18314
18482
  }
@@ -19281,7 +19449,7 @@ ${changed2.map((c) => ` - ${c}
19281
19449
  if (!existsSync34(dir)) throw new ReposError(`${dir} does not exist on disk`);
19282
19450
  const files = { exists: (r) => existsSync34(resolvePath(dir, r)), list: (r) => {
19283
19451
  try {
19284
- return readdirSync14(resolvePath(dir, r));
19452
+ return readdirSync15(resolvePath(dir, r));
19285
19453
  } catch {
19286
19454
  return [];
19287
19455
  }
@@ -19478,7 +19646,7 @@ ${changed2.map((c) => ` - ${c}
19478
19646
  ` : `${human}
19479
19647
  `);
19480
19648
  const plan2 = planRepoClone({ text, base, route: routeName, name: addName, resolved: resolvedRemotes, dir: value("dir"), isCheckout });
19481
- if (existsSync34(plan2.target) && (existsSync34(`${plan2.target}/.git`) || readdirSync14(plan2.target).length > 0)) {
19649
+ if (existsSync34(plan2.target) && (existsSync34(`${plan2.target}/.git`) || readdirSync15(plan2.target).length > 0)) {
19482
19650
  throw new ReposError(
19483
19651
  existsSync34(`${plan2.target}/.git`) ? `${plan2.target} is already a checkout \u2014 attach it with \`crew repos add ${routeName} ${plan2.name} ${plan2.target}\`` : `${plan2.target} exists and is not empty \u2014 pass --dir to clone somewhere else`
19484
19652
  );
@@ -21030,7 +21198,7 @@ Signed in as ${result.identity.email} \u2014 minted an API key in workspace "${r
21030
21198
  }
21031
21199
  const projectId = route.resolved.areaId ?? resolveRepoProject(await tracker.projectRows(), value("project"));
21032
21200
  const fresh = !existsSync34(join37(dir, ".git"));
21033
- if (existsSync34(dir) && fresh && readdirSync14(dir).length > 0) fail(`${dir} exists and is not a git checkout \u2014 move it aside`);
21201
+ if (existsSync34(dir) && fresh && readdirSync15(dir).length > 0) fail(`${dir} exists and is not a git checkout \u2014 move it aside`);
21034
21202
  if (dryRun) {
21035
21203
  say("dry-run", { dir, newCheckout: fresh, repoRow: row?.id ?? null }, `would ${fresh ? "create" : "reuse"} ${dir}${row ? "" : ", create the Repos row"}, and file one ${tracker.contract.statuses.approved} ticket`);
21036
21204
  break;
@@ -21516,6 +21684,8 @@ routes: ${cfg.routes.map((c) => c.route).join(", ")}
21516
21684
  process.stdout.write(` - release readiness: ${p2}
21517
21685
  `);
21518
21686
  }
21687
+ process.stdout.write(` release self-repair: ${selfRepairLine(config.release.isolateFailures?.maxCandidates)}
21688
+ `);
21519
21689
  const dh = doctorBaseReport(r, config.branch.base, baseHealth);
21520
21690
  process.stdout.write(dh.text);
21521
21691
  if (dh.failed) process.exitCode = 1;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tablation/crew",
3
- "version": "0.79.9",
3
+ "version": "0.80.0",
4
4
  "description": "A standing team of headless agents that picks work off a Tablation board, does it on a machine you control, and reports back.",
5
5
  "type": "module",
6
6
  "license": "MIT",