@tablation/crew 0.79.9 → 0.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +219 -49
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -2981,9 +2981,11 @@ function describeRelease(o, route) {
|
|
|
2981
2981
|
level: "fail",
|
|
2982
2982
|
headline: `${route}: release blocked \u2014 tests failed`,
|
|
2983
2983
|
detail: withHookTail(
|
|
2984
|
-
o.composition ? `the target stays on the previous release
|
|
2984
|
+
(o.composition ? `the target stays on the previous release
|
|
2985
2985
|
|
|
2986
|
-
${describeComposition(o.composition)}` : "the target stays on the previous release"
|
|
2986
|
+
${describeComposition(o.composition)}` : "the target stays on the previous release") + (o.isolation ? `
|
|
2987
|
+
|
|
2988
|
+
${o.isolation}` : ""),
|
|
2987
2989
|
o.hookFailure
|
|
2988
2990
|
),
|
|
2989
2991
|
...o.composition ? { head: o.composition.head } : {}
|
|
@@ -3013,7 +3015,9 @@ ${describeComposition(o.composition)}` : "the target stays on the previous relea
|
|
|
3013
3015
|
return {
|
|
3014
3016
|
level: "fail",
|
|
3015
3017
|
headline: `${route}: build FAILED`,
|
|
3016
|
-
detail: withHookTail(
|
|
3018
|
+
detail: withHookTail(`the target stays on the previous release \u2014 nothing was deployed${o.isolation ? `
|
|
3019
|
+
|
|
3020
|
+
${o.isolation}` : ""}`, o.hookFailure)
|
|
3017
3021
|
};
|
|
3018
3022
|
}
|
|
3019
3023
|
if (o.blockedCycles && o.blockedCycles >= 2) {
|
|
@@ -4326,7 +4330,7 @@ function reconcileHeldWorktrees(state, repo, log, stillRefused) {
|
|
|
4326
4330
|
}
|
|
4327
4331
|
}
|
|
4328
4332
|
function openShipAttention(state, nowMs = Date.now()) {
|
|
4329
|
-
const stale = staleRelease(state.releaseHeartbeat().at(), nowMs);
|
|
4333
|
+
const stale = state.releaseInProgress() ? null : staleRelease(state.releaseHeartbeat().at(), nowMs);
|
|
4330
4334
|
const stored = state.shipAttention().previous();
|
|
4331
4335
|
return stale ? [...stored, stale] : stored;
|
|
4332
4336
|
}
|
|
@@ -5465,7 +5469,7 @@ var HandoffTranscriptTracker = class {
|
|
|
5465
5469
|
};
|
|
5466
5470
|
|
|
5467
5471
|
// src/pending-handoff.ts
|
|
5468
|
-
import { existsSync as existsSync15, mkdirSync as mkdirSync8, readdirSync as
|
|
5472
|
+
import { existsSync as existsSync15, mkdirSync as mkdirSync8, readdirSync as readdirSync6, readFileSync as readFileSync13, rmSync as rmSync8, writeFileSync as writeFileSync9 } from "fs";
|
|
5469
5473
|
import { join as join16 } from "path";
|
|
5470
5474
|
|
|
5471
5475
|
// src/ticket-branch.ts
|
|
@@ -5476,7 +5480,7 @@ import { hostname } from "os";
|
|
|
5476
5480
|
import { TablationClient } from "@tablation/client";
|
|
5477
5481
|
|
|
5478
5482
|
// src/state.ts
|
|
5479
|
-
import { readFileSync as readFileSync12, writeFileSync as writeFileSync8, existsSync as existsSync12, mkdirSync as mkdirSync7, rmSync as rmSync7, statSync as statSync4 } from "fs";
|
|
5483
|
+
import { readFileSync as readFileSync12, writeFileSync as writeFileSync8, existsSync as existsSync12, mkdirSync as mkdirSync7, rmSync as rmSync7, statSync as statSync4, readdirSync as readdirSync5 } from "fs";
|
|
5480
5484
|
import { join as join13 } from "path";
|
|
5481
5485
|
var EPOCH = "1970-01-01T00:00:00.000Z";
|
|
5482
5486
|
var State = class _State {
|
|
@@ -5605,6 +5609,35 @@ var State = class _State {
|
|
|
5605
5609
|
}
|
|
5606
5610
|
};
|
|
5607
5611
|
}
|
|
5612
|
+
/**
|
|
5613
|
+
* Whether a release is running right now: any `.release-<scope>.lock` in the
|
|
5614
|
+
* state directory naming a live process (the locks `releasePhase` takes per
|
|
5615
|
+
* scope). A lock naming a dead pid is ignored, as `acquire` ignores it.
|
|
5616
|
+
*/
|
|
5617
|
+
releaseInProgress() {
|
|
5618
|
+
let names;
|
|
5619
|
+
try {
|
|
5620
|
+
names = readdirSync5(this.dir);
|
|
5621
|
+
} catch {
|
|
5622
|
+
return false;
|
|
5623
|
+
}
|
|
5624
|
+
for (const n of names) {
|
|
5625
|
+
if (!/^\.release-.+\.lock$/.test(n)) continue;
|
|
5626
|
+
let pid = 0;
|
|
5627
|
+
try {
|
|
5628
|
+
pid = Number.parseInt(readFileSync12(this.path(n), "utf8").trim(), 10);
|
|
5629
|
+
} catch {
|
|
5630
|
+
continue;
|
|
5631
|
+
}
|
|
5632
|
+
if (!Number.isFinite(pid) || pid <= 0) continue;
|
|
5633
|
+
try {
|
|
5634
|
+
process.kill(pid, 0);
|
|
5635
|
+
return true;
|
|
5636
|
+
} catch {
|
|
5637
|
+
}
|
|
5638
|
+
}
|
|
5639
|
+
return false;
|
|
5640
|
+
}
|
|
5608
5641
|
/**
|
|
5609
5642
|
* Like `acquire`, but for a resource with `max` interchangeable slots
|
|
5610
5643
|
* rather than one exclusive holder — this machine's overall concurrency
|
|
@@ -7519,7 +7552,7 @@ function listPendingHandoffs(stateDir, route) {
|
|
|
7519
7552
|
const dir = pendingDirFor(stateDir);
|
|
7520
7553
|
if (!existsSync15(dir)) return [];
|
|
7521
7554
|
const out = [];
|
|
7522
|
-
for (const f of
|
|
7555
|
+
for (const f of readdirSync6(dir).filter((n) => n.endsWith(".json")).sort()) {
|
|
7523
7556
|
try {
|
|
7524
7557
|
const rec = JSON.parse(readFileSync13(join16(dir, f), "utf8"));
|
|
7525
7558
|
if (rec.version === 1 && rec.ticketRecordId && (!route || rec.route === route)) out.push(rec);
|
|
@@ -7639,7 +7672,7 @@ function expandSkillMarkers(text, lookup, opts = {}) {
|
|
|
7639
7672
|
}
|
|
7640
7673
|
|
|
7641
7674
|
// src/skills.ts
|
|
7642
|
-
import { readFileSync as readFileSync15, readdirSync as
|
|
7675
|
+
import { readFileSync as readFileSync15, readdirSync as readdirSync7, existsSync as existsSync17 } from "fs";
|
|
7643
7676
|
import { join as join18 } from "path";
|
|
7644
7677
|
import { parse as parseYaml } from "yaml";
|
|
7645
7678
|
import { TablationClient as TablationClient3, StaleWriteError as StaleWriteError3, TablationApiError as TablationApiError2 } from "@tablation/client";
|
|
@@ -8138,7 +8171,7 @@ function parseSkillFile(path) {
|
|
|
8138
8171
|
function listLocalSkills(promptsDir) {
|
|
8139
8172
|
const dir = join18(promptsDir, "skills");
|
|
8140
8173
|
if (!existsSync17(dir)) return [];
|
|
8141
|
-
return
|
|
8174
|
+
return readdirSync7(dir).filter((f) => f.endsWith(".md")).map((f) => parseSkillFile(join18(dir, f)));
|
|
8142
8175
|
}
|
|
8143
8176
|
async function syncSkills(route, opts) {
|
|
8144
8177
|
if (!route.resolved) {
|
|
@@ -9725,7 +9758,7 @@ async function signOutSessions(o) {
|
|
|
9725
9758
|
}
|
|
9726
9759
|
|
|
9727
9760
|
// src/reset.ts
|
|
9728
|
-
import { existsSync as existsSync20, mkdirSync as mkdirSync12, readFileSync as readFileSync19, readdirSync as
|
|
9761
|
+
import { existsSync as existsSync20, mkdirSync as mkdirSync12, readFileSync as readFileSync19, readdirSync as readdirSync8, rmSync as rmSync11, statSync as statSync6, writeFileSync as writeFileSync13 } from "fs";
|
|
9729
9762
|
import { dirname as dirname9, join as join22 } from "path";
|
|
9730
9763
|
import { hostFromUrl as hostFromUrl3 } from "@tablation/client";
|
|
9731
9764
|
function clonedCheckoutsFile(stateDir) {
|
|
@@ -9820,7 +9853,7 @@ function removeState(stateDir, dryRun) {
|
|
|
9820
9853
|
const out = [];
|
|
9821
9854
|
const targets = [join22(stateDir, "resolved"), join22(stateDir, "keys"), clonedCheckoutsFile(stateDir), join22(stateDir, "connect-session.json"), join22(stateDir, "reload-requested"), join22(stateDir, "daemon.started")];
|
|
9822
9855
|
try {
|
|
9823
|
-
for (const name of
|
|
9856
|
+
for (const name of readdirSync8(stateDir)) {
|
|
9824
9857
|
if (name.startsWith(".") && statSync6(join22(stateDir, name)).isFile()) targets.push(join22(stateDir, name));
|
|
9825
9858
|
}
|
|
9826
9859
|
} catch {
|
|
@@ -11282,7 +11315,7 @@ function planBuildFailureBounce(cwd, c, base, remote) {
|
|
|
11282
11315
|
return { ...c, base, remote, worktree: worktreeForBranch(cwd, c.branch) };
|
|
11283
11316
|
}
|
|
11284
11317
|
function buildFailureComment(b) {
|
|
11285
|
-
const how = b.how === "files" ? `The failing output names files that only \`${b.branch}\` changed among the branches in that release.` : `Built alone on the previous \`${b.base}\`, \`${b.branch}\` fails the same hook.`;
|
|
11318
|
+
const how = b.how === "pair" && b.sibling ? `Each branch passes the hook alone; \`${b.branch}\` together with ${b.sibling.key} (\`${b.sibling.branch}\`) fails it. ${b.sibling.key} was verified first, so the fix belongs on this branch: \`git merge ${b.remote}/${b.sibling.branch}\` first, then fix what disagrees. Fix it on this branch, never a fresh one.` : b.how === "files" ? `The failing output names files that only \`${b.branch}\` changed among the branches in that release.` : `Built alone on the previous \`${b.base}\`, \`${b.branch}\` fails the same hook.`;
|
|
11286
11319
|
return `${BUILD_FAILURE_MARKER}
|
|
11287
11320
|
|
|
11288
11321
|
**Back with the dev lane \u2014 this branch builds on its own but not on current \`${b.base}\`.**
|
|
@@ -11297,8 +11330,46 @@ ${b.tail.trimEnd()}
|
|
|
11297
11330
|
|
|
11298
11331
|
Nothing was written to \`${b.base}\` for this ticket; the rest of the release shipped without it.`;
|
|
11299
11332
|
}
|
|
11300
|
-
|
|
11333
|
+
function priorBuildFailures(ticket, events) {
|
|
11334
|
+
const stamp = (e) => e.occurred_at ?? e.created_at ?? "";
|
|
11335
|
+
const mine = events.filter((e) => e.record_id === ticket.id);
|
|
11336
|
+
const pushes = mine.filter((e) => e.event_type === "pushed").map(stamp);
|
|
11337
|
+
return mine.filter((e) => e.event_type === "build_failed" && pushes.some((p) => p > stamp(e))).length;
|
|
11338
|
+
}
|
|
11339
|
+
function buildFailureEscalation(b) {
|
|
11340
|
+
return `${BUILD_FAILURE_MARKER}
|
|
11341
|
+
|
|
11342
|
+
**Failed the release gate again after a hand-back \u2014 stopping rather than looping.**
|
|
11343
|
+
|
|
11344
|
+
\`${b.branch}\` was handed back for breaking the \`${b.hook}\` hook, pushed, verified again, and breaks it once more. That is a call for a person, not another lap through the lanes.
|
|
11345
|
+
|
|
11346
|
+
\`\`\`
|
|
11347
|
+
${b.tail.trimEnd()}
|
|
11348
|
+
\`\`\``;
|
|
11349
|
+
}
|
|
11350
|
+
async function applyBuildFailureBounce(writer, b, contract, memberId, log, dryRun, prior = { count: 0 }) {
|
|
11301
11351
|
const t = { ticket: b.ticket.issue_id, step: "release" };
|
|
11352
|
+
if (prior.count > 0) {
|
|
11353
|
+
if (dryRun) {
|
|
11354
|
+
log.emit(`would escalate to ${contract.statuses.needsHuman} \u2014 handed back for the ${b.hook} hook before`, t);
|
|
11355
|
+
return { kind: "escalated" };
|
|
11356
|
+
}
|
|
11357
|
+
try {
|
|
11358
|
+
await writer.updateTicket(
|
|
11359
|
+
b.ticket.id,
|
|
11360
|
+
{ status: contract.statuses.needsHuman, ...prior.operatorId ? { assignee_id: prior.operatorId } : {} },
|
|
11361
|
+
b.ticket.updated_at
|
|
11362
|
+
);
|
|
11363
|
+
await writer.postEvent(b.ticket.id, buildFailureEscalation(b), memberId, "build_failed_escalated").catch(() => {
|
|
11364
|
+
log.warn("escalated, but the note failed to post", t);
|
|
11365
|
+
});
|
|
11366
|
+
log.warn(`breaks the ${b.hook} hook again after a hand-back \u2014 escalated to ${contract.statuses.needsHuman}`, t);
|
|
11367
|
+
return { kind: "escalated" };
|
|
11368
|
+
} catch (e) {
|
|
11369
|
+
if (e instanceof StaleWriteError8) return { kind: "stale" };
|
|
11370
|
+
return { kind: "failed", why: e.message };
|
|
11371
|
+
}
|
|
11372
|
+
}
|
|
11302
11373
|
if (dryRun) {
|
|
11303
11374
|
log.emit(`would hand ${b.ticket.issue_id} back to ${contract.statuses.building} \u2014 its branch breaks the ${b.hook} hook`, t);
|
|
11304
11375
|
return { kind: "handed-back" };
|
|
@@ -11312,6 +11383,13 @@ async function applyBuildFailureBounce(writer, b, contract, memberId, log, dryRu
|
|
|
11312
11383
|
await writer.postEvent(b.ticket.id, buildFailureComment(b), memberId, "build_failed").catch(() => {
|
|
11313
11384
|
log.warn("handed back, but the note failed to post", t);
|
|
11314
11385
|
});
|
|
11386
|
+
await writer.postEvent(
|
|
11387
|
+
b.ticket.id,
|
|
11388
|
+
`${b.ticket.issue_tag ?? b.ticket.issue_id} left out of the release: ${b.hook} hook, found by ${b.how}${b.sibling ? `, together with ${b.sibling.key} (${b.sibling.branch})` : ""}`,
|
|
11389
|
+
memberId,
|
|
11390
|
+
"release_culprit"
|
|
11391
|
+
).catch(() => {
|
|
11392
|
+
});
|
|
11315
11393
|
log.emit(`handed back to ${contract.statuses.building} \u2014 breaks the ${b.hook} hook once merged with the rest`, t);
|
|
11316
11394
|
return { kind: "handed-back" };
|
|
11317
11395
|
} catch (e) {
|
|
@@ -12643,6 +12721,25 @@ async function runReleaseCycle(o, movedPast2, localRefSkips) {
|
|
|
12643
12721
|
}
|
|
12644
12722
|
}
|
|
12645
12723
|
}
|
|
12724
|
+
var MAX_ISOLATE_PAIRS = 10;
|
|
12725
|
+
function pairOrder(n, max = MAX_ISOLATE_PAIRS) {
|
|
12726
|
+
const pairs = [];
|
|
12727
|
+
let truncated = false;
|
|
12728
|
+
outer: for (let j = 1; j < n; j++) {
|
|
12729
|
+
for (let i = 0; i < j; i++) {
|
|
12730
|
+
if (pairs.length >= max) {
|
|
12731
|
+
truncated = true;
|
|
12732
|
+
break outer;
|
|
12733
|
+
}
|
|
12734
|
+
pairs.push([i, j]);
|
|
12735
|
+
}
|
|
12736
|
+
}
|
|
12737
|
+
return { pairs, truncated };
|
|
12738
|
+
}
|
|
12739
|
+
function selfRepairLine(maxCandidates) {
|
|
12740
|
+
const cap = maxCandidates ?? DEFAULT_ISOLATE_MAX_CANDIDATES;
|
|
12741
|
+
return cap === 0 ? "culprit isolation is off (release.isolateFailures.maxCandidates: 0); a failed gate files an alert" : `culprit isolation up to ${cap} merged branches, ${MAX_ISOLATE_PAIRS} pairs before bisecting; a ticket handed back twice after a push escalates with an alert`;
|
|
12742
|
+
}
|
|
12646
12743
|
var isolateCap = (o) => o.repo.release.isolateFailures?.maxCandidates ?? DEFAULT_ISOLATE_MAX_CANDIDATES;
|
|
12647
12744
|
async function isolateAndRetry(o, name, output, merged, preMerge, decision, fresh, releaseBranch) {
|
|
12648
12745
|
const applied = merged.filter((c) => c.sha && c.branch);
|
|
@@ -12667,13 +12764,13 @@ async function isolateAndRetry(o, name, output, merged, preMerge, decision, fres
|
|
|
12667
12764
|
if (baseRun && baseRun.code !== 0) {
|
|
12668
12765
|
o.emit.warn(`${name} fails on the base without ${only.branch} \u2014 not blaming the branch`, { ticket: only.ticket.issue_id });
|
|
12669
12766
|
gitOk(o.cwd, ["reset", "--hard", head]);
|
|
12670
|
-
return
|
|
12767
|
+
return { tried: `Isolation found no culprit: the ${name} hook also fails on the base alone, without ${only.branch}, so there is no branch to hand back.` };
|
|
12671
12768
|
}
|
|
12672
12769
|
o.emit.warn(`${name} failed with only ${only.branch} merged and passes without it \u2014 treating it as the culprit`, { ticket: only.ticket.issue_id });
|
|
12673
12770
|
culprits.push({ c: only, how: "solo", tail: lastLines(output, HOOK_FAILURE_TAIL_LINES) });
|
|
12674
12771
|
} else if (applied.length > cap) {
|
|
12675
12772
|
o.emit.warn(`${name} failed with ${applied.length} branches merged, over release.isolateFailures.maxCandidates (${cap}) \u2014 not isolating`);
|
|
12676
|
-
return
|
|
12773
|
+
return { tried: `Isolation was skipped: ${applied.length} branches were merged, over release.isolateFailures.maxCandidates (${cap}).` };
|
|
12677
12774
|
} else {
|
|
12678
12775
|
o.emit.warn(`${name} failed with ${applied.length} branches merged \u2014 building each alone to find the culprit`);
|
|
12679
12776
|
for (const c of applied) {
|
|
@@ -12686,12 +12783,51 @@ async function isolateAndRetry(o, name, output, merged, preMerge, decision, fres
|
|
|
12686
12783
|
culprits.push({ c, how: "solo", tail: lastLines(r.output, HOOK_FAILURE_TAIL_LINES) });
|
|
12687
12784
|
}
|
|
12688
12785
|
}
|
|
12786
|
+
if (culprits.length === 0) {
|
|
12787
|
+
const { pairs, truncated } = pairOrder(applied.length);
|
|
12788
|
+
for (const [i, j] of pairs) {
|
|
12789
|
+
gitOk(o.cwd, ["reset", "--hard", preMerge]);
|
|
12790
|
+
const a = mergeOne(o, applied[i]);
|
|
12791
|
+
const b = typeof a === "object" && "conflict" in a ? a : mergeOne(o, applied[j]);
|
|
12792
|
+
if (typeof b === "object" && "conflict" in b) continue;
|
|
12793
|
+
const r = await hook(o, name);
|
|
12794
|
+
if (r && r.code !== 0) {
|
|
12795
|
+
o.emit.warn(`${applied[j].branch} fails the ${name} hook together with ${applied[i].branch}`, { ticket: applied[j].ticket.issue_id });
|
|
12796
|
+
culprits.push({ c: applied[j], how: "pair", sibling: applied[i], tail: lastLines(r.output, HOOK_FAILURE_TAIL_LINES) });
|
|
12797
|
+
break;
|
|
12798
|
+
}
|
|
12799
|
+
}
|
|
12800
|
+
if (culprits.length === 0 && truncated) {
|
|
12801
|
+
let lo = 2, hi = applied.length;
|
|
12802
|
+
let tail = lastLines(output, HOOK_FAILURE_TAIL_LINES);
|
|
12803
|
+
while (lo < hi) {
|
|
12804
|
+
const mid = Math.floor((lo + hi) / 2);
|
|
12805
|
+
gitOk(o.cwd, ["reset", "--hard", preMerge]);
|
|
12806
|
+
let clean = true;
|
|
12807
|
+
for (const c of applied.slice(0, mid)) {
|
|
12808
|
+
const m = mergeOne(o, c);
|
|
12809
|
+
if (typeof m === "object" && "conflict" in m) {
|
|
12810
|
+
clean = false;
|
|
12811
|
+
break;
|
|
12812
|
+
}
|
|
12813
|
+
}
|
|
12814
|
+
const r = clean ? await hook(o, name) : null;
|
|
12815
|
+
if (r && r.code !== 0) {
|
|
12816
|
+
hi = mid;
|
|
12817
|
+
tail = lastLines(r.output, HOOK_FAILURE_TAIL_LINES);
|
|
12818
|
+
} else lo = mid + 1;
|
|
12819
|
+
}
|
|
12820
|
+
const owner = applied[hi - 1];
|
|
12821
|
+
o.emit.warn(`${owner.branch} is the first branch whose prefix fails the ${name} hook (bisected over ${applied.length})`, { ticket: owner.ticket.issue_id });
|
|
12822
|
+
culprits.push({ c: owner, how: "pair", sibling: applied[hi - 2], tail });
|
|
12823
|
+
}
|
|
12824
|
+
}
|
|
12689
12825
|
}
|
|
12690
12826
|
gitOk(o.cwd, ["reset", "--hard", preMerge]);
|
|
12691
12827
|
if (culprits.length === 0) {
|
|
12692
12828
|
o.emit.warn("no single branch could be blamed \u2014 failing the release as a whole");
|
|
12693
12829
|
gitOk(o.cwd, ["reset", "--hard", head]);
|
|
12694
|
-
return
|
|
12830
|
+
return { tried: `Isolation found no culprit: each of the ${applied.length} merged branches passes the ${name} hook alone and no pair among them fails it.` };
|
|
12695
12831
|
}
|
|
12696
12832
|
const out = new Set(culprits.map((x) => x.c.ticket.id));
|
|
12697
12833
|
const rest = decision.merges.filter((c) => !out.has(c.ticket.id));
|
|
@@ -12703,7 +12839,8 @@ async function isolateAndRetry(o, name, output, merged, preMerge, decision, fres
|
|
|
12703
12839
|
branch: x.c.branch,
|
|
12704
12840
|
hook: name,
|
|
12705
12841
|
tail: x.tail,
|
|
12706
|
-
how: x.how
|
|
12842
|
+
how: x.how,
|
|
12843
|
+
...x.sibling ? { sibling: { key: x.sibling.ticket.issue_tag ?? x.sibling.ticket.issue_id, branch: x.sibling.branch } } : {}
|
|
12707
12844
|
}))
|
|
12708
12845
|
};
|
|
12709
12846
|
}
|
|
@@ -12714,6 +12851,7 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
|
|
|
12714
12851
|
const conflicts = [];
|
|
12715
12852
|
const unbuildable = [];
|
|
12716
12853
|
let applied = 0;
|
|
12854
|
+
let isolation;
|
|
12717
12855
|
for (const c of decision.merges) {
|
|
12718
12856
|
if (!c.branch || c.mergedSha) {
|
|
12719
12857
|
if (c.skipReason === "already-released") {
|
|
@@ -12826,7 +12964,8 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
|
|
|
12826
12964
|
}
|
|
12827
12965
|
if (r && r.code !== 0 && !isolated) {
|
|
12828
12966
|
const out = await isolateAndRetry(o, "test", r.output, merged, preMerge, decision, fresh, releaseBranch);
|
|
12829
|
-
if (out)
|
|
12967
|
+
if (out && "tried" in out) isolation = out.tried;
|
|
12968
|
+
else if (out) return out;
|
|
12830
12969
|
}
|
|
12831
12970
|
if (r && r.code !== 0) {
|
|
12832
12971
|
o.emit.error(`test gate FAILED \u2014 not deploying; the target stays on the previous release`);
|
|
@@ -12847,7 +12986,7 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
|
|
|
12847
12986
|
label: hookLabel(o.repo, "test"),
|
|
12848
12987
|
tail: failureSummary(r.output, HOOK_FAILURE_TAIL_LINES)
|
|
12849
12988
|
};
|
|
12850
|
-
return { merged, conflicts, unbuildable, deployed: false, stopped: "tests failed", decision, gateRed, hookFailure: hookFailure2, composition };
|
|
12989
|
+
return { merged, conflicts, unbuildable, deployed: false, stopped: "tests failed", decision, gateRed, hookFailure: hookFailure2, composition, ...isolation ? { isolation } : {} };
|
|
12851
12990
|
}
|
|
12852
12991
|
o.state?.clearTestGateFailed?.();
|
|
12853
12992
|
}
|
|
@@ -12886,7 +13025,8 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
|
|
|
12886
13025
|
}
|
|
12887
13026
|
if (r && r.code !== 0 && !isolated) {
|
|
12888
13027
|
const out = await isolateAndRetry(o, "build", r.output, merged, preMerge, decision, fresh, releaseBranch);
|
|
12889
|
-
if (out)
|
|
13028
|
+
if (out && "tried" in out) isolation = out.tried;
|
|
13029
|
+
else if (out) return out;
|
|
12890
13030
|
}
|
|
12891
13031
|
if (r && r.code !== 0) {
|
|
12892
13032
|
o.emit.error("build failed \u2014 not deploying");
|
|
@@ -12894,7 +13034,7 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = fal
|
|
|
12894
13034
|
git(o.cwd, ["reset", "--hard", preBump]);
|
|
12895
13035
|
o.emit.warn(`dropped the unreleased version commit for ${version} (reset ${o.repo.branch.base} to ${preBump.slice(0, 8)}) so a retry starts clean`);
|
|
12896
13036
|
}
|
|
12897
|
-
return { merged, conflicts, unbuildable, version, deployed: false, stopped: "build failed", decision, hookFailure: hookFailure(o, "build", r.output) };
|
|
13037
|
+
return { merged, conflicts, unbuildable, version, deployed: false, stopped: "build failed", decision, hookFailure: hookFailure(o, "build", r.output), ...isolation ? { isolation } : {} };
|
|
12898
13038
|
}
|
|
12899
13039
|
}
|
|
12900
13040
|
}
|
|
@@ -15506,7 +15646,7 @@ function schedulerInstalled(plan, exists, crontab) {
|
|
|
15506
15646
|
}
|
|
15507
15647
|
|
|
15508
15648
|
// src/worktree-sweep.ts
|
|
15509
|
-
import { existsSync as existsSync27, readFileSync as readFileSync25, readdirSync as
|
|
15649
|
+
import { existsSync as existsSync27, readFileSync as readFileSync25, readdirSync as readdirSync9 } from "fs";
|
|
15510
15650
|
import { basename as basename11, dirname as dirname18, join as join30, resolve as resolve4 } from "path";
|
|
15511
15651
|
function planWorktreeSweep(target, tickets, contract) {
|
|
15512
15652
|
const terminal = new Set(contract.statuses.resolved.filter((s) => s !== contract.statuses.verified));
|
|
@@ -15566,7 +15706,7 @@ function siblingWorktreeDirs(target) {
|
|
|
15566
15706
|
const registered = new Set(worktrees(target.dir).map((w) => resolve4(w.path)));
|
|
15567
15707
|
let names;
|
|
15568
15708
|
try {
|
|
15569
|
-
names =
|
|
15709
|
+
names = readdirSync9(parent);
|
|
15570
15710
|
} catch {
|
|
15571
15711
|
return [];
|
|
15572
15712
|
}
|
|
@@ -15782,14 +15922,14 @@ function applyRemoteBranchCleanup(cwd, remote, branches2, dryRun, log) {
|
|
|
15782
15922
|
}
|
|
15783
15923
|
|
|
15784
15924
|
// src/stream-sweep.ts
|
|
15785
|
-
import { existsSync as existsSync28, readdirSync as
|
|
15925
|
+
import { existsSync as existsSync28, readdirSync as readdirSync10, rmSync as rmSync13, statSync as statSync8 } from "fs";
|
|
15786
15926
|
import { join as join31 } from "path";
|
|
15787
15927
|
var DAY_MS2 = 24 * 60 * 60 * 1e3;
|
|
15788
15928
|
function planStreamSweep(stateDir, retentionDays, now = Date.now()) {
|
|
15789
15929
|
const dir = join31(stateDir, "streams");
|
|
15790
15930
|
if (!existsSync28(dir)) return [];
|
|
15791
15931
|
const cutoff = now - retentionDays * DAY_MS2;
|
|
15792
|
-
return
|
|
15932
|
+
return readdirSync10(dir).map((name) => join31(dir, name)).filter((path) => {
|
|
15793
15933
|
try {
|
|
15794
15934
|
return statSync8(path).mtimeMs < cutoff;
|
|
15795
15935
|
} catch {
|
|
@@ -15817,7 +15957,7 @@ function applyStreamSweep(paths, dryRun, log) {
|
|
|
15817
15957
|
function planEventsSweep(stateDir, retentionDays, now = Date.now()) {
|
|
15818
15958
|
if (!existsSync28(stateDir)) return [];
|
|
15819
15959
|
const cutoff = now - retentionDays * DAY_MS2;
|
|
15820
|
-
return
|
|
15960
|
+
return readdirSync10(stateDir).filter((name) => ROTATED_EVENTS_RE.test(name)).map((name) => join31(stateDir, name)).filter((path) => {
|
|
15821
15961
|
try {
|
|
15822
15962
|
return statSync8(path).mtimeMs < cutoff;
|
|
15823
15963
|
} catch {
|
|
@@ -15827,7 +15967,7 @@ function planEventsSweep(stateDir, retentionDays, now = Date.now()) {
|
|
|
15827
15967
|
}
|
|
15828
15968
|
|
|
15829
15969
|
// src/cli.ts
|
|
15830
|
-
import { realpathSync as realpathSync3, readFileSync as readFileSync31, existsSync as existsSync34, mkdirSync as mkdirSync22, writeFileSync as writeFileSync23, rmSync as rmSync15, unlinkSync as unlinkSync5, copyFileSync as copyFileSync2, renameSync as renameSync3, statSync as statSync10, readdirSync as
|
|
15970
|
+
import { realpathSync as realpathSync3, readFileSync as readFileSync31, existsSync as existsSync34, mkdirSync as mkdirSync22, writeFileSync as writeFileSync23, rmSync as rmSync15, unlinkSync as unlinkSync5, copyFileSync as copyFileSync2, renameSync as renameSync3, statSync as statSync10, readdirSync as readdirSync15 } from "fs";
|
|
15831
15971
|
import { createInterface } from "readline/promises";
|
|
15832
15972
|
import { dirname as dirOf, resolve as resolvePath } from "path";
|
|
15833
15973
|
|
|
@@ -16084,7 +16224,7 @@ function syncAllPassengerCheckouts(cfg) {
|
|
|
16084
16224
|
|
|
16085
16225
|
// src/tunnel.ts
|
|
16086
16226
|
import { spawn as spawn5 } from "child_process";
|
|
16087
|
-
import { existsSync as existsSync30, mkdirSync as mkdirSync19, readFileSync as readFileSync27, writeFileSync as writeFileSync20, unlinkSync as unlinkSync3, readdirSync as
|
|
16227
|
+
import { existsSync as existsSync30, mkdirSync as mkdirSync19, readFileSync as readFileSync27, writeFileSync as writeFileSync20, unlinkSync as unlinkSync3, readdirSync as readdirSync11 } from "fs";
|
|
16088
16228
|
import { randomInt } from "crypto";
|
|
16089
16229
|
import { join as join33 } from "path";
|
|
16090
16230
|
var DEFAULT_SPAWN_OPTIONS = { stdio: "ignore", detached: true };
|
|
@@ -16266,7 +16406,7 @@ function syncPassengerTunnels(plans, relayHost, relayPort, privateKeyPath, state
|
|
|
16266
16406
|
function readAllPersistedWorkspaceIds(stateDir) {
|
|
16267
16407
|
const dir = join33(stateDir, "passengers");
|
|
16268
16408
|
if (!existsSync30(dir)) return [];
|
|
16269
|
-
return
|
|
16409
|
+
return readdirSync11(dir).filter((f) => f.endsWith(".json") && !f.endsWith("-sync.json")).map((f) => f.slice(0, -".json".length));
|
|
16270
16410
|
}
|
|
16271
16411
|
|
|
16272
16412
|
// src/mcp-endpoint-registry.ts
|
|
@@ -16386,7 +16526,7 @@ async function unregisterEndpoint(workspaceId, info, d) {
|
|
|
16386
16526
|
import { createServer as createHttpServer } from "http";
|
|
16387
16527
|
import { spawn as spawn6 } from "child_process";
|
|
16388
16528
|
import { randomBytes } from "crypto";
|
|
16389
|
-
import { existsSync as existsSync31, mkdirSync as mkdirSync20, readFileSync as readFileSync28, writeFileSync as writeFileSync21, unlinkSync as unlinkSync4, readdirSync as
|
|
16529
|
+
import { existsSync as existsSync31, mkdirSync as mkdirSync20, readFileSync as readFileSync28, writeFileSync as writeFileSync21, unlinkSync as unlinkSync4, readdirSync as readdirSync12 } from "fs";
|
|
16390
16530
|
import { join as join34 } from "path";
|
|
16391
16531
|
|
|
16392
16532
|
// src/secret-compare.ts
|
|
@@ -16513,7 +16653,7 @@ function readAllPersistedSyncWorkspaceIds(stateDir) {
|
|
|
16513
16653
|
const dir = join34(stateDir, "passengers");
|
|
16514
16654
|
if (!existsSync31(dir)) return [];
|
|
16515
16655
|
const suffix = "-sync.json";
|
|
16516
|
-
return
|
|
16656
|
+
return readdirSync12(dir).filter((f) => f.endsWith(suffix)).map((f) => f.slice(0, -suffix.length));
|
|
16517
16657
|
}
|
|
16518
16658
|
function syncPassengerSyncDaemons(plans, cfg, stateDir, deps = {}) {
|
|
16519
16659
|
const alive = deps.isPidAlive ?? isPidAlive;
|
|
@@ -16599,11 +16739,11 @@ function describeCause(err) {
|
|
|
16599
16739
|
}
|
|
16600
16740
|
|
|
16601
16741
|
// src/daemon.ts
|
|
16602
|
-
import { existsSync as existsSync33, mkdirSync as mkdirSync21, readFileSync as readFileSync30, readdirSync as
|
|
16742
|
+
import { existsSync as existsSync33, mkdirSync as mkdirSync21, readFileSync as readFileSync30, readdirSync as readdirSync14, rmSync as rmSync14, statSync as statSync9, writeFileSync as writeFileSync22 } from "fs";
|
|
16603
16743
|
import { join as join36 } from "path";
|
|
16604
16744
|
|
|
16605
16745
|
// src/interrupted.ts
|
|
16606
|
-
import { existsSync as existsSync32, readdirSync as
|
|
16746
|
+
import { existsSync as existsSync32, readdirSync as readdirSync13, readFileSync as readFileSync29 } from "fs";
|
|
16607
16747
|
import { join as join35 } from "path";
|
|
16608
16748
|
var INTERRUPTED_MARKER = calloutHeader("NOTE", "Interrupted");
|
|
16609
16749
|
function interruptedComment(b) {
|
|
@@ -16635,7 +16775,7 @@ function orphanedStreams(o) {
|
|
|
16635
16775
|
if (!existsSync32(dir)) return [];
|
|
16636
16776
|
const ownerAlive = o.isOwnerAlive ?? pidAlive;
|
|
16637
16777
|
const out = [];
|
|
16638
|
-
for (const name of
|
|
16778
|
+
for (const name of readdirSync13(dir)) {
|
|
16639
16779
|
if (!name.endsWith(".jsonl") || name.endsWith(".events.jsonl")) continue;
|
|
16640
16780
|
const streamPath = join35(dir, name);
|
|
16641
16781
|
if (runFinished(streamPath)) continue;
|
|
@@ -16756,7 +16896,7 @@ function latestMtime(path) {
|
|
|
16756
16896
|
if (stat.isFile()) return stat.mtimeMs;
|
|
16757
16897
|
if (!stat.isDirectory()) return 0;
|
|
16758
16898
|
let latest = stat.mtimeMs;
|
|
16759
|
-
for (const entry of
|
|
16899
|
+
for (const entry of readdirSync14(path, { withFileTypes: true })) {
|
|
16760
16900
|
latest = Math.max(latest, latestMtime(join36(path, entry.name)));
|
|
16761
16901
|
}
|
|
16762
16902
|
return latest;
|
|
@@ -17531,6 +17671,7 @@ if (command === "passenger-sync-daemon") {
|
|
|
17531
17671
|
}, releaseRefused = function() {
|
|
17532
17672
|
const why = state.releaseSkipReason();
|
|
17533
17673
|
if (!why) return false;
|
|
17674
|
+
if (!dryRun) state.releaseHeartbeat().touch();
|
|
17534
17675
|
emit.emit(`${why} \u2014 release skipped`, { step: "release" });
|
|
17535
17676
|
return true;
|
|
17536
17677
|
}, doctorMachineId = function(stateDir) {
|
|
@@ -18264,21 +18405,25 @@ ${changed2.map((c) => ` - ${c}
|
|
|
18264
18405
|
dryRun
|
|
18265
18406
|
);
|
|
18266
18407
|
}
|
|
18408
|
+
const alertPlacement = async () => {
|
|
18409
|
+
const placement = {};
|
|
18410
|
+
try {
|
|
18411
|
+
const repoId = repoIdForName(c, target.name);
|
|
18412
|
+
if (repoId) {
|
|
18413
|
+
placement.repo_id = repoId;
|
|
18414
|
+
const projectId = (await tracker.repoRow(repoId))?.project_id;
|
|
18415
|
+
if (projectId) placement.project_id = projectId;
|
|
18416
|
+
}
|
|
18417
|
+
} catch {
|
|
18418
|
+
}
|
|
18419
|
+
return placement;
|
|
18420
|
+
};
|
|
18267
18421
|
const news = describeRelease(outcome, scope);
|
|
18268
18422
|
if (news) {
|
|
18269
18423
|
const notified = await notify(c, cfg.ship, news, remit, dryRun);
|
|
18270
18424
|
if (!notified && news.level === "fail") {
|
|
18271
18425
|
const memberId = releaseSeat(c);
|
|
18272
|
-
const placement =
|
|
18273
|
-
try {
|
|
18274
|
-
const repoId = repoIdForName(c, target.name);
|
|
18275
|
-
if (repoId) {
|
|
18276
|
-
placement.repo_id = repoId;
|
|
18277
|
-
const projectId = (await tracker.repoRow(repoId))?.project_id;
|
|
18278
|
-
if (projectId) placement.project_id = projectId;
|
|
18279
|
-
}
|
|
18280
|
-
} catch {
|
|
18281
|
-
}
|
|
18426
|
+
const placement = await alertPlacement();
|
|
18282
18427
|
await applyFailureAlert(tracker, news, scope, all, memberId, remit, dryRun, placement).catch((e) => {
|
|
18283
18428
|
remit.warn(`could not file/update a failure ticket: ${e.message}`, { step: "release" });
|
|
18284
18429
|
});
|
|
@@ -18302,13 +18447,36 @@ ${changed2.map((c) => ` - ${c}
|
|
|
18302
18447
|
}
|
|
18303
18448
|
if (outcome.buildCulprits?.length) {
|
|
18304
18449
|
const seat = releaseSeat(c);
|
|
18450
|
+
const prior = [...await tracker.events("build_failed"), ...await tracker.events("pushed")];
|
|
18305
18451
|
for (const cu of outcome.buildCulprits) {
|
|
18306
18452
|
try {
|
|
18307
18453
|
const bounce = planBuildFailureBounce(target.dir, cu, repo.branch.base, repo.branch.remote);
|
|
18308
|
-
const r = await applyBuildFailureBounce(
|
|
18454
|
+
const r = await applyBuildFailureBounce(
|
|
18455
|
+
tracker,
|
|
18456
|
+
bounce,
|
|
18457
|
+
tracker.contract,
|
|
18458
|
+
seat,
|
|
18459
|
+
remit,
|
|
18460
|
+
dryRun,
|
|
18461
|
+
{ count: priorBuildFailures(cu.ticket, prior), operatorId: c.resolved?.operator }
|
|
18462
|
+
);
|
|
18309
18463
|
if (r.kind === "failed") {
|
|
18310
18464
|
remit.warn(`could not hand back ${cu.ticket.issue_id}: ${r.why}`, { step: "release" });
|
|
18311
18465
|
}
|
|
18466
|
+
if (r.kind === "escalated" && !dryRun) {
|
|
18467
|
+
const key = cu.ticket.issue_tag ?? cu.ticket.issue_id;
|
|
18468
|
+
await applyFailureAlert(tracker, {
|
|
18469
|
+
level: "fail",
|
|
18470
|
+
headline: `${scope}: release blocked \u2014 ${key} broke the ${cu.hook} gate twice`,
|
|
18471
|
+
detail: `${key} (\`${cu.branch}\`) was handed back after a failed ${cu.hook} gate, pushed and re-verified, and failed it again. Isolation found it both times (${cu.how}), so it is escalated to the operator instead of another lap.
|
|
18472
|
+
|
|
18473
|
+
\`\`\`
|
|
18474
|
+
${cu.tail.trimEnd()}
|
|
18475
|
+
\`\`\``
|
|
18476
|
+
}, scope, all, seat, remit, dryRun, await alertPlacement()).catch((e) => {
|
|
18477
|
+
remit.warn(`could not file/update a failure ticket: ${e.message}`, { step: "release" });
|
|
18478
|
+
});
|
|
18479
|
+
}
|
|
18312
18480
|
} catch (e) {
|
|
18313
18481
|
remit.warn(`could not hand back ${cu.ticket.issue_id}: ${e.message}`, { step: "release" });
|
|
18314
18482
|
}
|
|
@@ -19281,7 +19449,7 @@ ${changed2.map((c) => ` - ${c}
|
|
|
19281
19449
|
if (!existsSync34(dir)) throw new ReposError(`${dir} does not exist on disk`);
|
|
19282
19450
|
const files = { exists: (r) => existsSync34(resolvePath(dir, r)), list: (r) => {
|
|
19283
19451
|
try {
|
|
19284
|
-
return
|
|
19452
|
+
return readdirSync15(resolvePath(dir, r));
|
|
19285
19453
|
} catch {
|
|
19286
19454
|
return [];
|
|
19287
19455
|
}
|
|
@@ -19478,7 +19646,7 @@ ${changed2.map((c) => ` - ${c}
|
|
|
19478
19646
|
` : `${human}
|
|
19479
19647
|
`);
|
|
19480
19648
|
const plan2 = planRepoClone({ text, base, route: routeName, name: addName, resolved: resolvedRemotes, dir: value("dir"), isCheckout });
|
|
19481
|
-
if (existsSync34(plan2.target) && (existsSync34(`${plan2.target}/.git`) ||
|
|
19649
|
+
if (existsSync34(plan2.target) && (existsSync34(`${plan2.target}/.git`) || readdirSync15(plan2.target).length > 0)) {
|
|
19482
19650
|
throw new ReposError(
|
|
19483
19651
|
existsSync34(`${plan2.target}/.git`) ? `${plan2.target} is already a checkout \u2014 attach it with \`crew repos add ${routeName} ${plan2.name} ${plan2.target}\`` : `${plan2.target} exists and is not empty \u2014 pass --dir to clone somewhere else`
|
|
19484
19652
|
);
|
|
@@ -21030,7 +21198,7 @@ Signed in as ${result.identity.email} \u2014 minted an API key in workspace "${r
|
|
|
21030
21198
|
}
|
|
21031
21199
|
const projectId = route.resolved.areaId ?? resolveRepoProject(await tracker.projectRows(), value("project"));
|
|
21032
21200
|
const fresh = !existsSync34(join37(dir, ".git"));
|
|
21033
|
-
if (existsSync34(dir) && fresh &&
|
|
21201
|
+
if (existsSync34(dir) && fresh && readdirSync15(dir).length > 0) fail(`${dir} exists and is not a git checkout \u2014 move it aside`);
|
|
21034
21202
|
if (dryRun) {
|
|
21035
21203
|
say("dry-run", { dir, newCheckout: fresh, repoRow: row?.id ?? null }, `would ${fresh ? "create" : "reuse"} ${dir}${row ? "" : ", create the Repos row"}, and file one ${tracker.contract.statuses.approved} ticket`);
|
|
21036
21204
|
break;
|
|
@@ -21516,6 +21684,8 @@ routes: ${cfg.routes.map((c) => c.route).join(", ")}
|
|
|
21516
21684
|
process.stdout.write(` - release readiness: ${p2}
|
|
21517
21685
|
`);
|
|
21518
21686
|
}
|
|
21687
|
+
process.stdout.write(` release self-repair: ${selfRepairLine(config.release.isolateFailures?.maxCandidates)}
|
|
21688
|
+
`);
|
|
21519
21689
|
const dh = doctorBaseReport(r, config.branch.base, baseHealth);
|
|
21520
21690
|
process.stdout.write(dh.text);
|
|
21521
21691
|
if (dh.failed) process.exitCode = 1;
|
package/package.json
CHANGED