@tablation/crew 0.43.4 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js
CHANGED
|
@@ -1432,6 +1432,7 @@ var RELEASE_KEYS = /* @__PURE__ */ new Set([
|
|
|
1432
1432
|
"ship",
|
|
1433
1433
|
"ci",
|
|
1434
1434
|
"verify",
|
|
1435
|
+
"isolateFailures",
|
|
1435
1436
|
"versioning",
|
|
1436
1437
|
"versionFiles",
|
|
1437
1438
|
"changelog",
|
|
@@ -1441,6 +1442,14 @@ var RELEASE_KEYS = /* @__PURE__ */ new Set([
|
|
|
1441
1442
|
var VERSIONINGS = ["auto", "none"];
|
|
1442
1443
|
var CI_KEYS = /* @__PURE__ */ new Set(["provider", "ref"]);
|
|
1443
1444
|
var VERIFY_KEYS = /* @__PURE__ */ new Set(["match", "timeoutSeconds", "intervalSeconds"]);
|
|
1445
|
+
var ISOLATE_KEYS = /* @__PURE__ */ new Set(["maxCandidates"]);
|
|
1446
|
+
var DEFAULT_ISOLATE_MAX_CANDIDATES = 4;
|
|
1447
|
+
function parseIsolateMax(raw, where, fail) {
|
|
1448
|
+
if (raw === void 0 || raw === null) return void 0;
|
|
1449
|
+
const n = Number(raw);
|
|
1450
|
+
if (!Number.isInteger(n) || n < 0) throw fail(`${where}.maxCandidates must be a non-negative integer (got "${raw}")`);
|
|
1451
|
+
return n;
|
|
1452
|
+
}
|
|
1444
1453
|
var MATCHES = ["commit", "version"];
|
|
1445
1454
|
var MODES = ["local", "integrate", "ci_manual", "ci_auto", "external"];
|
|
1446
1455
|
var PROVIDERS = ["github", "buildkite", "other", "none"];
|
|
@@ -1474,6 +1483,7 @@ function parseRepoConfig(text, file) {
|
|
|
1474
1483
|
rejectUnknown(raw.release, RELEASE_KEYS, "release", file);
|
|
1475
1484
|
rejectUnknown(raw.release?.ci, CI_KEYS, "release.ci", file);
|
|
1476
1485
|
rejectUnknown(raw.release?.verify, VERIFY_KEYS, "release.verify", file);
|
|
1486
|
+
rejectUnknown(raw.release?.isolateFailures, ISOLATE_KEYS, "release.isolateFailures", file);
|
|
1477
1487
|
rejectUnknown(raw.branch, BRANCH_KEYS, "branch", file);
|
|
1478
1488
|
rejectUnknown(raw.worktrees, /* @__PURE__ */ new Set(["copy", "prefix", "name"]), "worktrees", file);
|
|
1479
1489
|
rejectUnknown(raw.docs, /* @__PURE__ */ new Set(["triagePolicy", "designGuide"]), "docs", file);
|
|
@@ -1611,6 +1621,11 @@ function parseRepoConfig(text, file) {
|
|
|
1611
1621
|
`${file}: release.verify.intervalSeconds (${verify.intervalSeconds}) exceeds timeoutSeconds (${verify.timeoutSeconds}) \u2014 it would never be asked twice`
|
|
1612
1622
|
);
|
|
1613
1623
|
}
|
|
1624
|
+
const isolateMax = parseIsolateMax(
|
|
1625
|
+
raw.release?.isolateFailures?.maxCandidates,
|
|
1626
|
+
"release.isolateFailures",
|
|
1627
|
+
(m) => new RepoConfigError(`${file}: ${m}`)
|
|
1628
|
+
);
|
|
1614
1629
|
const versioning = String(raw.release?.versioning ?? "auto");
|
|
1615
1630
|
if (!VERSIONINGS.includes(versioning)) {
|
|
1616
1631
|
throw new RepoConfigError(`${file}: release.versioning must be auto|none (got "${versioning}")`);
|
|
@@ -1664,6 +1679,7 @@ function parseRepoConfig(text, file) {
|
|
|
1664
1679
|
ship,
|
|
1665
1680
|
ci: { provider, ref: raw.release?.ci?.ref ? String(raw.release.ci.ref) : void 0 },
|
|
1666
1681
|
verify,
|
|
1682
|
+
isolateFailures: isolateMax === void 0 ? void 0 : { maxCandidates: isolateMax },
|
|
1667
1683
|
versioning,
|
|
1668
1684
|
versionFiles,
|
|
1669
1685
|
tag,
|
|
@@ -1848,6 +1864,14 @@ function resolveRepoConfig(repo, ship, dir) {
|
|
|
1848
1864
|
DEFAULTS.intervalSeconds
|
|
1849
1865
|
)
|
|
1850
1866
|
},
|
|
1867
|
+
isolateFailures: {
|
|
1868
|
+
maxCandidates: pick(
|
|
1869
|
+
"release.isolateFailures.maxCandidates",
|
|
1870
|
+
repo?.release.isolateFailures?.maxCandidates,
|
|
1871
|
+
ship?.release?.isolateFailures?.maxCandidates,
|
|
1872
|
+
DEFAULT_ISOLATE_MAX_CANDIDATES
|
|
1873
|
+
)
|
|
1874
|
+
},
|
|
1851
1875
|
versioning: pick(
|
|
1852
1876
|
"release.versioning",
|
|
1853
1877
|
repo?.release.versioning === "auto" ? void 0 : repo?.release.versioning,
|
|
@@ -1999,6 +2023,7 @@ var RELEASE_OVERRIDE_KEYS = /* @__PURE__ */ new Set([
|
|
|
1999
2023
|
"mode",
|
|
2000
2024
|
"ci",
|
|
2001
2025
|
"verify",
|
|
2026
|
+
"isolateFailures",
|
|
2002
2027
|
"versioning",
|
|
2003
2028
|
"versionFiles",
|
|
2004
2029
|
"versionFile",
|
|
@@ -2008,6 +2033,7 @@ var RELEASE_OVERRIDE_KEYS = /* @__PURE__ */ new Set([
|
|
|
2008
2033
|
]);
|
|
2009
2034
|
var CI_OVERRIDE_KEYS = /* @__PURE__ */ new Set(["provider", "ref"]);
|
|
2010
2035
|
var VERIFY_OVERRIDE_KEYS = /* @__PURE__ */ new Set(["match", "timeoutSeconds", "intervalSeconds"]);
|
|
2036
|
+
var ISOLATE_OVERRIDE_KEYS = /* @__PURE__ */ new Set(["maxCandidates"]);
|
|
2011
2037
|
function rejectUnknownKeys(obj, allowed, where, file) {
|
|
2012
2038
|
if (!obj || typeof obj !== "object") return;
|
|
2013
2039
|
const unknown = Object.keys(obj).filter((k) => !allowed.has(k));
|
|
@@ -2022,6 +2048,7 @@ function parseReleaseOverride(raw, where, file) {
|
|
|
2022
2048
|
rejectUnknownKeys(raw, RELEASE_OVERRIDE_KEYS, `${where}.release`, file);
|
|
2023
2049
|
if (raw.ci !== void 0) rejectUnknownKeys(raw.ci, CI_OVERRIDE_KEYS, `${where}.release.ci`, file);
|
|
2024
2050
|
if (raw.verify !== void 0) rejectUnknownKeys(raw.verify, VERIFY_OVERRIDE_KEYS, `${where}.release.verify`, file);
|
|
2051
|
+
if (raw.isolateFailures !== void 0) rejectUnknownKeys(raw.isolateFailures, ISOLATE_OVERRIDE_KEYS, `${where}.release.isolateFailures`, file);
|
|
2025
2052
|
const mode = raw.mode === void 0 ? void 0 : String(raw.mode);
|
|
2026
2053
|
if (mode !== void 0 && !MODES.includes(mode)) {
|
|
2027
2054
|
throw new ConfigError(`${file}: ${where}.release.mode must be one of ${MODES.join("|")} (got "${raw.mode}")`);
|
|
@@ -2050,11 +2077,17 @@ function parseReleaseOverride(raw, where, file) {
|
|
|
2050
2077
|
};
|
|
2051
2078
|
const timeoutSeconds = num(raw.verify?.timeoutSeconds, "timeoutSeconds");
|
|
2052
2079
|
const intervalSeconds = num(raw.verify?.intervalSeconds, "intervalSeconds");
|
|
2080
|
+
const isolateMax = parseIsolateMax(
|
|
2081
|
+
raw.isolateFailures?.maxCandidates,
|
|
2082
|
+
`${where}.release.isolateFailures`,
|
|
2083
|
+
(m) => new ConfigError(`${file}: ${m}`)
|
|
2084
|
+
);
|
|
2053
2085
|
const versionFiles = Array.isArray(raw.versionFiles) ? raw.versionFiles : raw.versionFile ? [String(raw.versionFile)] : void 0;
|
|
2054
2086
|
return {
|
|
2055
2087
|
mode,
|
|
2056
2088
|
ci: provider !== void 0 || raw.ci?.ref !== void 0 ? { provider, ref: raw.ci?.ref !== void 0 ? String(raw.ci.ref) : void 0 } : void 0,
|
|
2057
2089
|
verify: match !== void 0 || timeoutSeconds !== void 0 || intervalSeconds !== void 0 ? { match, timeoutSeconds, intervalSeconds } : void 0,
|
|
2090
|
+
...isolateMax === void 0 ? {} : { isolateFailures: { maxCandidates: isolateMax } },
|
|
2058
2091
|
versioning,
|
|
2059
2092
|
versionFiles,
|
|
2060
2093
|
changelog: raw.changelog !== void 0 ? String(raw.changelog) : void 0,
|
|
@@ -2078,10 +2111,12 @@ function mergeRouteRelease(route, override) {
|
|
|
2078
2111
|
const match = override?.verify?.match ?? route.verify?.match;
|
|
2079
2112
|
const timeoutSeconds = override?.verify?.timeoutSeconds ?? route.verify?.timeoutSeconds;
|
|
2080
2113
|
const intervalSeconds = override?.verify?.intervalSeconds ?? route.verify?.intervalSeconds;
|
|
2114
|
+
const isolateMax = override?.isolateFailures?.maxCandidates ?? route.isolateFailures?.maxCandidates;
|
|
2081
2115
|
return {
|
|
2082
2116
|
mode: override?.mode ?? route.mode,
|
|
2083
2117
|
ci: provider !== void 0 || ref !== void 0 ? { provider, ref } : void 0,
|
|
2084
2118
|
verify: match !== void 0 || timeoutSeconds !== void 0 || intervalSeconds !== void 0 ? { match, timeoutSeconds, intervalSeconds } : void 0,
|
|
2119
|
+
...isolateMax === void 0 ? {} : { isolateFailures: { maxCandidates: isolateMax } },
|
|
2085
2120
|
versioning: override?.versioning ?? route.versioning,
|
|
2086
2121
|
versionFiles: override?.versionFiles ?? route.versionFiles,
|
|
2087
2122
|
changelog: override?.changelog ?? route.changelog,
|
|
@@ -3199,8 +3234,6 @@ var DEFAULT_CONTRACT = {
|
|
|
3199
3234
|
parent: "ticket_id",
|
|
3200
3235
|
body: "body",
|
|
3201
3236
|
author: "team_member_id",
|
|
3202
|
-
kind: "kind",
|
|
3203
|
-
eventKind: "event",
|
|
3204
3237
|
createdAt: "created_at",
|
|
3205
3238
|
ship: "ship_id"
|
|
3206
3239
|
},
|
|
@@ -3310,6 +3343,7 @@ function validateContract(c) {
|
|
|
3310
3343
|
}
|
|
3311
3344
|
return problems;
|
|
3312
3345
|
}
|
|
3346
|
+
var EVENT_LOG_MISSING = "the Issues table has no event log \u2014 crew writes its audit trail there (CREW-1574). In the app, open the Issues table's Data & lifecycle tab and turn on Event log, then rerun.";
|
|
3313
3347
|
|
|
3314
3348
|
// src/shell.ts
|
|
3315
3349
|
import { spawn } from "child_process";
|
|
@@ -5174,18 +5208,65 @@ var Tracker = class {
|
|
|
5174
5208
|
await this.client.records.update(this.models.crew, memberId, patch2);
|
|
5175
5209
|
}
|
|
5176
5210
|
/**
|
|
5177
|
-
*
|
|
5178
|
-
*
|
|
5179
|
-
*
|
|
5211
|
+
* One entry in the runner's audit trail, written to the Issues table's
|
|
5212
|
+
* event log (CREW-1574) rather than to Comments, so it can never be mistaken
|
|
5213
|
+
* for something a seat or person said. `type` is a short snake_case name
|
|
5214
|
+
* (`pushed`, `claimed`, `bounced`, …); `body` is the human message and keeps
|
|
5215
|
+
* whatever wording the caller already used. Attributed to the seat's Crew
|
|
5216
|
+
* row, with the posting ship in `metadata`.
|
|
5180
5217
|
*/
|
|
5181
|
-
async postEvent(ticketId, body, memberId) {
|
|
5218
|
+
async postEvent(ticketId, body, memberId, type = "note") {
|
|
5182
5219
|
const auth = await this.authorshipProbe();
|
|
5183
|
-
await this.
|
|
5184
|
-
|
|
5185
|
-
body,
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5220
|
+
const res = await this.eventsFetch("POST", `/records/${ticketId}/events`, {
|
|
5221
|
+
type,
|
|
5222
|
+
message: body,
|
|
5223
|
+
actorRecordId: memberId,
|
|
5224
|
+
...auth.shipId ? { metadata: { ship_id: auth.shipId } } : {}
|
|
5225
|
+
});
|
|
5226
|
+
if (res.status === 404) throw new ConfigError(EVENT_LOG_MISSING);
|
|
5227
|
+
if (!res.ok) throw new Error(`posting a ${type} event failed: HTTP ${res.status}`);
|
|
5228
|
+
}
|
|
5229
|
+
/**
|
|
5230
|
+
* Whether the Issues table has an event log: `GET /data-models/<issues>/events`
|
|
5231
|
+
* answers 200 when it does and 404 when the setting is off. Crew requires it
|
|
5232
|
+
* (CREW-1574); `connect` and doctor report `false` by name.
|
|
5233
|
+
*/
|
|
5234
|
+
async hasEventLog() {
|
|
5235
|
+
const res = await this.eventsFetch("GET", "/events?limit=1");
|
|
5236
|
+
if (res.status === 404) return false;
|
|
5237
|
+
if (!res.ok) throw new Error(`reading the event log failed: HTTP ${res.status}`);
|
|
5238
|
+
return true;
|
|
5239
|
+
}
|
|
5240
|
+
/**
|
|
5241
|
+
* Events of `type` across the Issues table, newest first. The runner's own
|
|
5242
|
+
* dedupe/escalation markers (`stalled`, `gate_red`, `conflict_handed_back`,
|
|
5243
|
+
* `stranded_flagged`) live here now, so the readers that used to scan
|
|
5244
|
+
* Comments for them read this instead. A tracker without the log answers 404
|
|
5245
|
+
* and reads as no events: the writes fail by name (`postEvent`), and a
|
|
5246
|
+
* poll must not die on a read.
|
|
5247
|
+
*/
|
|
5248
|
+
async events(type, limit = 500) {
|
|
5249
|
+
const filters = encodeURIComponent(JSON.stringify([{ columnName: "event_type", operator: "EQ", value: type }]));
|
|
5250
|
+
const res = await this.eventsFetch("GET", `/events?limit=${limit}&filters=${filters}`);
|
|
5251
|
+
if (res.status === 404) return [];
|
|
5252
|
+
if (!res.ok) throw new Error(`reading ${type} events failed: HTTP ${res.status}`);
|
|
5253
|
+
const rows = await res.json();
|
|
5254
|
+
return rows.filter((e) => e.event_type === type).sort((a, b) => Date.parse(b.occurred_at ?? b.created_at ?? "") - Date.parse(a.occurred_at ?? a.created_at ?? ""));
|
|
5255
|
+
}
|
|
5256
|
+
/** The newest event of `type` on a ticket, or undefined — e.g. the `pushed` tip QA verifies against. */
|
|
5257
|
+
async latestEvent(ticketId, type) {
|
|
5258
|
+
return (await this.events(type)).find((e) => e.record_id === ticketId);
|
|
5259
|
+
}
|
|
5260
|
+
/** The event-log routes hang off the Issues data model; @tablation/client has no resource for them yet. */
|
|
5261
|
+
eventsFetch(method, path, body) {
|
|
5262
|
+
return fetch(`${this.route.baseUrl}/api/data-models/${this.models.issues}${path}`, {
|
|
5263
|
+
method,
|
|
5264
|
+
headers: {
|
|
5265
|
+
Authorization: `Bearer ${resolveApiKey(this.route)}`,
|
|
5266
|
+
"User-Agent": this.userAgent,
|
|
5267
|
+
...body === void 0 ? {} : { "Content-Type": "application/json" }
|
|
5268
|
+
},
|
|
5269
|
+
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
5189
5270
|
});
|
|
5190
5271
|
}
|
|
5191
5272
|
/**
|
|
@@ -5460,7 +5541,7 @@ async function applySweep(writer, steps, memberId, log) {
|
|
|
5460
5541
|
{ ticket: step.ticket.issue_id, step: "sweep" }
|
|
5461
5542
|
);
|
|
5462
5543
|
try {
|
|
5463
|
-
await writer.postEvent(step.ticket.id, sweepComment(step), memberId);
|
|
5544
|
+
await writer.postEvent(step.ticket.id, sweepComment(step), memberId, "blocked_sweep");
|
|
5464
5545
|
} catch {
|
|
5465
5546
|
log.warn("status changed but the note failed to post", { ticket: step.ticket.issue_id, step: "sweep" });
|
|
5466
5547
|
}
|
|
@@ -5573,7 +5654,8 @@ async function noteReaccepted(tracker, result, seat, shipLabel) {
|
|
|
5573
5654
|
await tracker.postEvent(
|
|
5574
5655
|
result.ticket.id,
|
|
5575
5656
|
`Re-accepted while held by ${shipLabel}; resuming there, not re-claiming. Setting a held ticket back to \`accepted\` is not needed \u2014 to move it to another ship, clear \`held_by_ship_id\`.`,
|
|
5576
|
-
seat
|
|
5657
|
+
seat,
|
|
5658
|
+
"reaccepted"
|
|
5577
5659
|
);
|
|
5578
5660
|
} catch {
|
|
5579
5661
|
}
|
|
@@ -5606,7 +5688,8 @@ async function noteTakeover(tracker, result, seat, ships, myShipName, continuing
|
|
|
5606
5688
|
await tracker.postEvent(
|
|
5607
5689
|
result.ticket.id,
|
|
5608
5690
|
`Taken over from ${shipName(ships, result.tookOverFrom)} (not seen recently) by ${myShipName}; ` + (sha ? `continuing from \`${sha}\` on origin.` : "no branch commit was found on origin to continue from."),
|
|
5609
|
-
seat
|
|
5691
|
+
seat,
|
|
5692
|
+
"taken_over"
|
|
5610
5693
|
);
|
|
5611
5694
|
} catch {
|
|
5612
5695
|
}
|
|
@@ -5676,7 +5759,7 @@ function buildingRoleHasWork(role, i) {
|
|
|
5676
5759
|
);
|
|
5677
5760
|
const watchedIds = new Set(watched.map((t) => t.id));
|
|
5678
5761
|
const newComments = i.comments.filter(
|
|
5679
|
-
(c) => c.created_at > i.watermark && c.team_member_id !== me &&
|
|
5762
|
+
(c) => c.created_at > i.watermark && c.team_member_id !== me && watchedIds.has(c.ticket_id)
|
|
5680
5763
|
).length;
|
|
5681
5764
|
const unassigned = mine.filter(
|
|
5682
5765
|
(t) => t.status === "in_progress" && !t.assignee_id && !i.blocked.has(t.id)
|
|
@@ -5933,14 +6016,14 @@ function filedBy(t, i) {
|
|
|
5933
6016
|
return t.reporter_name ? `${t.reporter_name} (unverified)` : "\u2014";
|
|
5934
6017
|
}
|
|
5935
6018
|
function lastComment(i, ticketId) {
|
|
5936
|
-
const forTicket = i.comments.filter((c) => c.ticket_id === ticketId
|
|
6019
|
+
const forTicket = i.comments.filter((c) => c.ticket_id === ticketId).sort((a, b) => a.created_at < b.created_at ? -1 : a.created_at > b.created_at ? 1 : 0);
|
|
5937
6020
|
const last = forTicket[forTicket.length - 1];
|
|
5938
6021
|
if (!last) return "\u2014";
|
|
5939
6022
|
return `${last.created_at.slice(0, 16)}Z ${author(last, i)}`;
|
|
5940
6023
|
}
|
|
5941
6024
|
function newFromOthers(i, ticketId) {
|
|
5942
6025
|
return i.comments.filter(
|
|
5943
|
-
(c) => c.ticket_id === ticketId && c.team_member_id !== i.me && c.created_at > i.watermark
|
|
6026
|
+
(c) => c.ticket_id === ticketId && c.team_member_id !== i.me && c.created_at > i.watermark
|
|
5944
6027
|
).length;
|
|
5945
6028
|
}
|
|
5946
6029
|
function who(t, i) {
|
|
@@ -6142,7 +6225,7 @@ var ms = (iso) => {
|
|
|
6142
6225
|
const n = iso ? Date.parse(iso) : NaN;
|
|
6143
6226
|
return Number.isFinite(n) ? n : 0;
|
|
6144
6227
|
};
|
|
6145
|
-
function planStalled(tickets, comments, ships, contract, holds2 = /* @__PURE__ */ new Set(), now = Date.now(), wm) {
|
|
6228
|
+
function planStalled(tickets, comments, ships, contract, holds2 = /* @__PURE__ */ new Set(), now = Date.now(), wm, events = []) {
|
|
6146
6229
|
const watched = /* @__PURE__ */ new Set([contract.statuses.building, contract.statuses.handoff, contract.statuses.verifying]);
|
|
6147
6230
|
const byTicket = /* @__PURE__ */ new Map();
|
|
6148
6231
|
for (const c of comments) {
|
|
@@ -6155,7 +6238,7 @@ function planStalled(tickets, comments, ships, contract, holds2 = /* @__PURE__ *
|
|
|
6155
6238
|
if (!watched.has(t.status)) continue;
|
|
6156
6239
|
if (t.assignee_id && holds2.has(t.assignee_id)) continue;
|
|
6157
6240
|
const mine = byTicket.get(t.id) ?? [];
|
|
6158
|
-
const lastMarker = Math.max(0, ...
|
|
6241
|
+
const lastMarker = Math.max(0, ...events.filter((e) => e.record_id === t.id && e.event_type === "stalled").map((e) => ms(e.occurred_at ?? e.created_at ?? "")));
|
|
6159
6242
|
const lastActivityAt = Math.max(
|
|
6160
6243
|
ms(t.updated_at),
|
|
6161
6244
|
...mine.filter((c) => !(c.body ?? "").includes(STALLED_MARKER)).map((c) => ms(c.created_at))
|
|
@@ -6197,7 +6280,7 @@ async function applyStalled(writer, stalled, ships, memberId, myShipId, log, dry
|
|
|
6197
6280
|
continue;
|
|
6198
6281
|
}
|
|
6199
6282
|
try {
|
|
6200
|
-
await writer.postEvent(s.ticket.id, stalledComment(s, ships, now), memberId);
|
|
6283
|
+
await writer.postEvent(s.ticket.id, stalledComment(s, ships, now), memberId, "stalled");
|
|
6201
6284
|
r.announced++;
|
|
6202
6285
|
log.warn(`stalled at ${s.ticket.status} \u2014 ${s.holderShipId ? "flagged on the holder" : "commented, no holder"}`, x);
|
|
6203
6286
|
} catch (e) {
|
|
@@ -6231,13 +6314,14 @@ async function decideCycle(o) {
|
|
|
6231
6314
|
const { route, ship, state, emit } = o;
|
|
6232
6315
|
const tracker = new Tracker(route, ship);
|
|
6233
6316
|
emit.enter("poll");
|
|
6234
|
-
const [tickets, comments, crewRows, epicRows, ships, projectModes] = await Promise.all([
|
|
6317
|
+
const [tickets, comments, crewRows, epicRows, ships, projectModes, stalledEvents] = await Promise.all([
|
|
6235
6318
|
tracker.openTickets(),
|
|
6236
6319
|
tracker.comments(200),
|
|
6237
6320
|
tracker.crewRows(),
|
|
6238
6321
|
tracker.epicRows(),
|
|
6239
6322
|
tracker.shipRows(),
|
|
6240
|
-
tracker.projectWorkModes()
|
|
6323
|
+
tracker.projectWorkModes(),
|
|
6324
|
+
tracker.events("stalled")
|
|
6241
6325
|
]);
|
|
6242
6326
|
emit.emit(`${tickets.length} open ticket(s), ${comments.length} comment(s)`, {
|
|
6243
6327
|
data: { tickets: tickets.length, comments: comments.length }
|
|
@@ -6281,7 +6365,7 @@ async function decideCycle(o) {
|
|
|
6281
6365
|
}
|
|
6282
6366
|
const sweep = planSweep(tickets, info, blocked, workModes);
|
|
6283
6367
|
const holdReleases = planHoldReleases(tickets, tracker.contract);
|
|
6284
|
-
const stalled = planStalled(tickets, comments, ships, tracker.contract, holds2, Date.now(), workModes);
|
|
6368
|
+
const stalled = planStalled(tickets, comments, ships, tracker.contract, holds2, Date.now(), workModes, stalledEvents);
|
|
6285
6369
|
const skipped = skippedByMode(tickets, workModes);
|
|
6286
6370
|
if (skipped.manual || skipped.hybrid) {
|
|
6287
6371
|
emit.emit(`sweeps skipped ${skipped.manual} manual/project-less and ${skipped.hybrid} unassigned hybrid ticket(s)`, {
|
|
@@ -6567,15 +6651,15 @@ function worktreeForBranch(cwd, branch) {
|
|
|
6567
6651
|
}
|
|
6568
6652
|
return null;
|
|
6569
6653
|
}
|
|
6570
|
-
var priorBounces = (ticket,
|
|
6571
|
-
function planConflictBounce(cwd, ticket, branch, base, paths,
|
|
6654
|
+
var priorBounces = (ticket, events) => events.filter((e) => e.record_id === ticket.id && e.event_type === "conflict_handed_back").length;
|
|
6655
|
+
function planConflictBounce(cwd, ticket, branch, base, paths, events) {
|
|
6572
6656
|
return {
|
|
6573
6657
|
ticket,
|
|
6574
6658
|
branch,
|
|
6575
6659
|
base,
|
|
6576
6660
|
paths,
|
|
6577
6661
|
worktree: worktreeForBranch(cwd, branch),
|
|
6578
|
-
priorBounces: priorBounces(ticket,
|
|
6662
|
+
priorBounces: priorBounces(ticket, events)
|
|
6579
6663
|
};
|
|
6580
6664
|
}
|
|
6581
6665
|
function refreshBranch(worktree, base) {
|
|
@@ -6621,7 +6705,7 @@ async function applyConflictBounce(writer, b, contract, memberId, log, dryRun) {
|
|
|
6621
6705
|
}
|
|
6622
6706
|
try {
|
|
6623
6707
|
await writer.updateTicket(b.ticket.id, { status: contract.statuses.needsHuman }, b.ticket.updated_at);
|
|
6624
|
-
await writer.postEvent(b.ticket.id, escalateComment(b), memberId).catch(() => {
|
|
6708
|
+
await writer.postEvent(b.ticket.id, escalateComment(b), memberId, "conflict_escalated").catch(() => {
|
|
6625
6709
|
log.warn("escalated, but the note failed to post", t);
|
|
6626
6710
|
});
|
|
6627
6711
|
log.warn(`conflicts again after a hand-back \u2014 escalated to ${contract.statuses.needsHuman}`, t);
|
|
@@ -6658,7 +6742,7 @@ async function applyConflictBounce(writer, b, contract, memberId, log, dryRun) {
|
|
|
6658
6742
|
{ status: contract.statuses.building, assignee_id: null },
|
|
6659
6743
|
b.ticket.updated_at
|
|
6660
6744
|
);
|
|
6661
|
-
await writer.postEvent(b.ticket.id, handBackComment(b, merged), memberId).catch(() => {
|
|
6745
|
+
await writer.postEvent(b.ticket.id, handBackComment(b, merged), memberId, "conflict_handed_back").catch(() => {
|
|
6662
6746
|
log.warn("handed back, but the note failed to post", t);
|
|
6663
6747
|
});
|
|
6664
6748
|
log.emit(
|
|
@@ -6675,17 +6759,93 @@ async function applyConflictBounce(writer, b, contract, memberId, log, dryRun) {
|
|
|
6675
6759
|
}
|
|
6676
6760
|
}
|
|
6677
6761
|
|
|
6762
|
+
// src/build-failure.ts
|
|
6763
|
+
import { StaleWriteError as StaleWriteError6 } from "@tablation/client";
|
|
6764
|
+
var BUILD_FAILURE_MARKER = "<!-- crew:build-failure -->";
|
|
6765
|
+
function failingPaths(output, cwd) {
|
|
6766
|
+
const found = /* @__PURE__ */ new Set();
|
|
6767
|
+
const ext = String.raw`[\w.-]+\.[A-Za-z]{1,6}`;
|
|
6768
|
+
const dir = String.raw`(?:[\w@.~-]+/)*`;
|
|
6769
|
+
const patterns = [
|
|
6770
|
+
new RegExp(String.raw`^\s*(${dir}${ext})\(\d+,\d+\)\s*:`, "gm"),
|
|
6771
|
+
new RegExp(String.raw`(?:^|\s)(/?${dir}${ext}):\d+(?::\d+)?`, "gm"),
|
|
6772
|
+
new RegExp(String.raw`(?:FAIL|✖|✕|not ok \d+ -)\s+(/?${dir}${ext})`, "gm")
|
|
6773
|
+
];
|
|
6774
|
+
const root = cwd.endsWith("/") ? cwd : `${cwd}/`;
|
|
6775
|
+
for (const re of patterns) {
|
|
6776
|
+
for (const m of output.matchAll(re)) {
|
|
6777
|
+
let p = m[1];
|
|
6778
|
+
if (p.startsWith("/")) {
|
|
6779
|
+
if (!p.startsWith(root)) continue;
|
|
6780
|
+
p = p.slice(root.length);
|
|
6781
|
+
}
|
|
6782
|
+
p = p.replace(/^\.\//, "");
|
|
6783
|
+
if (p.includes("node_modules/")) continue;
|
|
6784
|
+
found.add(p);
|
|
6785
|
+
}
|
|
6786
|
+
}
|
|
6787
|
+
return [...found];
|
|
6788
|
+
}
|
|
6789
|
+
function culpritByFiles(candidates, paths) {
|
|
6790
|
+
if (paths.length === 0) return null;
|
|
6791
|
+
const hits = candidates.filter((c) => paths.some((p) => c.changed.has(p)));
|
|
6792
|
+
return hits.length === 1 ? hits[0] : null;
|
|
6793
|
+
}
|
|
6794
|
+
function planBuildFailureBounce(cwd, c, base, remote) {
|
|
6795
|
+
return { ...c, base, remote, worktree: worktreeForBranch(cwd, c.branch) };
|
|
6796
|
+
}
|
|
6797
|
+
function buildFailureComment(b) {
|
|
6798
|
+
const how = b.how === "files" ? `The failing output names files that only \`${b.branch}\` changed among the branches in that release.` : `Built alone on the previous \`${b.base}\`, \`${b.branch}\` fails the same hook.`;
|
|
6799
|
+
return `${BUILD_FAILURE_MARKER}
|
|
6800
|
+
**Back with the dev lane \u2014 this branch builds on its own but not on current \`${b.base}\`.**
|
|
6801
|
+
|
|
6802
|
+
\`${b.branch}\` was verified, but the release's \`${b.hook}\` hook failed once it was merged with the other branches in the same release. ${how}
|
|
6803
|
+
|
|
6804
|
+
First step: \`git merge ${b.remote}/${b.base}\` in ${b.worktree ? `\`${b.worktree}\`` : "the branch's worktree"}, rebuild, and fix what disagrees \u2014 the merge is clean at the text level, so this is a semantic conflict (a duplicate identifier, a changed signature) with work that landed on \`${b.base}\` after this branch was cut. Push as a plain fast-forward and hand back to QA as usual.
|
|
6805
|
+
|
|
6806
|
+
\`\`\`
|
|
6807
|
+
${b.tail.trimEnd()}
|
|
6808
|
+
\`\`\`
|
|
6809
|
+
|
|
6810
|
+
Nothing was written to \`${b.base}\` for this ticket; the rest of the release shipped without it.`;
|
|
6811
|
+
}
|
|
6812
|
+
async function applyBuildFailureBounce(writer, b, contract, memberId, log, dryRun) {
|
|
6813
|
+
const t = { ticket: b.ticket.issue_id, step: "release" };
|
|
6814
|
+
if (dryRun) {
|
|
6815
|
+
log.emit(`would hand ${b.ticket.issue_id} back to ${contract.statuses.building} \u2014 its branch breaks the ${b.hook} hook`, t);
|
|
6816
|
+
return { kind: "handed-back" };
|
|
6817
|
+
}
|
|
6818
|
+
try {
|
|
6819
|
+
await writer.updateTicket(
|
|
6820
|
+
b.ticket.id,
|
|
6821
|
+
{ status: contract.statuses.building, assignee_id: null },
|
|
6822
|
+
b.ticket.updated_at
|
|
6823
|
+
);
|
|
6824
|
+
await writer.postEvent(b.ticket.id, buildFailureComment(b), memberId, "build_failed").catch(() => {
|
|
6825
|
+
log.warn("handed back, but the note failed to post", t);
|
|
6826
|
+
});
|
|
6827
|
+
log.emit(`handed back to ${contract.statuses.building} \u2014 breaks the ${b.hook} hook once merged with the rest`, t);
|
|
6828
|
+
return { kind: "handed-back" };
|
|
6829
|
+
} catch (e) {
|
|
6830
|
+
if (e instanceof StaleWriteError6) {
|
|
6831
|
+
log.emit(`${b.ticket.issue_id} changed on the board since this run read it \u2014 not handing it back over it`, t);
|
|
6832
|
+
return { kind: "stale" };
|
|
6833
|
+
}
|
|
6834
|
+
return { kind: "failed", why: e.message };
|
|
6835
|
+
}
|
|
6836
|
+
}
|
|
6837
|
+
|
|
6678
6838
|
// src/stranded-verified.ts
|
|
6679
6839
|
var STRANDED_MARKER = "<!-- crew:stranded-verified -->";
|
|
6680
6840
|
var STRANDED_ESCALATE_AFTER_MS = 30 * 60 * 1e3;
|
|
6681
|
-
var priorStrandedFlags = (ticket,
|
|
6682
|
-
function planStrandedVerified(candidate, base, remote,
|
|
6683
|
-
const stamps =
|
|
6841
|
+
var priorStrandedFlags = (ticket, events) => events.filter((e) => e.record_id === ticket.id && e.event_type === "stranded_flagged").length;
|
|
6842
|
+
function planStrandedVerified(candidate, base, remote, events) {
|
|
6843
|
+
const stamps = events.filter((e) => e.record_id === candidate.ticket.id && e.event_type === "stranded_flagged").map((e) => Date.parse(e.occurred_at ?? e.created_at ?? "")).filter((n) => !Number.isNaN(n));
|
|
6684
6844
|
return {
|
|
6685
6845
|
candidate,
|
|
6686
6846
|
base,
|
|
6687
6847
|
remote,
|
|
6688
|
-
priorFlags: priorStrandedFlags(candidate.ticket,
|
|
6848
|
+
priorFlags: priorStrandedFlags(candidate.ticket, events),
|
|
6689
6849
|
lastFlaggedAt: stamps.length ? Math.max(...stamps) : void 0
|
|
6690
6850
|
};
|
|
6691
6851
|
}
|
|
@@ -6723,7 +6883,7 @@ async function applyStrandedVerified(writer, s, contract, operatorId, memberId,
|
|
|
6723
6883
|
status: contract.statuses.needsHuman,
|
|
6724
6884
|
...operatorId ? { assignee_id: operatorId } : {}
|
|
6725
6885
|
});
|
|
6726
|
-
await writer.postEvent(s.candidate.ticket.id, escalateComment2(s), memberId).catch(() => {
|
|
6886
|
+
await writer.postEvent(s.candidate.ticket.id, escalateComment2(s), memberId, "stranded_escalated").catch(() => {
|
|
6727
6887
|
log.warn("escalated, but the note failed to post", t);
|
|
6728
6888
|
});
|
|
6729
6889
|
log.warn(`still nothing to merge after a flag \u2014 escalated to ${contract.statuses.needsHuman}`, t);
|
|
@@ -6737,7 +6897,7 @@ async function applyStrandedVerified(writer, s, contract, operatorId, memberId,
|
|
|
6737
6897
|
return { kind: "flagged" };
|
|
6738
6898
|
}
|
|
6739
6899
|
try {
|
|
6740
|
-
await writer.postEvent(s.candidate.ticket.id, flagComment(s), memberId).catch(() => {
|
|
6900
|
+
await writer.postEvent(s.candidate.ticket.id, flagComment(s), memberId, "stranded_flagged").catch(() => {
|
|
6741
6901
|
log.warn("flagged, but the note failed to post", t);
|
|
6742
6902
|
});
|
|
6743
6903
|
log.emit("flagged as stranded at verified \u2014 will escalate if still unplaceable next cycle", t);
|
|
@@ -6772,7 +6932,7 @@ ${g.tail}
|
|
|
6772
6932
|
\`\`\`
|
|
6773
6933
|
`;
|
|
6774
6934
|
}
|
|
6775
|
-
var alreadyReported = (ticket, g,
|
|
6935
|
+
var alreadyReported = (ticket, g, events) => events.some((e) => e.record_id === ticket.id && e.event_type === "gate_red" && (e.message ?? "").includes(`${GATE_RED_MARKER} ${g.sha.slice(0, 8)}`));
|
|
6776
6936
|
|
|
6777
6937
|
// src/agent.ts
|
|
6778
6938
|
import { spawn as spawn2 } from "child_process";
|
|
@@ -7196,12 +7356,12 @@ function expandSkillMarkers(text, lookup, opts = {}) {
|
|
|
7196
7356
|
import { readFileSync as readFileSync10, readdirSync as readdirSync3, existsSync as existsSync11 } from "fs";
|
|
7197
7357
|
import { join as join12 } from "path";
|
|
7198
7358
|
import { parse as parseYaml } from "yaml";
|
|
7199
|
-
import { TablationClient as TablationClient3, StaleWriteError as
|
|
7359
|
+
import { TablationClient as TablationClient3, StaleWriteError as StaleWriteError8, TablationApiError as TablationApiError2 } from "@tablation/client";
|
|
7200
7360
|
|
|
7201
7361
|
// src/agents.ts
|
|
7202
7362
|
import { readFileSync as readFileSync9, existsSync as existsSync10 } from "fs";
|
|
7203
7363
|
import { join as join11 } from "path";
|
|
7204
|
-
import { TablationClient as TablationClient2, StaleWriteError as
|
|
7364
|
+
import { TablationClient as TablationClient2, StaleWriteError as StaleWriteError7, TablationApiError } from "@tablation/client";
|
|
7205
7365
|
var PERSONA_NAME = {
|
|
7206
7366
|
dev: "Developer",
|
|
7207
7367
|
design: "Design",
|
|
@@ -7380,7 +7540,7 @@ async function syncPersonas(route, opts) {
|
|
|
7380
7540
|
try {
|
|
7381
7541
|
anchoredAt = (await client2.records.update(agentsModel.id, row.id, promptSourceFields(agentsModel, defaultPrompt), row.updated_at)).updated_at;
|
|
7382
7542
|
} catch (e) {
|
|
7383
|
-
if (!(e instanceof
|
|
7543
|
+
if (!(e instanceof StaleWriteError7) && !isPromptGateRejection(e)) throw e;
|
|
7384
7544
|
}
|
|
7385
7545
|
}
|
|
7386
7546
|
const historyId = await fetchLatestHistoryId(route, agentsModel.id, row.id, opts.userAgent);
|
|
@@ -7452,7 +7612,7 @@ async function syncPersonas(route, opts) {
|
|
|
7452
7612
|
agentPersonas[role] = { agentId: row.id, lastSyncedUpdatedAt: updated.updated_at, ...historyId ? { historyId } : {} };
|
|
7453
7613
|
outcomes.push({ role, action: "updated", agentId: row.id });
|
|
7454
7614
|
} catch (e) {
|
|
7455
|
-
if (e instanceof
|
|
7615
|
+
if (e instanceof StaleWriteError7) {
|
|
7456
7616
|
outcomes.push({ role, action: "diverged", agentId: row.id });
|
|
7457
7617
|
continue;
|
|
7458
7618
|
}
|
|
@@ -7636,7 +7796,7 @@ async function syncSkills(route, opts) {
|
|
|
7636
7796
|
row.updated_at
|
|
7637
7797
|
)).updated_at;
|
|
7638
7798
|
} catch (e) {
|
|
7639
|
-
if (!(e instanceof
|
|
7799
|
+
if (!(e instanceof StaleWriteError8) && !isPromptGateRejection(e)) throw e;
|
|
7640
7800
|
}
|
|
7641
7801
|
}
|
|
7642
7802
|
agentSkills[skill.name] = { skillId: row.id, lastSyncedUpdatedAt: anchoredAt };
|
|
@@ -7692,7 +7852,7 @@ async function syncSkills(route, opts) {
|
|
|
7692
7852
|
agentSkills[skill.name] = { skillId: row.id, lastSyncedUpdatedAt: updated.updated_at };
|
|
7693
7853
|
outcomes.push({ name: skill.name, action: "updated", skillId: row.id });
|
|
7694
7854
|
} catch (e) {
|
|
7695
|
-
if (e instanceof
|
|
7855
|
+
if (e instanceof StaleWriteError8) {
|
|
7696
7856
|
outcomes.push({ name: skill.name, action: "diverged", skillId: row.id });
|
|
7697
7857
|
continue;
|
|
7698
7858
|
}
|
|
@@ -8384,7 +8544,7 @@ async function enforceHandoff(i) {
|
|
|
8384
8544
|
return { kind: "unverified", why: v.why };
|
|
8385
8545
|
}
|
|
8386
8546
|
if (v.kind === "pushed") {
|
|
8387
|
-
await i.writer.postEvent(t.id, pushedEventBody(v.sha, v.remote, v.branch), i.seatId);
|
|
8547
|
+
await i.writer.postEvent(t.id, pushedEventBody(v.sha, v.remote, v.branch), i.seatId, "pushed");
|
|
8388
8548
|
i.log.emit(`${t.issue_id} pushed at ${v.sha.slice(0, 7)} \u2014 handed on to QA`, log);
|
|
8389
8549
|
return { kind: "pushed", sha: v.sha };
|
|
8390
8550
|
}
|
|
@@ -8394,7 +8554,7 @@ async function enforceHandoff(i) {
|
|
|
8394
8554
|
...t.assignee_id ? {} : { assignee_id: i.seatId },
|
|
8395
8555
|
...i.hold && !heldBy ? { [i.hold.column]: i.hold.shipId } : {}
|
|
8396
8556
|
});
|
|
8397
|
-
await i.writer.postEvent(t.id, bounceComment(v, i.shipName), i.seatId).catch(() => {
|
|
8557
|
+
await i.writer.postEvent(t.id, bounceComment(v, i.shipName), i.seatId, "bounced").catch(() => {
|
|
8398
8558
|
i.log.warn("bounced, but the note failed to post", log);
|
|
8399
8559
|
});
|
|
8400
8560
|
i.log.warn(`${t.issue_id} bounced back to ${i.contract.statuses.building}: ${v.reason}`, log);
|
|
@@ -8452,7 +8612,7 @@ async function retireRunWorktree(tracker, dir, cfg, t, role, seatId, shipName2,
|
|
|
8452
8612
|
if (o.kind === "removed") log.emit(`${t.issue_id}: removed the run's worktree ${o.path} \u2014 the branch is on ${cfg.branch.remote}`, at);
|
|
8453
8613
|
else if (o.kind === "kept-handoff") {
|
|
8454
8614
|
log.emit(`${t.issue_id}: kept worktree ${o.path} \u2014 this repo's handoff hook serves from it`, at);
|
|
8455
|
-
await tracker.postEvent(t.id, `worktree kept on ship ${shipName2} at \`${o.path}\`: the repo's \`handoff\` hook serves from it. QA on another ship cuts its own from \`${cfg.branch.remote}/${branch}\` instead.`, seatId);
|
|
8615
|
+
await tracker.postEvent(t.id, `worktree kept on ship ${shipName2} at \`${o.path}\`: the repo's \`handoff\` hook serves from it. QA on another ship cuts its own from \`${cfg.branch.remote}/${branch}\` instead.`, seatId, "worktree_kept");
|
|
8456
8616
|
} else if (o.kind === "kept-dirty") log.warn(`${t.issue_id}: left worktree ${o.path} \u2014 it has uncommitted changes`, at);
|
|
8457
8617
|
else if (o.kind === "kept-unpushed") log.warn(`${t.issue_id}: left worktree ${o.path} \u2014 its branch has commits that are not on ${cfg.branch.remote}`, at);
|
|
8458
8618
|
} catch (e) {
|
|
@@ -8981,7 +9141,7 @@ ${h.prFailure.tail}
|
|
|
8981
9141
|
[contract.columns.status]: reviewing,
|
|
8982
9142
|
...h.prRef && columns.prRef ? { [contract.columns.prRef]: h.prRef } : {}
|
|
8983
9143
|
}, t.updated_at);
|
|
8984
|
-
await tracker.postEvent(t.id, lines.join("\n\n"), seat);
|
|
9144
|
+
await tracker.postEvent(t.id, lines.join("\n\n"), seat, "review");
|
|
8985
9145
|
emit.emit(`moved to ${reviewing}`, { ticket: t.issue_id });
|
|
8986
9146
|
} catch (e) {
|
|
8987
9147
|
emit.warn(`could not move to ${reviewing}: ${e.message}`, { ticket: t.issue_id });
|
|
@@ -8996,7 +9156,7 @@ ${h.prFailure.tail}
|
|
|
8996
9156
|
}
|
|
8997
9157
|
try {
|
|
8998
9158
|
await tracker.updateTicket(t.id, { [contract.columns.status]: contract.statuses.handoff }, t.updated_at);
|
|
8999
|
-
await tracker.postEvent(t.id, body, seat);
|
|
9159
|
+
await tracker.postEvent(t.id, body, seat, "requeued");
|
|
9000
9160
|
emit.emit(`returned to ${contract.statuses.handoff} \u2014 branch moved past ${q.verified.slice(0, 8)}`, { ticket: t.issue_id });
|
|
9001
9161
|
} catch (e) {
|
|
9002
9162
|
emit.warn(`could not return to ${contract.statuses.handoff}: ${e.message}`, { ticket: t.issue_id });
|
|
@@ -9474,8 +9634,62 @@ async function runRelease(o) {
|
|
|
9474
9634
|
}
|
|
9475
9635
|
}
|
|
9476
9636
|
}
|
|
9477
|
-
|
|
9637
|
+
var isolateCap = (o) => o.repo.release.isolateFailures?.maxCandidates ?? DEFAULT_ISOLATE_MAX_CANDIDATES;
|
|
9638
|
+
async function isolateAndRetry(o, name, output, merged, preMerge, decision, fresh, releaseBranch) {
|
|
9639
|
+
const applied = merged.filter((c) => c.sha && c.branch);
|
|
9640
|
+
const cap = isolateCap(o);
|
|
9641
|
+
if (o.dryRun || !preMerge || applied.length < 2 || cap === 0) return null;
|
|
9642
|
+
const head = headSha(o.cwd);
|
|
9643
|
+
const withFiles = applied.map((c) => ({
|
|
9644
|
+
c,
|
|
9645
|
+
changed: new Set(
|
|
9646
|
+
(gitOk(o.cwd, ["diff", "--name-only", `${preMerge}...${c.branch}`]) ?? "").split("\n").filter(Boolean)
|
|
9647
|
+
)
|
|
9648
|
+
}));
|
|
9649
|
+
const culprits = [];
|
|
9650
|
+
const byFile = culpritByFiles(withFiles, failingPaths(output, o.cwd));
|
|
9651
|
+
if (byFile) {
|
|
9652
|
+
o.emit.warn(`${name} failed; its output names files only ${byFile.c.branch} changed \u2014 treating it as the culprit`, { ticket: byFile.c.ticket.issue_id });
|
|
9653
|
+
culprits.push({ c: byFile.c, how: "files", tail: lastLines(output, HOOK_FAILURE_TAIL_LINES) });
|
|
9654
|
+
} else if (applied.length > cap) {
|
|
9655
|
+
o.emit.warn(`${name} failed with ${applied.length} branches merged, over release.isolateFailures.maxCandidates (${cap}) \u2014 not isolating`);
|
|
9656
|
+
return null;
|
|
9657
|
+
} else {
|
|
9658
|
+
o.emit.warn(`${name} failed with ${applied.length} branches merged \u2014 building each alone to find the culprit`);
|
|
9659
|
+
for (const c of applied) {
|
|
9660
|
+
gitOk(o.cwd, ["reset", "--hard", preMerge]);
|
|
9661
|
+
const m = mergeOne(o, c);
|
|
9662
|
+
if (typeof m === "object" && "conflict" in m) continue;
|
|
9663
|
+
const r = await hook(o, name);
|
|
9664
|
+
if (r && r.code !== 0) {
|
|
9665
|
+
o.emit.warn(`${c.branch} fails the ${name} hook on its own`, { ticket: c.ticket.issue_id });
|
|
9666
|
+
culprits.push({ c, how: "solo", tail: lastLines(r.output, HOOK_FAILURE_TAIL_LINES) });
|
|
9667
|
+
}
|
|
9668
|
+
}
|
|
9669
|
+
}
|
|
9670
|
+
gitOk(o.cwd, ["reset", "--hard", preMerge]);
|
|
9671
|
+
if (culprits.length === 0) {
|
|
9672
|
+
o.emit.warn("no single branch could be blamed \u2014 failing the release as a whole");
|
|
9673
|
+
gitOk(o.cwd, ["reset", "--hard", head]);
|
|
9674
|
+
return null;
|
|
9675
|
+
}
|
|
9676
|
+
const out = new Set(culprits.map((x) => x.c.ticket.id));
|
|
9677
|
+
const rest = decision.merges.filter((c) => !out.has(c.ticket.id));
|
|
9678
|
+
const retried = await mergeAndRelease(o, { ...decision, merges: rest }, fresh, releaseBranch, true);
|
|
9679
|
+
return {
|
|
9680
|
+
...retried,
|
|
9681
|
+
buildCulprits: culprits.map((x) => ({
|
|
9682
|
+
ticket: x.c.ticket,
|
|
9683
|
+
branch: x.c.branch,
|
|
9684
|
+
hook: name,
|
|
9685
|
+
tail: x.tail,
|
|
9686
|
+
how: x.how
|
|
9687
|
+
}))
|
|
9688
|
+
};
|
|
9689
|
+
}
|
|
9690
|
+
async function mergeAndRelease(o, decision, fresh, releaseBranch, isolated = false) {
|
|
9478
9691
|
o.emit.enter("merge");
|
|
9692
|
+
const preMerge = o.dryRun ? "" : headSha(o.cwd);
|
|
9479
9693
|
const merged = [];
|
|
9480
9694
|
const conflicts = [];
|
|
9481
9695
|
const unbuildable = [];
|
|
@@ -9590,6 +9804,10 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch) {
|
|
|
9590
9804
|
o.emit.warn("test gate failed with what looks like a worker crash, not a real failure \u2014 retrying once");
|
|
9591
9805
|
r = await hook(o, "test");
|
|
9592
9806
|
}
|
|
9807
|
+
if (r && r.code !== 0 && !isolated) {
|
|
9808
|
+
const out = await isolateAndRetry(o, "test", r.output, merged, preMerge, decision, fresh, releaseBranch);
|
|
9809
|
+
if (out) return out;
|
|
9810
|
+
}
|
|
9593
9811
|
if (r && r.code !== 0) {
|
|
9594
9812
|
o.emit.error(`test gate FAILED \u2014 not deploying; the target stays on the previous release`);
|
|
9595
9813
|
const sha = headSha(o.cwd);
|
|
@@ -9619,13 +9837,23 @@ async function mergeAndRelease(o, decision, fresh, releaseBranch) {
|
|
|
9619
9837
|
}
|
|
9620
9838
|
}
|
|
9621
9839
|
if (o.repo.hooks.build) {
|
|
9622
|
-
if (o.dryRun)
|
|
9623
|
-
|
|
9840
|
+
if (o.dryRun) {
|
|
9841
|
+
o.emit.emit(`would run: ${hookLabel(o.repo, "build")}`);
|
|
9842
|
+
if (applied > 1) {
|
|
9843
|
+
o.emit.emit(
|
|
9844
|
+
`if it fails, would find the one of ${applied} merged branches that broke it (by the files it names, else by building each alone, up to ${isolateCap(o)}), hand that back to the dev lane and ship the rest`
|
|
9845
|
+
);
|
|
9846
|
+
}
|
|
9847
|
+
} else {
|
|
9624
9848
|
let r = await hook(o, "build");
|
|
9625
9849
|
if (r && r.code !== 0 && looksLikeTransientResolveFailure(r.output)) {
|
|
9626
9850
|
o.emit.warn("build failed with what looks like a transient dependency-resolution glitch, not a real failure \u2014 retrying once");
|
|
9627
9851
|
r = await hook(o, "build");
|
|
9628
9852
|
}
|
|
9853
|
+
if (r && r.code !== 0 && !isolated) {
|
|
9854
|
+
const out = await isolateAndRetry(o, "build", r.output, merged, preMerge, decision, fresh, releaseBranch);
|
|
9855
|
+
if (out) return out;
|
|
9856
|
+
}
|
|
9629
9857
|
if (r && r.code !== 0) {
|
|
9630
9858
|
o.emit.error("build failed \u2014 not deploying");
|
|
9631
9859
|
if (version && !o.dryRun && headSha(o.cwd) !== preBump) {
|
|
@@ -9757,7 +9985,7 @@ async function applyFailureAlert(writer, n, route, openTickets, memberId, log, d
|
|
|
9757
9985
|
}
|
|
9758
9986
|
try {
|
|
9759
9987
|
await writer.postEvent(existing.id, `${FAILURE_ALERT_MARKER}
|
|
9760
|
-
Still failing: ${n.detail ?? n.headline}`, memberId);
|
|
9988
|
+
Still failing: ${n.detail ?? n.headline}`, memberId, "failure_alert");
|
|
9761
9989
|
log.warn(`release failure recurring \u2014 commented on ${label}`, { step: "release" });
|
|
9762
9990
|
return { kind: "commented", issueId: label };
|
|
9763
9991
|
} catch (e) {
|
|
@@ -9824,7 +10052,7 @@ function formatProjectsReport(report) {
|
|
|
9824
10052
|
}
|
|
9825
10053
|
|
|
9826
10054
|
// src/cli.ts
|
|
9827
|
-
import { StaleWriteError as
|
|
10055
|
+
import { StaleWriteError as StaleWriteError9 } from "@tablation/client";
|
|
9828
10056
|
|
|
9829
10057
|
// src/release-ship.ts
|
|
9830
10058
|
var DAY_MS = 24 * 60 * 60 * 1e3;
|
|
@@ -10382,6 +10610,13 @@ async function discover(o) {
|
|
|
10382
10610
|
} catch {
|
|
10383
10611
|
}
|
|
10384
10612
|
}
|
|
10613
|
+
if (out.models.issues) {
|
|
10614
|
+
try {
|
|
10615
|
+
await get(o, `/data-models/${out.models.issues}/events?limit=1`);
|
|
10616
|
+
} catch (e) {
|
|
10617
|
+
if (e instanceof ConnectHttpError && e.status === 404) problems.push(EVENT_LOG_MISSING);
|
|
10618
|
+
}
|
|
10619
|
+
}
|
|
10385
10620
|
out.areaModelId = find("Projects");
|
|
10386
10621
|
out.reposModelId = find("Repos");
|
|
10387
10622
|
out.shipsModelId = find("Ships");
|
|
@@ -12926,11 +13161,11 @@ ${changed2.map((c) => ` - ${c}
|
|
|
12926
13161
|
if (outcome.gateRed && !dryRun) {
|
|
12927
13162
|
try {
|
|
12928
13163
|
const g = outcome.gateRed;
|
|
12929
|
-
const
|
|
13164
|
+
const reported = await tracker.events("gate_red");
|
|
12930
13165
|
const seat = releaseSeat(c);
|
|
12931
13166
|
for (const t of gateRedTickets(outcome.merged, outcome.decision.merges)) {
|
|
12932
|
-
if (alreadyReported(t, g,
|
|
12933
|
-
await tracker.postEvent(t.id, gateRedComment(g), seat);
|
|
13167
|
+
if (alreadyReported(t, g, reported)) continue;
|
|
13168
|
+
await tracker.postEvent(t.id, gateRedComment(g), seat, "gate_red");
|
|
12934
13169
|
remit.warn(`test gate red for ${g.count} cycles \u2014 noted on the ticket`, { ticket: t.issue_id, step: "merge" });
|
|
12935
13170
|
}
|
|
12936
13171
|
state.release(scope).noteTestGateReported(g.sha);
|
|
@@ -12963,8 +13198,23 @@ ${changed2.map((c) => ` - ${c}
|
|
|
12963
13198
|
});
|
|
12964
13199
|
}
|
|
12965
13200
|
}
|
|
13201
|
+
if (outcome.buildCulprits?.length) {
|
|
13202
|
+
const seat = releaseSeat(c);
|
|
13203
|
+
for (const cu of outcome.buildCulprits) {
|
|
13204
|
+
try {
|
|
13205
|
+
const bounce = planBuildFailureBounce(target.dir, cu, repo.branch.base, repo.branch.remote);
|
|
13206
|
+
const r = await applyBuildFailureBounce(tracker, bounce, tracker.contract, seat, remit, dryRun);
|
|
13207
|
+
if (r.kind === "failed") {
|
|
13208
|
+
remit.warn(`could not hand back ${cu.ticket.issue_id}: ${r.why}`, { step: "release" });
|
|
13209
|
+
}
|
|
13210
|
+
} catch (e) {
|
|
13211
|
+
remit.warn(`could not hand back ${cu.ticket.issue_id}: ${e.message}`, { step: "release" });
|
|
13212
|
+
}
|
|
13213
|
+
}
|
|
13214
|
+
}
|
|
12966
13215
|
if (outcome.conflicts?.length || outcome.unbuildable?.length) {
|
|
12967
|
-
const
|
|
13216
|
+
const handedBack = await tracker.events("conflict_handed_back");
|
|
13217
|
+
const flagged = await tracker.events("stranded_flagged");
|
|
12968
13218
|
const seat = releaseSeat(c);
|
|
12969
13219
|
for (const f of outcome.conflicts ?? []) {
|
|
12970
13220
|
try {
|
|
@@ -12974,7 +13224,7 @@ ${changed2.map((c) => ` - ${c}
|
|
|
12974
13224
|
f.candidate.branch,
|
|
12975
13225
|
repo.branch.base,
|
|
12976
13226
|
f.paths,
|
|
12977
|
-
|
|
13227
|
+
handedBack
|
|
12978
13228
|
);
|
|
12979
13229
|
const r = await applyConflictBounce(tracker, bounce, tracker.contract, seat, remit, dryRun);
|
|
12980
13230
|
if (r.kind === "failed") {
|
|
@@ -12989,7 +13239,7 @@ ${changed2.map((c) => ` - ${c}
|
|
|
12989
13239
|
}
|
|
12990
13240
|
for (const cand of outcome.unbuildable ?? []) {
|
|
12991
13241
|
try {
|
|
12992
|
-
const stranded = planStrandedVerified(cand, repo.branch.base, repo.branch.remote,
|
|
13242
|
+
const stranded = planStrandedVerified(cand, repo.branch.base, repo.branch.remote, flagged);
|
|
12993
13243
|
const r = await applyStrandedVerified(
|
|
12994
13244
|
tracker,
|
|
12995
13245
|
stranded,
|
|
@@ -14658,7 +14908,7 @@ Signed in as ${result.identity.email} \u2014 minted an API key in workspace "${r
|
|
|
14658
14908
|
await tracker3.updateTicket(ticket.id, { [c.columns.assignee]: null }, ticket.updated_at);
|
|
14659
14909
|
emit.emit(`cleared assignee on ${label} \u2014 back up for grabs next cycle`);
|
|
14660
14910
|
} catch (e) {
|
|
14661
|
-
if (e instanceof
|
|
14911
|
+
if (e instanceof StaleWriteError9) {
|
|
14662
14912
|
process.stderr.write(`unassign: ${label} changed on the board since it was read \u2014 re-run to retry
|
|
14663
14913
|
`);
|
|
14664
14914
|
process.exit(2);
|
package/package.json
CHANGED
|
@@ -56,7 +56,7 @@ pass.
|
|
|
56
56
|
4. **Get the branch and read the diff.** The branch on the remote is what
|
|
57
57
|
you test, and you never need the builder's worktree: cut your own with
|
|
58
58
|
`crew worktree <number>` (it prints the path), unless the builder's
|
|
59
|
-
hand-off event says its ship kept a worktree and a server for you, in
|
|
59
|
+
hand-off event (in the ticket's event log) says its ship kept a worktree and a server for you, in
|
|
60
60
|
which case you may open that URL instead. Remove yours when you hand the
|
|
61
61
|
ticket on. The digest's `branch` column reads `<name> (on <remote> only)`
|
|
62
62
|
when another ship built it — still testable; only **MISSING** (gone
|
|
@@ -65,7 +65,9 @@ pass.
|
|
|
65
65
|
|
|
66
66
|
**Name the sha you test.** Your starting comment states the
|
|
67
67
|
`<remote>/<branch>` tip you fetched. The runner records the pushed tip on
|
|
68
|
-
the ticket
|
|
68
|
+
the ticket's activity (its event log, not a comment) as a `pushed` event
|
|
69
|
+
reading `pushed <sha> to <remote>/<branch>`; read it with the events list
|
|
70
|
+
on the Issues table, filtered to the `pushed` type. If your tip differs
|
|
69
71
|
from the latest such event, say so and verify the remote tip anyway — the
|
|
70
72
|
remote is the only git truth.
|
|
71
73
|
|
|
@@ -161,6 +163,11 @@ pass.
|
|
|
161
163
|
base, the bounce says to merge `<remote>/<base>` into the branch, resolve and
|
|
162
164
|
push fast-forward; a rebase would need a force push the builder cannot make.
|
|
163
165
|
Or pass the ticket and let the release's conflict path do the merge.
|
|
166
|
+
- **A branch that came back from a conflict or build-failure bounce is
|
|
167
|
+
re-verified against the current base, not only its own tip.** Before
|
|
168
|
+
passing it, merge current `<remote>/<base>` into the worktree (or run the
|
|
169
|
+
build on a throwaway merge) and run the build: a clean text merge can still
|
|
170
|
+
fail to compile, and it is far cheaper to find that here than at release.
|
|
164
171
|
- **You only ever enter the worktree of the ticket you are verifying.**
|
|
165
172
|
- Throwaway verification scripts go in a scratch directory outside the
|
|
166
173
|
repository — not the worktree root either.
|
|
@@ -363,7 +363,7 @@ before going further, and never change a ticket's `report_type` yourself.
|
|
|
363
363
|
`fixed` and `<remote>/<branch>` is missing or not at your worktree's
|
|
364
364
|
HEAD, it sets the ticket back to `in_progress` with an event saying what
|
|
365
365
|
it found, and QA never sees it. When it matches, the runner records
|
|
366
|
-
`pushed <sha> to <remote>/<branch>`
|
|
366
|
+
`pushed <sha> to <remote>/<branch>` in the ticket's activity (its event log); you still state the sha
|
|
367
367
|
in your last progress comment.
|
|
368
368
|
|
|
369
369
|
**The branch on the remote is what QA tests.** Leave it unmerged, and do
|
|
@@ -55,7 +55,7 @@ pass.
|
|
|
55
55
|
4. **Get the branch and read the diff.** The branch on the remote is what
|
|
56
56
|
you test, and you never need the builder's worktree: cut your own with
|
|
57
57
|
`crew worktree <number>` (it prints the path), unless the builder's
|
|
58
|
-
hand-off event says its ship kept a worktree and a server for you, in
|
|
58
|
+
hand-off event (in the ticket's event log) says its ship kept a worktree and a server for you, in
|
|
59
59
|
which case you may open that URL instead. Remove yours when you hand the
|
|
60
60
|
ticket on. The digest's `branch` column reads `<name> (on <remote> only)`
|
|
61
61
|
when another ship built it — still testable; only **MISSING** (gone
|
|
@@ -64,7 +64,9 @@ pass.
|
|
|
64
64
|
|
|
65
65
|
**Name the sha you test.** Your starting comment states the
|
|
66
66
|
`<remote>/<branch>` tip you fetched. The runner records the pushed tip on
|
|
67
|
-
the ticket
|
|
67
|
+
the ticket's activity (its event log, not a comment) as a `pushed` event
|
|
68
|
+
reading `pushed <sha> to <remote>/<branch>`; read it with the events list
|
|
69
|
+
on the Issues table, filtered to the `pushed` type. If your tip differs
|
|
68
70
|
from the latest such event, say so and verify the remote tip anyway — the
|
|
69
71
|
remote is the only git truth.
|
|
70
72
|
|
|
@@ -160,6 +162,11 @@ pass.
|
|
|
160
162
|
base, the bounce says to merge `<remote>/<base>` into the branch, resolve and
|
|
161
163
|
push fast-forward; a rebase would need a force push the builder cannot make.
|
|
162
164
|
Or pass the ticket and let the release's conflict path do the merge.
|
|
165
|
+
- **A branch that came back from a conflict or build-failure bounce is
|
|
166
|
+
re-verified against the current base, not only its own tip.** Before
|
|
167
|
+
passing it, merge current `<remote>/<base>` into the worktree (or run the
|
|
168
|
+
build on a throwaway merge) and run the build: a clean text merge can still
|
|
169
|
+
fail to compile, and it is far cheaper to find that here than at release.
|
|
163
170
|
- **You only ever enter the worktree of the ticket you are verifying.**
|
|
164
171
|
- Throwaway verification scripts go in a scratch directory outside the
|
|
165
172
|
repository — not the worktree root either.
|
|
@@ -363,7 +363,7 @@ before going further, and never change a ticket's `report_type` yourself.
|
|
|
363
363
|
`fixed` and `<remote>/<branch>` is missing or not at your worktree's
|
|
364
364
|
HEAD, it sets the ticket back to `in_progress` with an event saying what
|
|
365
365
|
it found, and QA never sees it. When it matches, the runner records
|
|
366
|
-
`pushed <sha> to <remote>/<branch>`
|
|
366
|
+
`pushed <sha> to <remote>/<branch>` in the ticket's activity (its event log); you still state the sha
|
|
367
367
|
in your last progress comment.
|
|
368
368
|
|
|
369
369
|
**The branch on the remote is what QA tests.** Leave it unmerged, and do
|