@coreplane/switchboard 1.261.4 → 1.262.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/deploy/cloudflare/coordinator.ts +18 -1
- package/dist/assets/deploy/cloudflare/worker.ts +4 -1
- package/dist/assets/deploy/cloudflare-memory/worker.ts +50 -2
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +1 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/core/coordinator/contract.ts +137 -4
- package/dist/assets/src/core/coordinator/driver.ts +192 -40
- package/dist/assets/src/core/refusal.ts +1 -0
- package/dist/assets/src/core/runEvents.ts +2 -0
- package/dist/assets/src/core/ship/coordinator.ts +91 -15
- package/dist/cli.js +786 -41
- package/package.json +1 -1
|
@@ -55,6 +55,7 @@ import {
|
|
|
55
55
|
cursorFinished,
|
|
56
56
|
nextAction,
|
|
57
57
|
openPlanCursor,
|
|
58
|
+
openRecoveredUnitPipeline,
|
|
58
59
|
openUnitPipeline,
|
|
59
60
|
readyUnits,
|
|
60
61
|
type PlanCursor,
|
|
@@ -76,6 +77,7 @@ import {
|
|
|
76
77
|
type ShipCaps,
|
|
77
78
|
type StepReturn,
|
|
78
79
|
type UnitEnding,
|
|
80
|
+
type UnitPipelineInput,
|
|
79
81
|
type UnitPipelineState,
|
|
80
82
|
stepPrefixOf,
|
|
81
83
|
type LeaseSegmentProgress,
|
|
@@ -114,6 +116,7 @@ export interface StepRunner {
|
|
|
114
116
|
}
|
|
115
117
|
|
|
116
118
|
export type CoordinatorStepRoute =
|
|
119
|
+
| "recover-unit"
|
|
117
120
|
| "plan"
|
|
118
121
|
| "unit-start"
|
|
119
122
|
| "branch"
|
|
@@ -159,6 +162,12 @@ export interface PlanRunSummary {
|
|
|
159
162
|
outcome: "completed" | "failed";
|
|
160
163
|
}
|
|
161
164
|
|
|
165
|
+
export interface OriginalUnitRecoveryParams {
|
|
166
|
+
kind: "recover-original-unit";
|
|
167
|
+
parentInstanceId: string;
|
|
168
|
+
unit: string;
|
|
169
|
+
}
|
|
170
|
+
|
|
162
171
|
// ---- reading the bot ---------------------------------------------------------------------------------
|
|
163
172
|
|
|
164
173
|
const REASON_MAX = 200;
|
|
@@ -199,6 +208,7 @@ const TRANSIENT = new Set([
|
|
|
199
208
|
"unit_not_started",
|
|
200
209
|
"not_host",
|
|
201
210
|
"queued",
|
|
211
|
+
"publication_ownership_unknown",
|
|
202
212
|
]);
|
|
203
213
|
export function transientRefusal(answer: BotAnswer): string | undefined {
|
|
204
214
|
const { ok, error, message } = answer.body;
|
|
@@ -917,6 +927,8 @@ async function tellStepThrew(
|
|
|
917
927
|
pr: { number: number; url: string } | undefined,
|
|
918
928
|
at: StepAt,
|
|
919
929
|
err: unknown,
|
|
930
|
+
recoveryUnitKey?: string,
|
|
931
|
+
recoveryWorkflowId?: string,
|
|
920
932
|
): Promise<void> {
|
|
921
933
|
const line = oneLineOf(err);
|
|
922
934
|
const where =
|
|
@@ -932,14 +944,19 @@ async function tellStepThrew(
|
|
|
932
944
|
pr !== undefined
|
|
933
945
|
? `Saved-work fact: the existing pull request is ${pr.url}, but its current branch and exact head were not verified.`
|
|
934
946
|
: "Saved-work fact: no open pull request or exact remote head was verified.",
|
|
935
|
-
|
|
947
|
+
recoveryUnitKey !== undefined
|
|
948
|
+
? `No review was started. The recovery attempt for \`${recoveryUnitKey}\` is terminal; a replacement pipeline is not available.`
|
|
949
|
+
: "No review was started. Next action: retry ship when the pull request is readable so Switchboard can verify the exact head first.",
|
|
936
950
|
].join("\n\n")
|
|
937
951
|
: [
|
|
938
952
|
`⚠️ Switchboard failed ${where}: ${line}`,
|
|
939
|
-
|
|
953
|
+
recoveryUnitKey !== undefined
|
|
954
|
+
? `The recovery attempt for \`${recoveryUnitKey}\` is terminal. No continuation or replacement pipeline was scheduled.`
|
|
955
|
+
: "The unit remains resumable from its recorded branch and pull request facts. No continuation action was scheduled. Next action: start ship again after the failing operation is available.",
|
|
940
956
|
].join("\n\n");
|
|
941
957
|
const body = {
|
|
942
958
|
...tag,
|
|
959
|
+
...(recoveryWorkflowId !== undefined ? { recoveryWorkflowId } : {}),
|
|
943
960
|
ending: {
|
|
944
961
|
kind: "failed",
|
|
945
962
|
cause: "step_threw",
|
|
@@ -974,7 +991,8 @@ async function runUnit(
|
|
|
974
991
|
// A renewal's segment names its steps under the segment (`U10/s2/…`), so
|
|
975
992
|
// the Workflow's durable step cache never answers segment two with segment
|
|
976
993
|
// one's results (decision 0046).
|
|
977
|
-
const
|
|
994
|
+
const row = plan.units.find((u) => u.unit === unit);
|
|
995
|
+
const prefix = row?.recovery !== undefined ? `${unit}/recovery` : stepPrefixOf(unit, session);
|
|
978
996
|
const tag = { parentInstanceId: instanceId, unit };
|
|
979
997
|
// Where the machine is, tracked for the step-threw ending (issue 2100). The
|
|
980
998
|
// start belongs in the same net as the rest of the unit even though no
|
|
@@ -982,54 +1000,106 @@ async function runUnit(
|
|
|
982
1000
|
let last: StepAt = { step: `${prefix}/start` };
|
|
983
1001
|
let start: ReturnType<typeof readUnitStart>;
|
|
984
1002
|
try {
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
1003
|
+
if (row?.recovery !== undefined) start = { at: row.recovery.claimedAt };
|
|
1004
|
+
else {
|
|
1005
|
+
const startStep = `${prefix}/start`;
|
|
1006
|
+
last = { step: startStep };
|
|
1007
|
+
start = readUnitStart(
|
|
1008
|
+
answerOf("unit-start", await step.do(startStep, STEP_CONFIG, () => call(bot, "unit-start", tag))),
|
|
1009
|
+
);
|
|
1010
|
+
}
|
|
990
1011
|
} catch (err) {
|
|
991
|
-
await tellStepThrew(
|
|
1012
|
+
await tellStepThrew(
|
|
1013
|
+
step,
|
|
1014
|
+
bot,
|
|
1015
|
+
prefix,
|
|
1016
|
+
tag,
|
|
1017
|
+
undefined,
|
|
1018
|
+
last,
|
|
1019
|
+
err,
|
|
1020
|
+
row?.recovery !== undefined ? `${instanceId}:${unit}` : undefined,
|
|
1021
|
+
row?.recovery?.workflowId,
|
|
1022
|
+
);
|
|
992
1023
|
throw err;
|
|
993
1024
|
}
|
|
994
1025
|
// A resume at review (agent-ship item 10) rides the unit's row: the pull
|
|
995
1026
|
// request of ship's own the requester named opens the pipeline at its first
|
|
996
1027
|
// review round, with no pre-check, no branch and no round 0.
|
|
997
|
-
const row = plan.units.find((u) => u.unit === unit);
|
|
998
1028
|
const resume = row?.resume;
|
|
999
1029
|
// A previous attempt's `review_pending` head: the machine's pre-check starts
|
|
1000
1030
|
// at the review round when the open pull request still heads exactly there.
|
|
1001
1031
|
const lastPush = row?.lastPush;
|
|
1002
|
-
|
|
1003
|
-
{
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
},
|
|
1025
|
-
|
|
1026
|
-
|
|
1032
|
+
const input: UnitPipelineInput = {
|
|
1033
|
+
unit: { id: unit, branch: node.branch },
|
|
1034
|
+
repo: plan.repo,
|
|
1035
|
+
base: plan.base,
|
|
1036
|
+
caps: plan.caps,
|
|
1037
|
+
// The instance's field decides who merges (record 0031's merge grant),
|
|
1038
|
+
// carried here by the plan route: the hand-off wrote `runner` on a
|
|
1039
|
+
// seeded plan and `person` on a task, and the door re-checks it — the
|
|
1040
|
+
// branch's name never decides.
|
|
1041
|
+
merge: plan.merge,
|
|
1042
|
+
addressSeverity: plan.addressSeverity,
|
|
1043
|
+
addressSeveritySource: plan.addressSeveritySource,
|
|
1044
|
+
// Recovery spends only the lease already carried by the claim. It never
|
|
1045
|
+
// opens another segment or idles for a renewal in this checkpoint.
|
|
1046
|
+
grant: row?.recovery !== undefined ? { renewals: 0 } : plan.grant,
|
|
1047
|
+
grantSource: plan.grantSource,
|
|
1048
|
+
verbosity: plan.verbosity,
|
|
1049
|
+
idleDays: row?.recovery !== undefined ? 0 : plan.idleDays,
|
|
1050
|
+
generated: plan.generated,
|
|
1051
|
+
...(plan.runPageBase !== undefined ? { runPageBase: plan.runPageBase } : {}),
|
|
1052
|
+
...(resume !== undefined ? { resume } : {}),
|
|
1053
|
+
...(lastPush !== undefined ? { lastPush } : {}),
|
|
1054
|
+
...(session !== undefined ? { session } : {}),
|
|
1055
|
+
...(row?.recovery !== undefined
|
|
1056
|
+
? { recovery: { remainingMs: row.recovery.remainingMs, unitKey: `${instanceId}:${unit}` } }
|
|
1057
|
+
: {}),
|
|
1058
|
+
};
|
|
1059
|
+
let state: UnitPipelineState =
|
|
1060
|
+
row?.recovery !== undefined
|
|
1061
|
+
? openRecoveredUnitPipeline(input, start.at, {
|
|
1062
|
+
kind: row.recovery.kind,
|
|
1063
|
+
round: row.recovery.round,
|
|
1064
|
+
pr: row.pr!,
|
|
1065
|
+
expectedHeadSha: row.recovery.expectedHeadSha,
|
|
1066
|
+
reviewRunId: row.recovery.reviewRunId,
|
|
1067
|
+
...(row.recovery.findingsRunId !== undefined ? { findingsRunId: row.recovery.findingsRunId } : {}),
|
|
1068
|
+
...(row.recovery.findings !== undefined ? { findings: row.recovery.findings } : {}),
|
|
1069
|
+
})
|
|
1070
|
+
: openUnitPipeline(input, start.at);
|
|
1027
1071
|
let notes = 0;
|
|
1028
1072
|
let endedAt: number | undefined;
|
|
1029
1073
|
try {
|
|
1030
1074
|
pipeline: for (;;) {
|
|
1031
1075
|
const action = nextAction(state);
|
|
1032
|
-
if (action.type === "end")
|
|
1076
|
+
if (action.type === "end") {
|
|
1077
|
+
if (row?.recovery !== undefined && endedAt === undefined) {
|
|
1078
|
+
const endStep = `${prefix}/end`;
|
|
1079
|
+
const ending = action.ending;
|
|
1080
|
+
const body = {
|
|
1081
|
+
...tag,
|
|
1082
|
+
recoveryWorkflowId: row.recovery.workflowId,
|
|
1083
|
+
ending: {
|
|
1084
|
+
kind: ending.kind === "idle" && ending.humanGate !== undefined ? "held" : ending.kind,
|
|
1085
|
+
...(ending.kind === "held" && ending.cause !== undefined ? { holdCause: ending.cause } : {}),
|
|
1086
|
+
report: renderUnitReport(state),
|
|
1087
|
+
threadReport: renderUnitReport(state, undefined, state.input.verbosity ?? DEFAULT_VERBOSITY),
|
|
1088
|
+
...(ending.kind === "idle" && ending.humanGate !== undefined ? { humanGate: ending.humanGate } : {}),
|
|
1089
|
+
},
|
|
1090
|
+
...(state.pr !== undefined ? { pr: state.pr } : {}),
|
|
1091
|
+
};
|
|
1092
|
+
const reply = await step.do(endStep, STEP_CONFIG, async () => {
|
|
1093
|
+
const candidate = await call(bot, "unit-end", body);
|
|
1094
|
+
const answer = answerOf("unit-end", candidate);
|
|
1095
|
+
if (answer.status !== 200 || answer.body.ok !== true)
|
|
1096
|
+
throw new UnreadableAnswer("unit-end", answer, "successful settlement");
|
|
1097
|
+
return candidate;
|
|
1098
|
+
});
|
|
1099
|
+
endedAt = answerOf("unit-end", reply).body.at;
|
|
1100
|
+
}
|
|
1101
|
+
return { ...action.ending, ...(endedAt !== undefined ? { endedAt } : {}) };
|
|
1102
|
+
}
|
|
1033
1103
|
// Keep the current round across its non-round phases (for example merge)
|
|
1034
1104
|
// while replacing it whenever the pipeline names a new one.
|
|
1035
1105
|
last = {
|
|
@@ -1046,6 +1116,7 @@ async function runUnit(
|
|
|
1046
1116
|
if (note.type === "round") {
|
|
1047
1117
|
const body = {
|
|
1048
1118
|
...tag,
|
|
1119
|
+
...(row?.recovery !== undefined ? { recoveryWorkflowId: row.recovery.workflowId } : {}),
|
|
1049
1120
|
index: note.index,
|
|
1050
1121
|
agent: note.agent,
|
|
1051
1122
|
outcome: note.outcome,
|
|
@@ -1053,7 +1124,13 @@ async function runUnit(
|
|
|
1053
1124
|
};
|
|
1054
1125
|
const noteStep = `${prefix}/note/${++notes}`;
|
|
1055
1126
|
last = { step: noteStep, round: { index: note.index, kind: note.agent } };
|
|
1056
|
-
await step.do(noteStep, STEP_CONFIG, () =>
|
|
1127
|
+
await step.do(noteStep, STEP_CONFIG, async () => {
|
|
1128
|
+
const reply = await call(bot, "round", body);
|
|
1129
|
+
const answer = answerOf("round", reply);
|
|
1130
|
+
if (answer.status !== 200 || answer.body.ok !== true)
|
|
1131
|
+
throw new UnreadableAnswer("round", answer, "successful round persistence");
|
|
1132
|
+
return reply;
|
|
1133
|
+
});
|
|
1057
1134
|
} else {
|
|
1058
1135
|
// A merge_ready ending names the pull request as it is at the APPROVED
|
|
1059
1136
|
// head (agent-ship item 9): one more pr-check reads the facts fresh —
|
|
@@ -1116,10 +1193,15 @@ async function runUnit(
|
|
|
1116
1193
|
// ending (agent-ship item 14).
|
|
1117
1194
|
const body = {
|
|
1118
1195
|
...tag,
|
|
1196
|
+
...(row?.recovery !== undefined ? { recoveryWorkflowId: row.recovery.workflowId } : {}),
|
|
1119
1197
|
// Two copies (routing-and-config item 28): the full report for the
|
|
1120
1198
|
// row and the board, and the thread's at the request's verbosity.
|
|
1121
1199
|
ending: {
|
|
1122
|
-
kind:
|
|
1200
|
+
kind:
|
|
1201
|
+
row?.recovery !== undefined && ending.kind === "idle" && ending.humanGate !== undefined
|
|
1202
|
+
? "held"
|
|
1203
|
+
: ending.kind,
|
|
1204
|
+
...(ending.kind === "held" && ending.cause !== undefined ? { holdCause: ending.cause } : {}),
|
|
1123
1205
|
report: renderUnitReport(state, endFacts),
|
|
1124
1206
|
threadReport: renderUnitReport(state, endFacts, state.input.verbosity ?? DEFAULT_VERBOSITY),
|
|
1125
1207
|
// An idle ending carries its continuation facts (record 0051): the
|
|
@@ -1135,6 +1217,9 @@ async function runUnit(
|
|
|
1135
1217
|
...(ending.humanGate !== undefined ? { humanGate: ending.humanGate } : {}),
|
|
1136
1218
|
}
|
|
1137
1219
|
: {}),
|
|
1220
|
+
...(row?.recovery !== undefined && ending.kind === "idle" && ending.humanGate !== undefined
|
|
1221
|
+
? { humanGate: ending.humanGate }
|
|
1222
|
+
: {}),
|
|
1138
1223
|
},
|
|
1139
1224
|
...(state.pr !== undefined ? { pr: state.pr } : {}),
|
|
1140
1225
|
// A merge_ready ending retains the exact head its final approval
|
|
@@ -1165,7 +1250,13 @@ async function runUnit(
|
|
|
1165
1250
|
last = { step: endStep, ...(last.round !== undefined ? { round: last.round } : {}) };
|
|
1166
1251
|
const endAnswer = answerOf(
|
|
1167
1252
|
"unit-end",
|
|
1168
|
-
await step.do(endStep, STEP_CONFIG, () =>
|
|
1253
|
+
await step.do(endStep, STEP_CONFIG, async () => {
|
|
1254
|
+
const reply = await call(bot, "unit-end", body);
|
|
1255
|
+
const answer = answerOf("unit-end", reply);
|
|
1256
|
+
if (answer.status !== 200 || answer.body.ok !== true)
|
|
1257
|
+
throw new UnreadableAnswer("unit-end", answer, "successful settlement");
|
|
1258
|
+
return reply;
|
|
1259
|
+
}),
|
|
1169
1260
|
);
|
|
1170
1261
|
endedAt = endAnswer.body.at;
|
|
1171
1262
|
}
|
|
@@ -1174,7 +1265,17 @@ async function runUnit(
|
|
|
1174
1265
|
} catch (err) {
|
|
1175
1266
|
// The walk-dies-before-unit-end path (issue 2100): the throw becomes the
|
|
1176
1267
|
// unit's ending before it fails the instance, never the bare seal.
|
|
1177
|
-
await tellStepThrew(
|
|
1268
|
+
await tellStepThrew(
|
|
1269
|
+
step,
|
|
1270
|
+
bot,
|
|
1271
|
+
prefix,
|
|
1272
|
+
tag,
|
|
1273
|
+
state.pr,
|
|
1274
|
+
last,
|
|
1275
|
+
err,
|
|
1276
|
+
row?.recovery !== undefined ? `${instanceId}:${unit}` : undefined,
|
|
1277
|
+
row?.recovery?.workflowId,
|
|
1278
|
+
);
|
|
1178
1279
|
throw err;
|
|
1179
1280
|
}
|
|
1180
1281
|
}
|
|
@@ -1496,3 +1597,54 @@ export async function runPlan(step: StepRunner, bot: CoordinatorBot, instanceId:
|
|
|
1496
1597
|
await finish(summary.outcome);
|
|
1497
1598
|
return summary;
|
|
1498
1599
|
}
|
|
1600
|
+
|
|
1601
|
+
/** A separate Workflow execution checkpoint that drives exactly one claimed
|
|
1602
|
+
* original unit. Its platform instance id is transport only: every bot call,
|
|
1603
|
+
* child tag and idempotency key continues to name `parentInstanceId + unit`. */
|
|
1604
|
+
export async function runOriginalUnitRecovery(
|
|
1605
|
+
step: StepRunner,
|
|
1606
|
+
bot: CoordinatorBot,
|
|
1607
|
+
workflowId: string,
|
|
1608
|
+
params: OriginalUnitRecoveryParams,
|
|
1609
|
+
): Promise<PlanRunSummary> {
|
|
1610
|
+
const claim = answerOf(
|
|
1611
|
+
"recover-unit",
|
|
1612
|
+
await step.do("recovery-claim", STEP_CONFIG, async () => {
|
|
1613
|
+
const reply = await call(bot, "recover-unit", {
|
|
1614
|
+
parentInstanceId: params.parentInstanceId,
|
|
1615
|
+
unit: params.unit,
|
|
1616
|
+
workflowId,
|
|
1617
|
+
});
|
|
1618
|
+
const answer = answerOf("recover-unit", reply);
|
|
1619
|
+
if (answer.status !== 200 || answer.body.ok !== true)
|
|
1620
|
+
throw new UnreadableAnswer("recover-unit", answer, "successful recovery claim");
|
|
1621
|
+
return reply;
|
|
1622
|
+
}),
|
|
1623
|
+
);
|
|
1624
|
+
if (claim.body.ok !== true) throw new UnreadableAnswer("recover-unit", claim, "not ok");
|
|
1625
|
+
const plan = readPlan(
|
|
1626
|
+
answerOf(
|
|
1627
|
+
"plan",
|
|
1628
|
+
await step.do("recovery-plan", STEP_CONFIG, () =>
|
|
1629
|
+
call(bot, "plan", { parentInstanceId: params.parentInstanceId }),
|
|
1630
|
+
),
|
|
1631
|
+
),
|
|
1632
|
+
);
|
|
1633
|
+
const row = plan.units.filter((candidate) => candidate.unit === params.unit);
|
|
1634
|
+
if (row.length !== 1 || row[0]!.recovery?.workflowId !== workflowId)
|
|
1635
|
+
throw new Error("the original unit's durable recovery claim no longer names this Workflow");
|
|
1636
|
+
const unit = row[0]!;
|
|
1637
|
+
const ending = await runUnit(
|
|
1638
|
+
step,
|
|
1639
|
+
bot,
|
|
1640
|
+
params.parentInstanceId,
|
|
1641
|
+
{ id: unit.unit, title: unit.title ?? unit.unit, slug: unit.slug, branch: unit.branch, dependsOn: unit.dependsOn },
|
|
1642
|
+
plan,
|
|
1643
|
+
);
|
|
1644
|
+
return {
|
|
1645
|
+
instance: params.parentInstanceId,
|
|
1646
|
+
...(plan.planId !== undefined ? { planId: plan.planId } : {}),
|
|
1647
|
+
units: { [params.unit]: ending.kind },
|
|
1648
|
+
outcome: isSettledOutcome(ending.kind) ? "completed" : "failed",
|
|
1649
|
+
};
|
|
1650
|
+
}
|
|
@@ -62,6 +62,7 @@ const CAUSE_OF = {
|
|
|
62
62
|
// one pipeline per thread (record 0060): the host key's claim answered thread-live
|
|
63
63
|
ship_thread_live: "request",
|
|
64
64
|
setup_failed: "system",
|
|
65
|
+
publication_ownership_unknown: "system",
|
|
65
66
|
// the click on a confirmation (confirm.ts)
|
|
66
67
|
confirmation_used: "request",
|
|
67
68
|
confirmation_expired: "request",
|
|
@@ -963,6 +963,8 @@ export type RunEvent =
|
|
|
963
963
|
type: "coordinator_tag";
|
|
964
964
|
parentInstanceId: string;
|
|
965
965
|
unit?: string;
|
|
966
|
+
transportWorkflowId?: string;
|
|
967
|
+
recovery?: import("./coordinator/contract.js").CoordinatorTag["recovery"];
|
|
966
968
|
base?: string;
|
|
967
969
|
publication?: import("./coordinator/contract.js").ExistingPrPublicationBinding;
|
|
968
970
|
seq?: number;
|
|
@@ -1176,6 +1176,10 @@ export interface UnitPipelineInput {
|
|
|
1176
1176
|
* the report's pointer at a child's write-up links its run page with it and
|
|
1177
1177
|
* names the run id without it — the machine never reads an environment. */
|
|
1178
1178
|
runPageBase?: string;
|
|
1179
|
+
/** A terminal original unit re-entered at its unchanged reviewed head. The
|
|
1180
|
+
* exact remaining lease is carried in milliseconds; the step prefix keeps
|
|
1181
|
+
* every new durable step under `<original instance>:<unit>/recovery/...`. */
|
|
1182
|
+
recovery?: { remainingMs: number; unitKey: string };
|
|
1179
1183
|
}
|
|
1180
1184
|
|
|
1181
1185
|
type Phase =
|
|
@@ -1398,8 +1402,56 @@ export function openUnitPipeline(input: UnitPipelineInput, at: number): UnitPipe
|
|
|
1398
1402
|
return resumed;
|
|
1399
1403
|
}
|
|
1400
1404
|
|
|
1405
|
+
/** Open the narrow unchanged-head recovery directly at the one authorized
|
|
1406
|
+
* child boundary. No branch, pre-check, fresh clock or round zero is invented;
|
|
1407
|
+
* the caller has already CAS-claimed the durable row and verified GitHub. */
|
|
1408
|
+
export function openRecoveredUnitPipeline(
|
|
1409
|
+
input: UnitPipelineInput,
|
|
1410
|
+
at: number,
|
|
1411
|
+
recovery: {
|
|
1412
|
+
kind: "findings" | "review";
|
|
1413
|
+
round: number;
|
|
1414
|
+
pr: PrRef;
|
|
1415
|
+
expectedHeadSha: string;
|
|
1416
|
+
reviewRunId: string;
|
|
1417
|
+
findingsRunId?: string;
|
|
1418
|
+
findings?: Finding[];
|
|
1419
|
+
},
|
|
1420
|
+
): UnitPipelineState {
|
|
1421
|
+
const base: UnitPipelineState = {
|
|
1422
|
+
input,
|
|
1423
|
+
startedAt: at,
|
|
1424
|
+
clock: at,
|
|
1425
|
+
phase: { at: "ended" },
|
|
1426
|
+
reviewRounds: recovery.round,
|
|
1427
|
+
// These counters name actions inside this new recovery Workflow's distinct
|
|
1428
|
+
// `/recovery/` step namespace; they are not lifetime-unit counters or
|
|
1429
|
+
// budget inputs. The lifetime review-round count is the durable `round`
|
|
1430
|
+
// above, while wall-clock and dollar enforcement come from the claim.
|
|
1431
|
+
rebaseAttempts: 0,
|
|
1432
|
+
reviewRestarts: 0,
|
|
1433
|
+
pr: recovery.pr,
|
|
1434
|
+
lastReviewHead: recovery.expectedHeadSha,
|
|
1435
|
+
// Legacy terminal rows do not carry a complete lifetime category split or
|
|
1436
|
+
// cumulative dollars. Recovery therefore measures only this checkpoint;
|
|
1437
|
+
// admission refuses a cost-capped row rather than resetting an enforced
|
|
1438
|
+
// total, and reports label this split as checkpoint-local.
|
|
1439
|
+
spentMs: { coding: 0, review: 0, waiting: 0 },
|
|
1440
|
+
spendUsd: 0,
|
|
1441
|
+
findingsByRound: recovery.kind === "findings" ? { [recovery.round]: recovery.findings ?? [] } : {},
|
|
1442
|
+
humanAnswersByRound: {},
|
|
1443
|
+
dispositionsByRound: {},
|
|
1444
|
+
reviewRunByRound:
|
|
1445
|
+
recovery.findingsRunId !== undefined
|
|
1446
|
+
? { [recovery.round - 1]: recovery.reviewRunId }
|
|
1447
|
+
: { [recovery.round]: recovery.reviewRunId },
|
|
1448
|
+
findingsRunByRound: recovery.findingsRunId !== undefined ? { [recovery.round - 1]: recovery.findingsRunId } : {},
|
|
1449
|
+
};
|
|
1450
|
+
return enterRound(base, { index: recovery.round, kind: recovery.kind }).state;
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1401
1453
|
const deadlineAt = (s: UnitPipelineState) =>
|
|
1402
|
-
s.startedAt + (s.input.session?.resume?.leaseMs ?? s.input.caps.maxMinutes * MIN);
|
|
1454
|
+
s.startedAt + (s.input.recovery?.remainingMs ?? s.input.session?.resume?.leaseMs ?? s.input.caps.maxMinutes * MIN);
|
|
1403
1455
|
const remainingMs = (s: UnitPipelineState) => deadlineAt(s) - s.clock;
|
|
1404
1456
|
|
|
1405
1457
|
/** The next slice of a wait: a chunk; the remainder when less is left before
|
|
@@ -1431,7 +1483,8 @@ export const stepPrefixOf = (unit: string, session: LeaseSegmentProgress | undef
|
|
|
1431
1483
|
const segment = session !== undefined && session.segment > 1 ? `${unit}/s${session.segment}` : unit;
|
|
1432
1484
|
return session?.resume !== undefined ? `${segment}/r${session.resume.attempt}` : segment;
|
|
1433
1485
|
};
|
|
1434
|
-
const stepPrefix = (s: UnitPipelineState) =>
|
|
1486
|
+
const stepPrefix = (s: UnitPipelineState) =>
|
|
1487
|
+
s.input.recovery !== undefined ? `${s.input.unit.id}/recovery` : stepPrefixOf(s.input.unit.id, s.input.session);
|
|
1435
1488
|
const roundStep = (s: UnitPipelineState, round: RoundRef) =>
|
|
1436
1489
|
`${stepPrefix(s)}/${round.index}/${round.kind}${round.attempt !== undefined ? `/a${round.attempt}` : ""}`;
|
|
1437
1490
|
|
|
@@ -1592,7 +1645,9 @@ export function nextAction(s: UnitPipelineState): CoordinatorAction {
|
|
|
1592
1645
|
return {
|
|
1593
1646
|
type: "pr-check",
|
|
1594
1647
|
step: `${roundStep(s, p.round)}/pr-check`,
|
|
1595
|
-
...(p.dead !== undefined || p.recover === true
|
|
1648
|
+
...(p.dead !== undefined || p.recover === true || p.findingsReady === true
|
|
1649
|
+
? { recover: { runId: p.runId } }
|
|
1650
|
+
: {}),
|
|
1596
1651
|
// The adopted pull request rides the check so the bot can follow it
|
|
1597
1652
|
// when nothing heads the unit's branch (issue 1799).
|
|
1598
1653
|
...(s.pr !== undefined ? { pr: s.pr.number } : {}),
|
|
@@ -3610,9 +3665,14 @@ function writeUpPointer(s: UnitPipelineState, kind: RoundKind, runId: string | u
|
|
|
3610
3665
|
}
|
|
3611
3666
|
|
|
3612
3667
|
/** How the budget went, in the card's words: coding, review, waiting minutes. */
|
|
3613
|
-
function budgetSplitLine(
|
|
3668
|
+
function budgetSplitLine(s: UnitPipelineState, spent: ShipBudgetSpent): string {
|
|
3614
3669
|
const min = (ms: number) => Math.round(ms / MIN);
|
|
3615
|
-
|
|
3670
|
+
const recoveryMinutes = s.input.recovery === undefined ? undefined : min(s.input.recovery.remainingMs);
|
|
3671
|
+
const label =
|
|
3672
|
+
recoveryMinutes === undefined
|
|
3673
|
+
? `Budget split (${s.input.caps.maxMinutes} min)`
|
|
3674
|
+
: `Recovery checkpoint split (${recoveryMinutes} min carried time budget; earlier activity is not included)`;
|
|
3675
|
+
return `${label}: coding ${min(spent.coding)} min, review ${min(spent.review)} min, waiting ${min(spent.waiting)} min.`;
|
|
3616
3676
|
}
|
|
3617
3677
|
|
|
3618
3678
|
/** The cap report's declined-vs-unaddressed split over the last review round's findings. */
|
|
@@ -3738,6 +3798,8 @@ function renderUnitReportWithWake(
|
|
|
3738
3798
|
const restart = s.input.generated
|
|
3739
3799
|
? `then start ship again with ${prUrl ?? "the original task"}`
|
|
3740
3800
|
: "then start this plan again";
|
|
3801
|
+
if (s.input.recovery !== undefined)
|
|
3802
|
+
return `The recovery checkpoint for \`${s.input.recovery.unitKey}\` is terminal and its evidence is consumed; no replacement pipeline is available.`;
|
|
3741
3803
|
const blocked = s.input.generated ? "" : "The unit's dependents in this plan stay blocked. ";
|
|
3742
3804
|
return `${blocked}Next action: ${reconcile}, ${restart}.`;
|
|
3743
3805
|
};
|
|
@@ -3900,14 +3962,14 @@ function renderUnitReportWithWake(
|
|
|
3900
3962
|
if (!shows(verbosity, "verbose")) return `🧢 Out of budget — no approval after ${rounds}.${prLine}`;
|
|
3901
3963
|
return join([
|
|
3902
3964
|
`🧢 Ship stopped at a cap: the remaining pipeline time (~${Math.max(0, Math.round(e.remainingMs / MIN))} min of the ${s.input.caps.maxMinutes}-minute budget) cannot hold another round${e.refused ? ` (the ${e.refused.round} round would get ${e.refused.minutes} min, under its floor of ${e.refused.floor})` : ""} — no approval after ${rounds}.${prLine}`,
|
|
3903
|
-
budgetSplitLine(e.spent
|
|
3965
|
+
budgetSplitLine(s, e.spent),
|
|
3904
3966
|
splitReport(s),
|
|
3905
3967
|
nextAction(),
|
|
3906
3968
|
]);
|
|
3907
3969
|
case "review_pending":
|
|
3908
3970
|
return join([
|
|
3909
3971
|
`⏳ Review pending: the coding child shipped ${e.pr.url}${e.headSha !== undefined ? ` (head \`${e.headSha.slice(0, 7)}\`)` : ""} but the remaining pipeline time cannot hold the review round — the work stands, only the review is missing. The next pipeline starts at the review round while the pull request still heads at the child's own last push.`,
|
|
3910
|
-
aside(budgetSplitLine(e.spent
|
|
3972
|
+
aside(budgetSplitLine(s, e.spent)),
|
|
3911
3973
|
aside(nextAction()),
|
|
3912
3974
|
]);
|
|
3913
3975
|
case "stopped":
|
|
@@ -3936,7 +3998,9 @@ function renderUnitReportWithWake(
|
|
|
3936
3998
|
observed !== undefined
|
|
3937
3999
|
? `Saved-work fact: The run ended at \`${observed}\`, but remote state was not checked; Switchboard cannot claim that commit is on the branch or pull request.`
|
|
3938
4000
|
: "Saved-work fact: The run did not record a final commit, and remote state was not checked.",
|
|
3939
|
-
|
|
4001
|
+
s.input.recovery !== undefined
|
|
4002
|
+
? nextAction()
|
|
4003
|
+
: "Next action: a new findings run must record every disposition, the updated pull request description and its final commit before remote reconciliation.",
|
|
3940
4004
|
]);
|
|
3941
4005
|
if (e.findingsStop === "unfinished")
|
|
3942
4006
|
return join([
|
|
@@ -3944,24 +4008,32 @@ function renderUnitReportWithWake(
|
|
|
3944
4008
|
observed !== undefined
|
|
3945
4009
|
? `Saved-work fact: the stopped run recorded \`${observed}\`, but Switchboard did not verify it on the remote branch or pull request.`
|
|
3946
4010
|
: "Saved-work fact: no completed commit was recorded, and remote state was not checked.",
|
|
3947
|
-
|
|
4011
|
+
s.input.recovery !== undefined
|
|
4012
|
+
? nextAction()
|
|
4013
|
+
: "Next action: start a new findings run; review can begin only after it completes every required result and Switchboard verifies its exact remote head.",
|
|
3948
4014
|
]);
|
|
3949
4015
|
if (e.findingsStop === "missing_remote")
|
|
3950
4016
|
return join([
|
|
3951
4017
|
`⚠️ Review did not restart: The completed changes ended at \`${observed}\`, but no open pull request was found for \`${branch}\`.`,
|
|
3952
4018
|
`Saved-work fact: The run recorded completed commit \`${observed}\`; Switchboard did not verify that commit on the remote branch or a pull request.`,
|
|
3953
|
-
|
|
4019
|
+
s.input.recovery !== undefined
|
|
4020
|
+
? nextAction()
|
|
4021
|
+
: "Next action: reconcile the branch and open pull request, then start ship again so it can verify the exact head before review.",
|
|
3954
4022
|
]);
|
|
3955
4023
|
if (e.findingsStop === "head_mismatch")
|
|
3956
4024
|
return join([
|
|
3957
4025
|
`⚠️ Review did not restart: The completed changes ended at \`${observed}\`, but the pull request is at \`${e.remoteHead?.slice(0, 7)}\` (${s.pr?.url}).`,
|
|
3958
4026
|
"Saved-work fact: the pull request's current commit is verified, but the completed changes are not verified there. No review ran on either commit.",
|
|
3959
|
-
|
|
4027
|
+
s.input.recovery !== undefined
|
|
4028
|
+
? nextAction()
|
|
4029
|
+
: "Next action: reconcile the pull request to the completed commit, then start ship again so it can verify that exact head before review.",
|
|
3960
4030
|
]);
|
|
3961
4031
|
return join([
|
|
3962
4032
|
`⚠️ Review did not restart: The pull request exists (${s.pr?.url}), but its exact head could not be verified.`,
|
|
3963
4033
|
`Saved-work fact: the run recorded completed commit \`${observed}\`; Switchboard did not start review for \`${observed}\` and cannot claim the pull request contains it.`,
|
|
3964
|
-
|
|
4034
|
+
s.input.recovery !== undefined
|
|
4035
|
+
? nextAction()
|
|
4036
|
+
: "Next action: retry ship when the pull request state is readable so it can verify the exact head before review.",
|
|
3965
4037
|
]);
|
|
3966
4038
|
}
|
|
3967
4039
|
if (!shows(verbosity, "verbose")) return `⚠️ Aborted after ${rounds}: ${e.reason}${prLine}`;
|
|
@@ -3987,7 +4059,7 @@ function renderUnitReportWithWake(
|
|
|
3987
4059
|
durableWake
|
|
3988
4060
|
? `🔁 The unit's budget ran out with the unit unfinished — ${e.line}. A reply in this thread is recorded as its continuation action; ${e.renewalsLeft} renewal${e.renewalsLeft === 1 ? "" : "s"} remain${e.spendUsd !== null ? `, $${e.spendUsd.toFixed(2)} spent so far` : ""}.`
|
|
3989
4061
|
: `🔁 The unit's budget ran out with the unit unfinished — ${e.line}. A fresh ${s.input.caps.maxMinutes}-minute budget opens in this thread${e.from !== undefined ? ` from \`${e.from.slice(0, 7)}\`` : ""}, with the last run's write-up as its request; ${e.renewalsLeft} renewal${e.renewalsLeft === 1 ? "" : "s"} remain${e.spendUsd !== null ? `, $${e.spendUsd.toFixed(2)} spent so far` : ""}.`,
|
|
3990
|
-
aside(budgetSplitLine(e.spent
|
|
4062
|
+
aside(budgetSplitLine(s, e.spent)),
|
|
3991
4063
|
]);
|
|
3992
4064
|
case "transient":
|
|
3993
4065
|
if (!shows(verbosity, "verbose"))
|
|
@@ -4008,7 +4080,11 @@ function renderUnitReportWithWake(
|
|
|
4008
4080
|
nextAction(),
|
|
4009
4081
|
]);
|
|
4010
4082
|
case "interrupted":
|
|
4011
|
-
return join([
|
|
4083
|
+
return join([
|
|
4084
|
+
shipInterruptedNote(prUrl, e.cause, durableWake),
|
|
4085
|
+
checkpointLine(e.checkpoint),
|
|
4086
|
+
s.input.recovery !== undefined ? nextAction(e.checkpoint) : undefined,
|
|
4087
|
+
]);
|
|
4012
4088
|
case "idle_expired":
|
|
4013
4089
|
return "⌛ Idle expired: no reply continued this unit before its idle window closed.";
|
|
4014
4090
|
case "refused":
|
|
@@ -4019,6 +4095,6 @@ function renderUnitReportWithWake(
|
|
|
4019
4095
|
case "idle":
|
|
4020
4096
|
// Keep the old kind's outcome at this copy's level, but render the
|
|
4021
4097
|
// indexed wake's durable reply action instead of a terminal restart.
|
|
4022
|
-
return renderUnitReportWithWake({ ...s, ending: e.idled }, facts, verbosity,
|
|
4098
|
+
return renderUnitReportWithWake({ ...s, ending: e.idled }, facts, verbosity, s.input.recovery === undefined);
|
|
4023
4099
|
}
|
|
4024
4100
|
}
|