@coreplane/switchboard 1.261.4 → 1.262.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/deploy/cloudflare/coordinator.ts +18 -1
- package/dist/assets/deploy/cloudflare/worker.ts +4 -1
- package/dist/assets/deploy/cloudflare-memory/worker.ts +50 -2
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +1 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/core/coordinator/contract.ts +137 -4
- package/dist/assets/src/core/coordinator/driver.ts +192 -40
- package/dist/assets/src/core/refusal.ts +1 -0
- package/dist/assets/src/core/runEvents.ts +2 -0
- package/dist/assets/src/core/ship/coordinator.ts +91 -15
- package/dist/cli.js +786 -41
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -3967,7 +3967,7 @@ function matchDispositions(findings, dispositions) {
|
|
|
3967
3967
|
const dropped = [...new Set(dispositions.filter((d) => !issued.has(d.findingId)).map((d) => d.findingId))];
|
|
3968
3968
|
return { matched, dropped };
|
|
3969
3969
|
}
|
|
3970
|
-
var MIN, PLAN_ID_PATTERN, PLAN_BRANCH, UNIT_REF, SLUG_MAX, INSTANCE_ID_MAX, WAIT_MARGIN_MS, WAIT_CHUNK_MS, MERGE_WAIT_CHUNK_MS, BUSY_RETRY_MS;
|
|
3970
|
+
var MIN, PLAN_ID_PATTERN, PLAN_BRANCH, UNIT_REF, SLUG_MAX, INSTANCE_ID_MAX, WAIT_MARGIN_MS, WAIT_CHUNK_MS, MERGE_WAIT_CHUNK_MS, BUSY_RETRY_MS, stepPrefixOf;
|
|
3971
3971
|
var init_coordinator = __esm({
|
|
3972
3972
|
"../../src/core/ship/coordinator.ts"() {
|
|
3973
3973
|
"use strict";
|
|
@@ -3987,6 +3987,10 @@ var init_coordinator = __esm({
|
|
|
3987
3987
|
WAIT_CHUNK_MS = SHIP_WAIT.chunkMinutes * MIN;
|
|
3988
3988
|
MERGE_WAIT_CHUNK_MS = SHIP_WAIT.mergeChunkMinutes * MIN;
|
|
3989
3989
|
BUSY_RETRY_MS = SHIP_WAIT.busyRetryMinutes * MIN;
|
|
3990
|
+
stepPrefixOf = (unit, session) => {
|
|
3991
|
+
const segment = session !== void 0 && session.segment > 1 ? `${unit}/s${session.segment}` : unit;
|
|
3992
|
+
return session?.resume !== void 0 ? `${segment}/r${session.resume.attempt}` : segment;
|
|
3993
|
+
};
|
|
3990
3994
|
}
|
|
3991
3995
|
});
|
|
3992
3996
|
|
|
@@ -5804,6 +5808,13 @@ function isCoordinatorUnit(v) {
|
|
|
5804
5808
|
(x) => isObject(x) && isFinite(x.index) && isText(x.agent) && isText(x.outcome) && isFinite(x.at) && (x.gate === void 0 || isRoundGate(x.gate))
|
|
5805
5809
|
))
|
|
5806
5810
|
return false;
|
|
5811
|
+
if (r.recovery !== void 0 && !(isObject(r.recovery) && (r.recovery.kind === "findings" || r.recovery.kind === "review") && typeof r.recovery.round === "number" && Number.isInteger(r.recovery.round) && r.recovery.round >= 1 && typeof r.recovery.expectedHeadSha === "string" && /^[0-9a-f]{40}$/i.test(r.recovery.expectedHeadSha) && isFinite(r.recovery.remainingMs) && r.recovery.remainingMs > 0 && isFinite(r.recovery.claimedAt) && typeof r.recovery.step === "string" && STEP_NAME_PATTERN.test(r.recovery.step) && isText(r.recovery.reviewRunId) && (r.recovery.findingsRunId === void 0 || isText(r.recovery.findingsRunId)) && (r.recovery.findingsKey === void 0 || isText(r.recovery.findingsKey)) && (r.recovery.findingsRunId === void 0 && r.recovery.findingsKey === void 0 || r.recovery.kind === "review" && r.recovery.findingsRunId !== void 0 && r.recovery.findingsKey !== void 0) && (r.recovery.findings === void 0 || Array.isArray(r.recovery.findings) && r.recovery.findings.every(isFindingShape)) && isObject(r.recovery.previousEnding) && isText(r.recovery.previousEnding.kind) && typeof r.recovery.previousEnding.report === "string" && isFinite(r.recovery.previousEnding.at) && typeof r.recovery.workflowId === "string" && INSTANCE_ID_PATTERN.test(r.recovery.workflowId) && isFinite(r.recovery.deadlineAt) && isText(r.recovery.reviewKey)))
|
|
5812
|
+
return false;
|
|
5813
|
+
if (r.recoveryReceipt !== void 0 && (!isObject(r.recoveryReceipt) || !isText(r.recoveryReceipt.reviewRunId) || typeof r.recoveryReceipt.workflowId !== "string" || !INSTANCE_ID_PATTERN.test(r.recoveryReceipt.workflowId) || !isFinite(r.recoveryReceipt.at)))
|
|
5814
|
+
return false;
|
|
5815
|
+
if (r.recoveryHold !== void 0 && (!isObject(r.recoveryHold) || !(r.recoveryHold.cause === "human" && isHumanGatePending(r.recoveryHold.gate) || r.recoveryHold.cause === "draft" && isObject(r.recoveryHold.pr) && Number.isInteger(r.recoveryHold.pr.number) && r.recoveryHold.pr.number > 0 && isText(r.recoveryHold.pr.url))))
|
|
5816
|
+
return false;
|
|
5817
|
+
if (r.recovery !== void 0 && (r.idle !== void 0 || r.ending !== void 0)) return false;
|
|
5807
5818
|
if (r.ending !== void 0 && !(isObject(r.ending) && isText(r.ending.kind) && isText(r.ending.report, MAX_REPORT) && isFinite(r.ending.at) && (r.ending.cause === void 0 || isText(r.ending.cause, 64)) && (r.ending.step === void 0 || typeof r.ending.step === "string" && STEP_NAME_PATTERN.test(r.ending.step)) && (r.ending.round === void 0 || typeof r.ending.round === "number" && Number.isInteger(r.ending.round) && r.ending.round >= 0)))
|
|
5808
5819
|
return false;
|
|
5809
5820
|
if (r.startedAt !== void 0 && !isFinite(r.startedAt)) return false;
|
|
@@ -5850,7 +5861,7 @@ async function sendChildSignal(workflow, signal2) {
|
|
|
5850
5861
|
if (!workflow) return { kind: "no-binding", instance };
|
|
5851
5862
|
const type = kind === "interrupted" ? childInterruptedEventType(runId3) : childResumedEventType(runId3);
|
|
5852
5863
|
try {
|
|
5853
|
-
const handle2 = await workflow.get(instance);
|
|
5864
|
+
const handle2 = await workflow.get(signal2.transportWorkflowId ?? instance);
|
|
5854
5865
|
await handle2.sendEvent({ type, payload: { runId: runId3, kind, reason: reason2, at, parentInstanceId: instance } });
|
|
5855
5866
|
return { kind: "sent", instance, type };
|
|
5856
5867
|
} catch (err2) {
|
|
@@ -15534,6 +15545,7 @@ var init_refusal = __esm({
|
|
|
15534
15545
|
// one pipeline per thread (record 0060): the host key's claim answered thread-live
|
|
15535
15546
|
ship_thread_live: "request",
|
|
15536
15547
|
setup_failed: "system",
|
|
15548
|
+
publication_ownership_unknown: "system",
|
|
15537
15549
|
// the click on a confirmation (confirm.ts)
|
|
15538
15550
|
confirmation_used: "request",
|
|
15539
15551
|
confirmation_expired: "request",
|
|
@@ -34693,6 +34705,9 @@ var init_instanceStore = __esm({
|
|
|
34693
34705
|
async listUnits(_instanceId) {
|
|
34694
34706
|
return [];
|
|
34695
34707
|
}
|
|
34708
|
+
async listActiveRecoveries() {
|
|
34709
|
+
return [];
|
|
34710
|
+
}
|
|
34696
34711
|
async reserveDecisionRecord(_repo, _taskKey, _claimed, _existing) {
|
|
34697
34712
|
return { ok: false, reason: "unavailable" };
|
|
34698
34713
|
}
|
|
@@ -34783,6 +34798,15 @@ var init_instanceStore = __esm({
|
|
|
34783
34798
|
throw new Error("coordinator store /runs/coordinator/units/list: the answer is not a list of unit rows");
|
|
34784
34799
|
return d.units;
|
|
34785
34800
|
}
|
|
34801
|
+
async listActiveRecoveries() {
|
|
34802
|
+
const r = await this.post("/runs/coordinator/units/list-active-recoveries", {});
|
|
34803
|
+
const d = r.data;
|
|
34804
|
+
if (!Array.isArray(d.units) || !d.units.every(isCoordinatorUnit) || d.units.some((unit) => unit.recovery === void 0))
|
|
34805
|
+
throw new Error(
|
|
34806
|
+
"coordinator store /runs/coordinator/units/list-active-recoveries: the answer is not a recovery list"
|
|
34807
|
+
);
|
|
34808
|
+
return d.units;
|
|
34809
|
+
}
|
|
34786
34810
|
async reserveDecisionRecord(repo, taskKey, claimed2, existing) {
|
|
34787
34811
|
const r = await this.post("/runs/decision-record/reserve", {
|
|
34788
34812
|
repo,
|
|
@@ -46184,6 +46208,8 @@ async function claimRun(deps, ctx) {
|
|
|
46184
46208
|
type: "coordinator_tag",
|
|
46185
46209
|
parentInstanceId: coordinator.parentInstanceId,
|
|
46186
46210
|
...unit !== void 0 ? { unit } : {},
|
|
46211
|
+
...coordinator.transportWorkflowId !== void 0 ? { transportWorkflowId: coordinator.transportWorkflowId } : {},
|
|
46212
|
+
...coordinator.recovery !== void 0 ? { recovery: coordinator.recovery } : {},
|
|
46187
46213
|
...coordinator.base !== void 0 ? { base: coordinator.base } : {},
|
|
46188
46214
|
...coordinator.publication !== void 0 ? { publication: coordinator.publication } : {},
|
|
46189
46215
|
at: clock4()
|
|
@@ -54984,7 +55010,7 @@ function shimAddress(opts) {
|
|
|
54984
55010
|
};
|
|
54985
55011
|
return { base, bearer };
|
|
54986
55012
|
}
|
|
54987
|
-
async function createInstanceViaShim(opts, id) {
|
|
55013
|
+
async function createInstanceViaShim(opts, id, params = {}) {
|
|
54988
55014
|
const at = shimAddress(opts);
|
|
54989
55015
|
if ("reason" in at) return { kind: "unanswered", reason: at.reason };
|
|
54990
55016
|
const fetchImpl = opts.fetch ?? fetch;
|
|
@@ -54992,7 +55018,7 @@ async function createInstanceViaShim(opts, id) {
|
|
|
54992
55018
|
const res = await fetchImpl(`${at.base}${COORDINATOR_INSTANCES_PATH}`, {
|
|
54993
55019
|
method: "POST",
|
|
54994
55020
|
headers: { authorization: `Bearer ${at.bearer}`, "content-type": "application/json" },
|
|
54995
|
-
body: JSON.stringify({ id, params
|
|
55021
|
+
body: JSON.stringify({ id, params }),
|
|
54996
55022
|
signal: AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS)
|
|
54997
55023
|
});
|
|
54998
55024
|
return readCreateInstanceAnswer(res.status, await res.text().catch(() => ""));
|
|
@@ -55052,11 +55078,51 @@ var init_instancesClient = __esm({
|
|
|
55052
55078
|
});
|
|
55053
55079
|
|
|
55054
55080
|
// ../../src/core/dispatch/ship.ts
|
|
55081
|
+
function parseOriginalUnitRecoveryRequest(text) {
|
|
55082
|
+
const match = /^recover\s+unit\s+(\S+)\s*$/i.exec(text.trim());
|
|
55083
|
+
return match ? parseUnitKey(match[1]) : void 0;
|
|
55084
|
+
}
|
|
55055
55085
|
async function runShipBranch(deps, msg, io, ctx) {
|
|
55056
55086
|
const { agent, profile, card, directives, history, repoCtx, label, ending, trace, closeLines, refuse: refuse3, doneLines } = ctx;
|
|
55057
55087
|
const root = trace.root;
|
|
55058
55088
|
const clock4 = deps.clock ?? systemClock;
|
|
55059
55089
|
const shell = createCardShell({ label, startedAt: ctx.startedAt, now: clock4 });
|
|
55090
|
+
const recovery = parseOriginalUnitRecoveryRequest(directives.text);
|
|
55091
|
+
if (recovery === void 0 && namesOriginalUnitRecovery(directives.text)) {
|
|
55092
|
+
const reason2 = "original-unit recovery must be `recover unit <instanceId>:<unit>`";
|
|
55093
|
+
await refuse3(
|
|
55094
|
+
refusalOf("setup_failed", reason2),
|
|
55095
|
+
() => card.done(shell.close({ kind: "refused", icon: "\u{1F6AB}", reason: reason2, ...closeLines(clock4(), false) }))
|
|
55096
|
+
);
|
|
55097
|
+
return { hostedLive: false };
|
|
55098
|
+
}
|
|
55099
|
+
if (recovery !== void 0) {
|
|
55100
|
+
const answer = await root.span(
|
|
55101
|
+
"dispatch.ship_recover_original_unit",
|
|
55102
|
+
() => deps.recoverOriginalUnit === void 0 ? Promise.resolve({ status: 503, body: { error: "original-unit recovery is unavailable" } }) : deps.recoverOriginalUnit(recovery, { userId: msg.userId, threadKey: msg.threadKey })
|
|
55103
|
+
);
|
|
55104
|
+
if (answer.status !== 200) {
|
|
55105
|
+
const reason2 = typeof answer.body.error === "string" ? answer.body.error : "original-unit recovery was refused";
|
|
55106
|
+
await refuse3(
|
|
55107
|
+
refusalOf("setup_failed", reason2),
|
|
55108
|
+
() => card.done(shell.close({ kind: "refused", icon: "\u{1F6AB}", reason: reason2, ...closeLines(clock4(), false) }))
|
|
55109
|
+
);
|
|
55110
|
+
return { hostedLive: false };
|
|
55111
|
+
}
|
|
55112
|
+
const workflowId = typeof answer.body.workflowId === "string" ? answer.body.workflowId : "unknown";
|
|
55113
|
+
const outcome2 = answer.body.outcome === "already_started" ? "already running" : answer.body.outcome === "indeterminate" ? "has an indeterminate start; its claim remains held for same-checkpoint replay" : "started";
|
|
55114
|
+
const text = `Original unit \`${recovery.instanceId}:${recovery.unit}\` recovery ${outcome2} as durable checkpoint \`${workflowId}\`.`;
|
|
55115
|
+
await replyAck(io, ctx.verbosity, text);
|
|
55116
|
+
await card.done(
|
|
55117
|
+
shell.close({
|
|
55118
|
+
kind: "done",
|
|
55119
|
+
icon: answer.body.outcome === "indeterminate" ? "\u26A0\uFE0F" : "\u2705",
|
|
55120
|
+
detail: text,
|
|
55121
|
+
...closeLines(clock4(), true)
|
|
55122
|
+
})
|
|
55123
|
+
);
|
|
55124
|
+
return { hostedLive: false };
|
|
55125
|
+
}
|
|
55060
55126
|
const listSlugs = deps.residentSlugs ?? residentSlugsLister(deps.config.config.execution?.resident);
|
|
55061
55127
|
const repoCandidates = repoCtx.repo ? void 0 : await listSlugs?.().catch(() => void 0);
|
|
55062
55128
|
const pre = await root.span(
|
|
@@ -55465,6 +55531,7 @@ function handOffAck(outcome, verbosity2) {
|
|
|
55465
55531
|
return outcome.instanceId !== void 0 && shows(verbosity2, "debug") ? `${outcome.reply}
|
|
55466
55532
|
\u2022 runner instance \`${outcome.instanceId}\`` : outcome.reply;
|
|
55467
55533
|
}
|
|
55534
|
+
var namesOriginalUnitRecovery;
|
|
55468
55535
|
var init_ship = __esm({
|
|
55469
55536
|
"../../src/core/dispatch/ship.ts"() {
|
|
55470
55537
|
"use strict";
|
|
@@ -55477,6 +55544,7 @@ var init_ship = __esm({
|
|
|
55477
55544
|
init_budgets();
|
|
55478
55545
|
init_instancesClient();
|
|
55479
55546
|
init_instanceStore();
|
|
55547
|
+
init_contract2();
|
|
55480
55548
|
init_shipPipeline();
|
|
55481
55549
|
init_preflight();
|
|
55482
55550
|
init_runEvents();
|
|
@@ -55495,6 +55563,7 @@ var init_ship = __esm({
|
|
|
55495
55563
|
init_reply();
|
|
55496
55564
|
init_verbosity();
|
|
55497
55565
|
init_reply();
|
|
55566
|
+
namesOriginalUnitRecovery = (text) => /^recover\s+unit(?:\s|$)/i.test(text.trim());
|
|
55498
55567
|
}
|
|
55499
55568
|
});
|
|
55500
55569
|
|
|
@@ -55592,9 +55661,13 @@ function carriedCoordinatorTag(row2, events) {
|
|
|
55592
55661
|
const tag = events.find((e) => e.type === "coordinator_tag");
|
|
55593
55662
|
const base = tag?.type === "coordinator_tag" ? tag.base : void 0;
|
|
55594
55663
|
const publication = tag?.type === "coordinator_tag" ? tag.publication : void 0;
|
|
55664
|
+
const transportWorkflowId = tag?.type === "coordinator_tag" ? tag.transportWorkflowId : void 0;
|
|
55665
|
+
const recovery = tag?.type === "coordinator_tag" ? tag.recovery : void 0;
|
|
55595
55666
|
return {
|
|
55596
55667
|
parentInstanceId,
|
|
55597
55668
|
idempotencyKey,
|
|
55669
|
+
...transportWorkflowId !== void 0 ? { transportWorkflowId } : {},
|
|
55670
|
+
...recovery !== void 0 ? { recovery } : {},
|
|
55598
55671
|
...base !== void 0 ? { base } : {},
|
|
55599
55672
|
...publication !== void 0 ? { publication } : {}
|
|
55600
55673
|
};
|
|
@@ -55617,6 +55690,7 @@ async function announceChildRoll(ctx) {
|
|
|
55617
55690
|
const sent = await sendChildSignal(workflow, {
|
|
55618
55691
|
runId: runId3,
|
|
55619
55692
|
parentInstanceId: coordinator.parentInstanceId,
|
|
55693
|
+
...coordinator.transportWorkflowId !== void 0 ? { transportWorkflowId: coordinator.transportWorkflowId } : {},
|
|
55620
55694
|
kind,
|
|
55621
55695
|
reason: reason2,
|
|
55622
55696
|
at
|
|
@@ -55689,6 +55763,7 @@ async function recordRestartDeath(ctx) {
|
|
|
55689
55763
|
const sent = await sendChildSignal(workflow, {
|
|
55690
55764
|
runId: closed.id,
|
|
55691
55765
|
parentInstanceId: coordinator.parentInstanceId,
|
|
55766
|
+
...coordinator.transportWorkflowId !== void 0 ? { transportWorkflowId: coordinator.transportWorkflowId } : {},
|
|
55692
55767
|
kind: "interrupted",
|
|
55693
55768
|
reason: summary,
|
|
55694
55769
|
at
|
|
@@ -56070,14 +56145,49 @@ async function answerUnitOwnedThread(deps, ctx) {
|
|
|
56070
56145
|
if (!appended.ok) return "route-fresh";
|
|
56071
56146
|
try {
|
|
56072
56147
|
if (deps.workflow === void 0) throw new Error("no workflow sender in this process");
|
|
56073
|
-
const
|
|
56148
|
+
const workflowId = owner.unit.recovery?.workflowId ?? owner.instanceId;
|
|
56149
|
+
const handle2 = await deps.workflow.get(workflowId);
|
|
56074
56150
|
await handle2.sendEvent({ type: unitNudgeEventType(key), payload: {} });
|
|
56075
56151
|
} catch (err2) {
|
|
56076
56152
|
const readStatus = deps.fetchCoordinatorInstanceStatus ?? ((id) => fetchInstanceStatusViaShim(processShimOptions(), id));
|
|
56077
|
-
const
|
|
56153
|
+
const workflowId = owner.unit.recovery?.workflowId ?? owner.instanceId;
|
|
56154
|
+
const status3 = await readStatus(workflowId);
|
|
56078
56155
|
const gone = status3.kind === "absent" || status3.kind === "status" && ["complete", "errored", "terminated"].includes(status3.status);
|
|
56079
56156
|
if (gone) {
|
|
56080
56157
|
const why = status3.kind === "absent" ? "no such instance" : status3.status;
|
|
56158
|
+
if (owner.unit.recovery !== void 0) {
|
|
56159
|
+
const recovery = owner.unit.recovery;
|
|
56160
|
+
const recoveryInstance = await store.get(owner.instanceId).catch(() => null);
|
|
56161
|
+
const { recovery: _recovery, ...withoutRecovery } = owner.unit;
|
|
56162
|
+
const closed = {
|
|
56163
|
+
...withoutRecovery,
|
|
56164
|
+
recoveryReceipt: { reviewRunId: recovery.reviewRunId, workflowId: recovery.workflowId, at },
|
|
56165
|
+
ending: {
|
|
56166
|
+
kind: "terminated",
|
|
56167
|
+
report: `the original-unit recovery checkpoint is gone (${why}); no replacement pipeline was started`,
|
|
56168
|
+
at
|
|
56169
|
+
}
|
|
56170
|
+
};
|
|
56171
|
+
const replaced = await store.compareAndReplaceUnit(owner.unit, closed).catch(() => void 0);
|
|
56172
|
+
if (replaced?.ok !== true) {
|
|
56173
|
+
await replyAck(
|
|
56174
|
+
io,
|
|
56175
|
+
ctx.verbosity,
|
|
56176
|
+
`\u{1F4CC} Noted for unit ${owner.unit.unit} (\`${unitKey}\`): its recovery checkpoint is gone, but the durable claim could not be closed safely \u2014 your message remains queued and no replacement ran.`
|
|
56177
|
+
);
|
|
56178
|
+
return "acked";
|
|
56179
|
+
}
|
|
56180
|
+
if (recoveryInstance !== null && owner.unit.pr !== void 0)
|
|
56181
|
+
deps.runnerOwnership?.release(recoveryInstance.repo, owner.unit.pr.number, {
|
|
56182
|
+
instanceId: owner.instanceId,
|
|
56183
|
+
unit: owner.unit.unit
|
|
56184
|
+
});
|
|
56185
|
+
await store.markConsumed(key, [appended.seq], `recovery-terminal:${recovery.workflowId}`).catch(() => void 0);
|
|
56186
|
+
await io.reply(
|
|
56187
|
+
`\u26A0\uFE0F Unit ${owner.unit.unit}'s original-unit recovery checkpoint \`${recovery.workflowId}\` is gone (${why}). The claim is closed; this message did not start replacement work.`
|
|
56188
|
+
);
|
|
56189
|
+
return "acked";
|
|
56190
|
+
}
|
|
56081
56191
|
await store.putUnits([
|
|
56082
56192
|
{
|
|
56083
56193
|
...owner.unit,
|
|
@@ -56166,6 +56276,8 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56166
56276
|
const ended = { status: "completed" };
|
|
56167
56277
|
const resume = opts.resume;
|
|
56168
56278
|
const restart = opts.restart;
|
|
56279
|
+
const coordinator = opts.coordinator ?? (resume ? carriedCoordinatorTag(resume.row, resume.events) : void 0);
|
|
56280
|
+
const recovery = opts.recovery ?? coordinator?.recovery;
|
|
56169
56281
|
const clock4 = deps.clock ?? systemClock;
|
|
56170
56282
|
const trace = opts.trace ?? startRequestRoot(deps, {
|
|
56171
56283
|
channel: channelOf(msg.channelId),
|
|
@@ -56422,6 +56534,13 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56422
56534
|
return ended;
|
|
56423
56535
|
}
|
|
56424
56536
|
if (owner.kind === "pipeline" && directives.agent === void 0) {
|
|
56537
|
+
if (owner.unit.recoveryReceipt !== void 0 || owner.unit.recoveryHold !== void 0) {
|
|
56538
|
+
await io.reply(
|
|
56539
|
+
`The original-unit recovery for \`${owner.instanceId}:${owner.unit.unit}\` is terminal and its evidence is consumed. This reply was not turned into replacement work.`
|
|
56540
|
+
);
|
|
56541
|
+
await recordPendingOperator();
|
|
56542
|
+
return ended;
|
|
56543
|
+
}
|
|
56425
56544
|
const instance = await deps.coordinatorInstances.get(owner.instanceId).catch(() => null);
|
|
56426
56545
|
if (instance === null) {
|
|
56427
56546
|
await io.reply(
|
|
@@ -56603,7 +56722,14 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56603
56722
|
return true;
|
|
56604
56723
|
};
|
|
56605
56724
|
reserveLegacyOwnership = () => {
|
|
56606
|
-
|
|
56725
|
+
try {
|
|
56726
|
+
token2 = runnerOwnership.reserve(instance.repo, recordedPr.number, reservationOwner);
|
|
56727
|
+
} catch {
|
|
56728
|
+
return refusalOf(
|
|
56729
|
+
"publication_ownership_unknown",
|
|
56730
|
+
`Runner ownership for ${instance.repo}#${recordedPr.number} is being rebuilt, so continuation did not start. Nothing else ran.`
|
|
56731
|
+
);
|
|
56732
|
+
}
|
|
56607
56733
|
if (token2 !== void 0) return void 0;
|
|
56608
56734
|
const competing = runnerOwnership.owner(instance.repo, recordedPr.number);
|
|
56609
56735
|
const sameContinuation = competing?.instanceId === reservationOwner.instanceId && competing.unit === reservationOwner.unit;
|
|
@@ -56706,6 +56832,8 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56706
56832
|
if ((await authorizeAgent(deps, { msg, io, refuse: refuse3, agentName: resolved.agentName })).kind === "refused")
|
|
56707
56833
|
return ended;
|
|
56708
56834
|
const agent = resolved.agentName === "ship" ? shipPresetFor(deps.config.config.ship) : getAgent(resolved.agentName);
|
|
56835
|
+
const recoveryRemainingMs = recovery === void 0 ? void 0 : Math.max(0, recovery.deadlineAt - clock4());
|
|
56836
|
+
const inheritedRemainingMs = parent?.remainingMs === void 0 ? recoveryRemainingMs : recoveryRemainingMs === void 0 ? parent.remainingMs : Math.min(parent.remainingMs, recoveryRemainingMs);
|
|
56709
56837
|
const profileGate = await authorizeProfile(deps, {
|
|
56710
56838
|
msg,
|
|
56711
56839
|
io,
|
|
@@ -56716,14 +56844,14 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56716
56844
|
resolved,
|
|
56717
56845
|
resume,
|
|
56718
56846
|
budget: directives.budget,
|
|
56719
|
-
...
|
|
56847
|
+
...inheritedRemainingMs !== void 0 ? { parentRemainingMs: inheritedRemainingMs } : {}
|
|
56720
56848
|
})
|
|
56721
56849
|
});
|
|
56722
56850
|
if (profileGate.kind === "refused") return ended;
|
|
56723
|
-
|
|
56851
|
+
let { profile } = profileGate;
|
|
56724
56852
|
const legacyOwnershipRefusal = reserveLegacyOwnership?.();
|
|
56725
56853
|
if (legacyOwnershipRefusal !== void 0) {
|
|
56726
|
-
await
|
|
56854
|
+
await refuse3(legacyOwnershipRefusal);
|
|
56727
56855
|
await recordPendingOperator();
|
|
56728
56856
|
return ended;
|
|
56729
56857
|
}
|
|
@@ -56836,6 +56964,26 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56836
56964
|
if (references.refused.length > 0)
|
|
56837
56965
|
await renderRefusal(refusalOf(referenceRefusalCode(references.refused[0]), REFERENCE_REFUSAL), io);
|
|
56838
56966
|
let repoCtx = await repoCtxP;
|
|
56967
|
+
if (recovery !== void 0 && (agent.name === "review" || agent.name === "coding") && (repoCtx.repo?.toLowerCase() !== recovery.repo.toLowerCase() || repoCtx.pr !== recovery.pr || repoCtx.ref !== recovery.headRef || repoCtx.baseRef !== recovery.baseRef || repoCtx.headSha !== recovery.expectedHeadSha)) {
|
|
56968
|
+
const reason2 = "the recovered pull request moved or no longer matches its durable target";
|
|
56969
|
+
await refuse3(
|
|
56970
|
+
refusalOf("setup_failed", reason2),
|
|
56971
|
+
() => card.done(shell.close({ kind: "refused", icon: "\u{1F6AB}", reason: reason2, ...closeLines(clock4(), false) }))
|
|
56972
|
+
);
|
|
56973
|
+
return ended;
|
|
56974
|
+
}
|
|
56975
|
+
if (recovery !== void 0) {
|
|
56976
|
+
const remainingMinutes = Math.floor((recovery.deadlineAt - clock4()) / 6e4);
|
|
56977
|
+
if (remainingMinutes <= 0) {
|
|
56978
|
+
const reason2 = "the original unit's absolute recovery deadline expired during setup";
|
|
56979
|
+
await refuse3(
|
|
56980
|
+
refusalOf("setup_failed", reason2),
|
|
56981
|
+
() => card.done(shell.close({ kind: "refused", icon: "\u{1F6AB}", reason: reason2, ...closeLines(clock4(), false) }))
|
|
56982
|
+
);
|
|
56983
|
+
return ended;
|
|
56984
|
+
}
|
|
56985
|
+
profile = { ...profile, minutes: Math.min(profile.minutes, remainingMinutes) };
|
|
56986
|
+
}
|
|
56839
56987
|
const contractBranch = opts.contract?.rebase.branch;
|
|
56840
56988
|
if (opts.coordinator !== void 0 && contractBranch !== void 0 && agent.name !== "review" && repoCtx.refFromPr === true && repoCtx.ref !== contractBranch) {
|
|
56841
56989
|
const { refFromPr: _refFromPr, headSha: _headSha, ...kept2 } = repoCtx;
|
|
@@ -56968,7 +57116,6 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56968
57116
|
});
|
|
56969
57117
|
if (headPreflight.kind === "refused") return ended;
|
|
56970
57118
|
const parentRunId = parent?.runId;
|
|
56971
|
-
const coordinator = opts.coordinator ?? (resume ? carriedCoordinatorTag(resume.row, resume.events) : void 0);
|
|
56972
57119
|
const registration = await registerRun(deps, {
|
|
56973
57120
|
msg,
|
|
56974
57121
|
io,
|
|
@@ -57332,6 +57479,14 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
57332
57479
|
});
|
|
57333
57480
|
if (headGate.kind === "refused") return ended;
|
|
57334
57481
|
repoCtx = headGate.repoCtx;
|
|
57482
|
+
if (recovery !== void 0 && (agent.name === "review" || agent.name === "coding") && (repoCtx.repo?.toLowerCase() !== recovery.repo.toLowerCase() || repoCtx.pr !== recovery.pr || repoCtx.ref !== recovery.headRef || repoCtx.baseRef !== recovery.baseRef || repoCtx.headSha !== recovery.expectedHeadSha)) {
|
|
57483
|
+
const reason2 = "the recovered pull request moved during setup and no longer matches its durable target";
|
|
57484
|
+
await refuse3(
|
|
57485
|
+
refusalOf("setup_failed", reason2),
|
|
57486
|
+
() => card.done(shell.close({ kind: "refused", icon: "\u{1F6AB}", reason: reason2, ...closeLines(clock4(), false) }))
|
|
57487
|
+
);
|
|
57488
|
+
return ended;
|
|
57489
|
+
}
|
|
57335
57490
|
const verifiedAtAttach = headGate.verifiedAtAttach;
|
|
57336
57491
|
if (headGate.headAdopted) publishMeta(repoCtx);
|
|
57337
57492
|
const bound = round2.selection.binding;
|
|
@@ -57678,6 +57833,51 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
57678
57833
|
...restartRequest.coordinator !== void 0 ? { coordinator: restartRequest.coordinator } : {},
|
|
57679
57834
|
...deps.workflow !== void 0 ? { workflow: deps.workflow } : {}
|
|
57680
57835
|
});
|
|
57836
|
+
} else if (settled.kind === "handed-on" && recovery !== void 0 && coordinator !== void 0 && deps.coordinatorInstances !== void 0) {
|
|
57837
|
+
const unit = unitOfIdempotencyKey(coordinator.idempotencyKey);
|
|
57838
|
+
if (unit === void 0) {
|
|
57839
|
+
console.error(
|
|
57840
|
+
`[dispatch] ${msg.threadKey} could not retain ${settled.pending.length} recovery follow-up(s): the coordinator key names no unit`
|
|
57841
|
+
);
|
|
57842
|
+
} else {
|
|
57843
|
+
const key = { instanceId: coordinator.parentInstanceId, unit };
|
|
57844
|
+
let retained = 0;
|
|
57845
|
+
for (const pending2 of settled.pending) {
|
|
57846
|
+
const attachments = [...pending2.msg.images ?? [], ...pending2.msg.documents ?? []].map((asset) => ({
|
|
57847
|
+
mediaType: asset.mediaType,
|
|
57848
|
+
data: asset.data,
|
|
57849
|
+
...asset.name !== void 0 ? { name: asset.name } : {}
|
|
57850
|
+
}));
|
|
57851
|
+
try {
|
|
57852
|
+
const appended = await deps.coordinatorInstances.appendEvent(key, {
|
|
57853
|
+
...pending2.msg.messageId !== void 0 ? { id: pending2.msg.messageId } : {},
|
|
57854
|
+
sender: pending2.msg.userId,
|
|
57855
|
+
...pending2.msg.userName !== void 0 ? { senderName: pending2.msg.userName } : {},
|
|
57856
|
+
text: pending2.msg.text,
|
|
57857
|
+
...attachments.length > 0 ? { attachments } : {},
|
|
57858
|
+
mode: "steer",
|
|
57859
|
+
at: pending2.at
|
|
57860
|
+
});
|
|
57861
|
+
if (appended.ok) retained++;
|
|
57862
|
+
else {
|
|
57863
|
+
console.error(`[dispatch] ${msg.threadKey} could not retain a recovery follow-up: ${appended.reason}`);
|
|
57864
|
+
await pending2.io.reply(
|
|
57865
|
+
"Your reply could not be saved on the recovered unit. No replacement work was started; please retry the reply."
|
|
57866
|
+
).catch(() => void 0);
|
|
57867
|
+
}
|
|
57868
|
+
} catch (err2) {
|
|
57869
|
+
console.error(
|
|
57870
|
+
`[dispatch] ${msg.threadKey} could not retain a recovery follow-up: ${err2 instanceof Error ? err2.message : String(err2)}`
|
|
57871
|
+
);
|
|
57872
|
+
await pending2.io.reply(
|
|
57873
|
+
"Your reply could not be saved on the recovered unit. No replacement work was started; please retry the reply."
|
|
57874
|
+
).catch(() => void 0);
|
|
57875
|
+
}
|
|
57876
|
+
}
|
|
57877
|
+
console.log(
|
|
57878
|
+
`[dispatch] ${msg.threadKey} retained ${retained}/${settled.pending.length} recovery follow-up(s) on ${key.instanceId}:${key.unit}; no fresh turn ran`
|
|
57879
|
+
);
|
|
57880
|
+
}
|
|
57681
57881
|
} else if (settled.kind === "handed-on") {
|
|
57682
57882
|
const fresh = prepareFreshTurn(deps, { agent: settled.agent, pending: settled.pending, clock: clock4 });
|
|
57683
57883
|
await dispatch(deps, fresh.msg, fresh.io, fresh.opts).catch(
|
|
@@ -71741,6 +71941,12 @@ async function recoverRunnerOwnedPullOwners(instanceIds, instances) {
|
|
|
71741
71941
|
owned.set(runnerOwnedPullKey(instance.repo, unit.pr.number), { instanceId, unit: unit.unit });
|
|
71742
71942
|
}
|
|
71743
71943
|
}
|
|
71944
|
+
for (const unit of await instances.listActiveRecoveries()) {
|
|
71945
|
+
if (unit.pr === void 0) continue;
|
|
71946
|
+
const instance = await instances.get(unit.instanceId);
|
|
71947
|
+
if (instance !== null)
|
|
71948
|
+
owned.set(runnerOwnedPullKey(instance.repo, unit.pr.number), { instanceId: unit.instanceId, unit: unit.unit });
|
|
71949
|
+
}
|
|
71744
71950
|
return owned;
|
|
71745
71951
|
}
|
|
71746
71952
|
var runnerOwnedPullKey, RunnerOwnershipFence;
|
|
@@ -71763,6 +71969,7 @@ var init_runnerOwnership = __esm({
|
|
|
71763
71969
|
* synchronous, so a caller owns the process-local fence before its next
|
|
71764
71970
|
* awaited durable transition. Repeating the same owner's claim is idempotent. */
|
|
71765
71971
|
claim(repo, prNumber, owner) {
|
|
71972
|
+
if (!this.recoveryComplete) throw new Error("runner ownership recovery is still in progress");
|
|
71766
71973
|
const key = runnerOwnedPullKey(repo, prNumber);
|
|
71767
71974
|
const current = this.claimedOwners.get(key) ?? this.recoveredOwners.get(key);
|
|
71768
71975
|
const alreadyOwned = this.claimed.has(key) || this.recovered.has(key);
|
|
@@ -71777,6 +71984,7 @@ var init_runnerOwnership = __esm({
|
|
|
71777
71984
|
* transition. Even the same runner identity cannot reserve it twice: the
|
|
71778
71985
|
* returned token, not the owner fields, identifies the one caller. */
|
|
71779
71986
|
reserve(repo, prNumber, owner) {
|
|
71987
|
+
if (!this.recoveryComplete) throw new Error("runner ownership recovery is still in progress");
|
|
71780
71988
|
const key = runnerOwnedPullKey(repo, prNumber);
|
|
71781
71989
|
if (this.claimed.has(key) || this.recovered.has(key) || this.reservations.has(key)) return void 0;
|
|
71782
71990
|
const token2 = Symbol(key);
|
|
@@ -72573,6 +72781,8 @@ function parseBrief(v) {
|
|
|
72573
72781
|
if (!review.ok) return review;
|
|
72574
72782
|
const checks = checkRows();
|
|
72575
72783
|
if (!checks.ok) return checks;
|
|
72784
|
+
if (b.headSha !== void 0 && (typeof b.headSha !== "string" || !/^[0-9a-f]{40}$/i.test(b.headSha)))
|
|
72785
|
+
return invalid("brief.headSha must be a full commit sha");
|
|
72576
72786
|
return {
|
|
72577
72787
|
ok: true,
|
|
72578
72788
|
value: {
|
|
@@ -72580,6 +72790,7 @@ function parseBrief(v) {
|
|
|
72580
72790
|
unit: b.unit,
|
|
72581
72791
|
pr: n2.value,
|
|
72582
72792
|
reviewRunId: review.value,
|
|
72793
|
+
...b.headSha !== void 0 ? { headSha: b.headSha } : {},
|
|
72583
72794
|
...checks.value !== void 0 ? { checks: checks.value } : {}
|
|
72584
72795
|
}
|
|
72585
72796
|
};
|
|
@@ -72795,6 +73006,32 @@ async function spawn6(body, deps) {
|
|
|
72795
73006
|
if (live) return answerForLive(live, key, threadKey, at);
|
|
72796
73007
|
const done = await finishedWithKey(deps.runs, instance, threadKey, key);
|
|
72797
73008
|
if (done) return json3(200, { ok: true, runId: done.id, threadKey, alreadySpawned: true, at });
|
|
73009
|
+
if (row2?.recovery !== void 0) {
|
|
73010
|
+
const expectedKind = req.preset === "coding" ? "findings" : "review";
|
|
73011
|
+
const stepMatch = new RegExp(`^${row2.unit}/recovery/[1-9][0-9]*/${expectedKind}(?:/a[1-9][0-9]*)?$`).test(req.step);
|
|
73012
|
+
const briefHead = req.brief !== void 0 && "headSha" in req.brief && typeof req.brief.headSha === "string" ? req.brief.headSha : void 0;
|
|
73013
|
+
if (!stepMatch || row2.publication === void 0 || briefHead !== row2.publication.expectedHeadSha || typeof req.budget !== "number" || req.budget > Math.floor((row2.recovery.deadlineAt - at) / minutesToMs(1)))
|
|
73014
|
+
return json3(409, { ok: false, error: "recovery_claim_mismatch", at });
|
|
73015
|
+
let facts2;
|
|
73016
|
+
try {
|
|
73017
|
+
facts2 = await deps.fetchPrFacts({ repo: instance.repo, number: row2.publication.pr });
|
|
73018
|
+
} catch (err2) {
|
|
73019
|
+
return json3(502, { ok: false, error: "github_unavailable", message: describe12(err2), at });
|
|
73020
|
+
}
|
|
73021
|
+
const verified = facts2?.verifiedHead;
|
|
73022
|
+
if (facts2?.state !== "open" || facts2.sameRepoHead !== true || facts2.headBranchExists !== true || facts2.headRef !== row2.publication.headRef || facts2.baseRef !== row2.publication.baseRef || facts2.headSha !== row2.publication.expectedHeadSha || verified?.repo.toLowerCase() !== instance.repo.toLowerCase() || verified.ref !== row2.publication.publicationRef || verified.sha !== row2.publication.expectedHeadSha)
|
|
73023
|
+
return json3(409, {
|
|
73024
|
+
ok: false,
|
|
73025
|
+
error: fullHead(facts2?.headSha) && facts2.headSha !== row2.publication.expectedHeadSha ? "recovery_head_moved" : "recovery_facts_mismatch",
|
|
73026
|
+
at
|
|
73027
|
+
});
|
|
73028
|
+
try {
|
|
73029
|
+
if (deps.runnerOwnership?.claim(instance.repo, row2.publication.pr, row2.publication.owner) !== true)
|
|
73030
|
+
return json3(409, { ok: false, error: "publication_ownership_changed", at });
|
|
73031
|
+
} catch (err2) {
|
|
73032
|
+
return json3(409, { ok: false, error: "publication_ownership_unknown", message: describe12(err2), at });
|
|
73033
|
+
}
|
|
73034
|
+
}
|
|
72798
73035
|
const io = deps.ioFor({ threadKey, userId: instance.userId });
|
|
72799
73036
|
if (!io) return json3(503, { ok: false, error: "no_channel", at });
|
|
72800
73037
|
let turn;
|
|
@@ -72830,6 +73067,13 @@ ${foldThreadEvents(folded)}` };
|
|
|
72830
73067
|
at
|
|
72831
73068
|
});
|
|
72832
73069
|
try {
|
|
73070
|
+
if (row2.recovery !== void 0 && deps.runnerOwnership?.claim(instance.repo, row2.publication.pr, row2.publication.owner) !== true)
|
|
73071
|
+
return json3(409, {
|
|
73072
|
+
ok: false,
|
|
73073
|
+
error: "publication_ownership_changed",
|
|
73074
|
+
message: "the durable recovery claim no longer owns the pull request",
|
|
73075
|
+
at
|
|
73076
|
+
});
|
|
72833
73077
|
const owner = deps.runnerOwnership?.owner(instance.repo, row2.publication.pr);
|
|
72834
73078
|
if (owner === void 0 || owner.instanceId !== row2.publication.owner.instanceId || owner.unit !== row2.publication.owner.unit)
|
|
72835
73079
|
return json3(409, {
|
|
@@ -72873,6 +73117,7 @@ ${foldThreadEvents(folded)}` };
|
|
|
72873
73117
|
const releaseAdmission = leaveAdmission ?? (() => {
|
|
72874
73118
|
});
|
|
72875
73119
|
let startedId;
|
|
73120
|
+
let deadlineTimer;
|
|
72876
73121
|
let lastReply;
|
|
72877
73122
|
let resolveStarted;
|
|
72878
73123
|
const started = new Promise((resolve13) => {
|
|
@@ -72881,6 +73126,14 @@ ${foldThreadEvents(folded)}` };
|
|
|
72881
73126
|
const child = watched(io, {
|
|
72882
73127
|
started: (id) => {
|
|
72883
73128
|
startedId = id;
|
|
73129
|
+
if (row2?.recovery !== void 0) {
|
|
73130
|
+
const stopAtDeadline = () => {
|
|
73131
|
+
void deps.runs.stopRun(id, "hard", { kind: "access", id: "original-unit-recovery-deadline" });
|
|
73132
|
+
};
|
|
73133
|
+
const left = row2.recovery.deadlineAt - (deps.clock ?? systemClock)();
|
|
73134
|
+
if (left <= 0) stopAtDeadline();
|
|
73135
|
+
else deadlineTimer = setTimeout(stopAtDeadline, left);
|
|
73136
|
+
}
|
|
72884
73137
|
releaseAdmission();
|
|
72885
73138
|
resolveStarted(id);
|
|
72886
73139
|
},
|
|
@@ -72896,6 +73149,17 @@ ${foldThreadEvents(folded)}` };
|
|
|
72896
73149
|
const tag = {
|
|
72897
73150
|
parentInstanceId: instance.id,
|
|
72898
73151
|
idempotencyKey: key,
|
|
73152
|
+
...row2?.recovery !== void 0 ? { transportWorkflowId: row2.recovery.workflowId } : {},
|
|
73153
|
+
...row2?.recovery !== void 0 && row2.publication !== void 0 ? {
|
|
73154
|
+
recovery: {
|
|
73155
|
+
repo: row2.publication.repo,
|
|
73156
|
+
pr: row2.publication.pr,
|
|
73157
|
+
headRef: row2.publication.headRef,
|
|
73158
|
+
baseRef: row2.publication.baseRef,
|
|
73159
|
+
expectedHeadSha: row2.recovery.expectedHeadSha,
|
|
73160
|
+
deadlineAt: row2.recovery.deadlineAt
|
|
73161
|
+
}
|
|
73162
|
+
} : {},
|
|
72899
73163
|
...instance.base !== void 0 ? { base: instance.base } : {},
|
|
72900
73164
|
...publication !== void 0 ? { publication } : {}
|
|
72901
73165
|
};
|
|
@@ -72903,6 +73167,16 @@ ${foldThreadEvents(folded)}` };
|
|
|
72903
73167
|
try {
|
|
72904
73168
|
dispatching = deps.dispatch(msg, child, {
|
|
72905
73169
|
coordinator: tag,
|
|
73170
|
+
...row2?.recovery !== void 0 && row2.publication !== void 0 ? {
|
|
73171
|
+
recovery: {
|
|
73172
|
+
repo: row2.publication.repo,
|
|
73173
|
+
pr: row2.publication.pr,
|
|
73174
|
+
headRef: row2.publication.headRef,
|
|
73175
|
+
baseRef: row2.publication.baseRef,
|
|
73176
|
+
expectedHeadSha: row2.publication.expectedHeadSha,
|
|
73177
|
+
deadlineAt: row2.recovery.deadlineAt
|
|
73178
|
+
}
|
|
73179
|
+
} : {},
|
|
72906
73180
|
...turn.contract !== void 0 ? { contract: turn.contract } : {}
|
|
72907
73181
|
});
|
|
72908
73182
|
} catch (err2) {
|
|
@@ -72913,6 +73187,9 @@ ${foldThreadEvents(folded)}` };
|
|
|
72913
73187
|
(outcome) => ({ kind: "ended", outcome }),
|
|
72914
73188
|
(err2) => ({ kind: "threw", err: err2 })
|
|
72915
73189
|
);
|
|
73190
|
+
void settled.finally(() => {
|
|
73191
|
+
if (deadlineTimer !== void 0) clearTimeout(deadlineTimer);
|
|
73192
|
+
});
|
|
72916
73193
|
void settled.then((end) => {
|
|
72917
73194
|
releaseAdmission();
|
|
72918
73195
|
if (end.kind === "threw")
|
|
@@ -73372,6 +73649,41 @@ async function steerChild(body, deps) {
|
|
|
73372
73649
|
const steered = await deps.steerChild?.(body.runId, text, instance) ?? false;
|
|
73373
73650
|
return json3(200, { ok: true, outcome: steered ? "steered" : "not_live", at });
|
|
73374
73651
|
}
|
|
73652
|
+
async function advanceRecoveryPublication(deps, instance, row2, facts2, runId3) {
|
|
73653
|
+
if (row2.recovery === void 0 || row2.publication === void 0 || facts2.headSha === row2.publication.expectedHeadSha)
|
|
73654
|
+
return row2;
|
|
73655
|
+
const child = await deps.runs.getRun(runId3);
|
|
73656
|
+
const expectedPrefix = `${instance.id}:${row2.unit}/recovery/`;
|
|
73657
|
+
const owner = { instanceId: instance.id, unit: row2.unit };
|
|
73658
|
+
if (!child.ok || child.value.finished !== true || child.value.status !== "completed" || child.value.agent !== "coding" || child.value.parentInstanceId !== instance.id || child.value.userId !== instance.userId || child.value.repo?.toLowerCase() !== instance.repo.toLowerCase() || child.value.threadKey !== (row2.threadKey ?? instance.threadKey) || child.value.idempotencyKey?.startsWith(expectedPrefix) !== true || !/\/findings(?:\/a[1-9][0-9]*)?$/.test(child.value.idempotencyKey) || !fullHead(facts2.headSha) || child.value.headSha !== facts2.headSha || child.value.pushed?.some((push) => push.ref === row2.branch && push.sha === facts2.headSha) !== true || facts2.state !== "open" || facts2.sameRepoHead !== true || facts2.headBranchExists !== true || facts2.headRef !== row2.publication.headRef || facts2.baseRef !== row2.publication.baseRef || facts2.verifiedHead?.repo.toLowerCase() !== instance.repo.toLowerCase() || facts2.verifiedHead.ref !== row2.publication.publicationRef || facts2.verifiedHead.sha !== facts2.headSha || !samePublicationOwner(row2.publication.owner, owner))
|
|
73659
|
+
throw new PublicationBindingRefusal("publication_facts_mismatch");
|
|
73660
|
+
try {
|
|
73661
|
+
if (deps.runnerOwnership?.claim(instance.repo, row2.publication.pr, owner) !== true)
|
|
73662
|
+
throw new PublicationBindingRefusal("publication_ownership_changed");
|
|
73663
|
+
} catch (err2) {
|
|
73664
|
+
if (err2 instanceof PublicationBindingRefusal) throw err2;
|
|
73665
|
+
throw new PublicationBindingRefusal("publication_ownership_unknown");
|
|
73666
|
+
}
|
|
73667
|
+
const updated = {
|
|
73668
|
+
...row2,
|
|
73669
|
+
lastPush: facts2.headSha,
|
|
73670
|
+
publication: { ...row2.publication, expectedHeadSha: facts2.headSha },
|
|
73671
|
+
recovery: { ...row2.recovery, expectedHeadSha: facts2.headSha }
|
|
73672
|
+
};
|
|
73673
|
+
let replaced;
|
|
73674
|
+
try {
|
|
73675
|
+
replaced = await deps.instances.compareAndReplaceUnit(row2, updated);
|
|
73676
|
+
} catch {
|
|
73677
|
+
const reread = await deps.instances.listUnits(instance.id).catch(() => void 0);
|
|
73678
|
+
const current = reread?.filter((candidate) => candidate.unit === row2.unit);
|
|
73679
|
+
if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(updated)) replaced = { ok: true };
|
|
73680
|
+
}
|
|
73681
|
+
if (replaced?.ok !== true)
|
|
73682
|
+
throw new PublicationBindingRefusal(
|
|
73683
|
+
replaced?.reason === "stale" ? "publication_binding_stale" : "publication_store_unavailable"
|
|
73684
|
+
);
|
|
73685
|
+
return updated;
|
|
73686
|
+
}
|
|
73375
73687
|
async function bindOpenPullRequest(deps, instance, row2, pr, facts2) {
|
|
73376
73688
|
if (row2 === void 0) return;
|
|
73377
73689
|
const verified = facts2.verifiedHead;
|
|
@@ -73400,8 +73712,13 @@ async function bindOpenPullRequest(deps, instance, row2, pr, facts2) {
|
|
|
73400
73712
|
}
|
|
73401
73713
|
if (priorOwner !== void 0 && !samePublicationOwner(priorOwner, owner))
|
|
73402
73714
|
throw new PublicationBindingRefusal("publication_ownership_changed");
|
|
73403
|
-
|
|
73404
|
-
|
|
73715
|
+
try {
|
|
73716
|
+
if (!fence.claim(instance.repo, pr.number, owner))
|
|
73717
|
+
throw new PublicationBindingRefusal("publication_ownership_changed");
|
|
73718
|
+
} catch (err2) {
|
|
73719
|
+
if (err2 instanceof PublicationBindingRefusal) throw err2;
|
|
73720
|
+
throw new PublicationBindingRefusal("publication_ownership_unknown");
|
|
73721
|
+
}
|
|
73405
73722
|
let claimedHere = priorOwner === void 0;
|
|
73406
73723
|
const releaseClaim = () => {
|
|
73407
73724
|
if (!claimedHere) return;
|
|
@@ -73434,6 +73751,323 @@ async function bindOpenPullRequest(deps, instance, row2, pr, facts2) {
|
|
|
73434
73751
|
);
|
|
73435
73752
|
}
|
|
73436
73753
|
}
|
|
73754
|
+
async function recoverOriginalUnit(body, deps, caller) {
|
|
73755
|
+
const id = parseInstanceId(body.parentInstanceId);
|
|
73756
|
+
if (!id.ok) return json3(400, { ok: false, error: id.error });
|
|
73757
|
+
if (typeof body.unit !== "string" || !UNIT_ID.test(body.unit))
|
|
73758
|
+
return json3(400, { ok: false, error: "unit must be a unit id" });
|
|
73759
|
+
const at = (deps.clock ?? systemClock)();
|
|
73760
|
+
const instance = await deps.instances.get(id.value);
|
|
73761
|
+
if (instance === null) return json3(404, { ok: false, error: "unknown_instance", at });
|
|
73762
|
+
const requestedWorkflowId = typeof body.workflowId === "string" ? body.workflowId : void 0;
|
|
73763
|
+
if (caller === void 0 && requestedWorkflowId === void 0)
|
|
73764
|
+
return json3(403, { ok: false, error: "recovery_caller_required", at });
|
|
73765
|
+
if (caller !== void 0 && caller.userId !== instance.userId)
|
|
73766
|
+
return json3(403, { ok: false, error: "recovery_requester_mismatch", at });
|
|
73767
|
+
const rows = await deps.instances.listUnits(instance.id);
|
|
73768
|
+
const matches = rows.filter((candidate) => candidate.unit === body.unit);
|
|
73769
|
+
if (matches.length !== 1) return json3(409, { ok: false, error: "unit_evidence_ambiguous", at });
|
|
73770
|
+
const row2 = matches[0];
|
|
73771
|
+
if (caller !== void 0 && caller.threadKey !== (row2.threadKey ?? instance.threadKey))
|
|
73772
|
+
return json3(403, { ok: false, error: "recovery_requester_mismatch", at });
|
|
73773
|
+
if (row2.idle !== void 0 && row2.ending !== void 0)
|
|
73774
|
+
return json3(409, { ok: false, error: "unit_lifecycle_ambiguous", at });
|
|
73775
|
+
const originalOwner = { instanceId: instance.id, unit: row2.unit };
|
|
73776
|
+
const publication = row2.publication;
|
|
73777
|
+
const pr = row2.pr;
|
|
73778
|
+
const base = instance.base;
|
|
73779
|
+
let expectedHead = row2.lastPush;
|
|
73780
|
+
if (pr === void 0 || base === void 0 || !fullHead(expectedHead) || publication === void 0 || publication.repo.toLowerCase() !== instance.repo.toLowerCase() || publication.pr !== pr.number || publication.headRef !== row2.branch || publication.baseRef !== base || publication.publicationRef !== row2.branch || !fullHead(publication.expectedHeadSha) || !samePublicationOwner(publication.owner, originalOwner))
|
|
73781
|
+
return json3(409, { ok: false, error: "recovery_binding_mismatch", at });
|
|
73782
|
+
const existingClaim = row2.recovery;
|
|
73783
|
+
let kind;
|
|
73784
|
+
let round2;
|
|
73785
|
+
let remainingMs;
|
|
73786
|
+
let reviewRunId;
|
|
73787
|
+
let stepName;
|
|
73788
|
+
let workflowId;
|
|
73789
|
+
let reviewKey;
|
|
73790
|
+
let deadlineAt;
|
|
73791
|
+
let findings;
|
|
73792
|
+
let findingsRunId;
|
|
73793
|
+
let findingsKey;
|
|
73794
|
+
let claimRow = row2;
|
|
73795
|
+
let facts2;
|
|
73796
|
+
let originalRow;
|
|
73797
|
+
if (existingClaim !== void 0) {
|
|
73798
|
+
({
|
|
73799
|
+
kind,
|
|
73800
|
+
round: round2,
|
|
73801
|
+
remainingMs,
|
|
73802
|
+
reviewRunId,
|
|
73803
|
+
step: stepName,
|
|
73804
|
+
workflowId,
|
|
73805
|
+
reviewKey,
|
|
73806
|
+
deadlineAt,
|
|
73807
|
+
findings,
|
|
73808
|
+
findingsRunId,
|
|
73809
|
+
findingsKey
|
|
73810
|
+
} = existingClaim);
|
|
73811
|
+
if (existingClaim.expectedHeadSha !== expectedHead || publication.expectedHeadSha !== expectedHead)
|
|
73812
|
+
return json3(409, { ok: false, error: "recovery_binding_mismatch", at });
|
|
73813
|
+
const { recovery: _recovery, ...withoutRecovery } = row2;
|
|
73814
|
+
originalRow = { ...withoutRecovery, ending: existingClaim.previousEnding };
|
|
73815
|
+
} else {
|
|
73816
|
+
if (row2.idle !== void 0 || row2.ending === void 0)
|
|
73817
|
+
return json3(409, { ok: false, error: "unit_not_terminal", at });
|
|
73818
|
+
let boundaryPosition = -1;
|
|
73819
|
+
for (let index = row2.rounds.length - 1; index >= 0; index -= 1)
|
|
73820
|
+
if (row2.rounds[index].agent === "review" && row2.rounds[index].outcome !== "started") {
|
|
73821
|
+
boundaryPosition = index;
|
|
73822
|
+
break;
|
|
73823
|
+
}
|
|
73824
|
+
const boundary = boundaryPosition >= 0 ? row2.rounds[boundaryPosition] : void 0;
|
|
73825
|
+
if (boundary === void 0 || boundary.outcome !== "request_changes" && boundary.outcome !== "no_verdict")
|
|
73826
|
+
return json3(409, { ok: false, error: "recovery_ending_unsupported", at });
|
|
73827
|
+
if (row2.rounds.slice(boundaryPosition + 1).some((candidate) => candidate.agent === "review"))
|
|
73828
|
+
return json3(409, { ok: false, error: "recovery_stage_ambiguous", at });
|
|
73829
|
+
kind = boundary.outcome === "request_changes" ? "findings" : "review";
|
|
73830
|
+
if (kind === "review" && row2.ending.kind !== "no_verdict")
|
|
73831
|
+
return json3(409, { ok: false, error: "recovery_ending_mismatch", at });
|
|
73832
|
+
round2 = boundary.index;
|
|
73833
|
+
const caps = instance.caps;
|
|
73834
|
+
if (caps === void 0 || row2.startedAt === void 0 || !Number.isFinite(row2.ending.at))
|
|
73835
|
+
return json3(409, { ok: false, error: "recovery_budget_unknown", at });
|
|
73836
|
+
if (instance.grant?.costCapUsd !== void 0) return json3(409, { ok: false, error: "recovery_budget_unknown", at });
|
|
73837
|
+
if (round2 >= caps.maxRounds) return json3(409, { ok: false, error: "recovery_rounds_exhausted", at });
|
|
73838
|
+
const latestSegmentIndex = row2.segments?.reduce((latest, candidate) => Math.max(latest, candidate.index), 1) ?? 1;
|
|
73839
|
+
const latestSegments = row2.segments?.filter((candidate) => candidate.index === latestSegmentIndex) ?? [];
|
|
73840
|
+
if (latestSegments.length > 1) return json3(409, { ok: false, error: "recovery_budget_unknown", at });
|
|
73841
|
+
const segment = latestSegments[0];
|
|
73842
|
+
const leaseStartedAt = segment?.at ?? row2.startedAt;
|
|
73843
|
+
const resumedLeases = Object.values(row2.wakes ?? {}).filter(
|
|
73844
|
+
(answer) => answer.kind === "segment" && answer.index === latestSegmentIndex && answer.leaseMs !== void 0
|
|
73845
|
+
);
|
|
73846
|
+
if (resumedLeases.length > 0) return json3(409, { ok: false, error: "recovery_budget_unknown", at });
|
|
73847
|
+
remainingMs = minutesToMs(caps.maxMinutes) - (at - leaseStartedAt);
|
|
73848
|
+
const reviewThreadKey = row2.reviewThread?.threadKey ?? row2.threadKey ?? instance.threadKey;
|
|
73849
|
+
const unitThreadKey = row2.threadKey ?? instance.threadKey;
|
|
73850
|
+
const listing = await deps.runs.listRuns({
|
|
73851
|
+
status: "finished",
|
|
73852
|
+
visibleTo: EVERY_RUN,
|
|
73853
|
+
threadKey: reviewThreadKey,
|
|
73854
|
+
limit: RUN_LIST_MAX_LIMIT
|
|
73855
|
+
});
|
|
73856
|
+
const unitListing = unitThreadKey === reviewThreadKey ? listing : await deps.runs.listRuns({
|
|
73857
|
+
status: "finished",
|
|
73858
|
+
visibleTo: EVERY_RUN,
|
|
73859
|
+
threadKey: unitThreadKey,
|
|
73860
|
+
limit: RUN_LIST_MAX_LIMIT
|
|
73861
|
+
});
|
|
73862
|
+
const segmentPrefix = stepPrefixOf(
|
|
73863
|
+
row2.unit,
|
|
73864
|
+
latestSegmentIndex > 1 ? { segment: latestSegmentIndex, renewalsSpent: 0, spendUsd: null } : void 0
|
|
73865
|
+
);
|
|
73866
|
+
const reviewedHead = publication.expectedHeadSha;
|
|
73867
|
+
const reviewKeyPrefix = `${instance.id}:${segmentPrefix}/${round2}/review`;
|
|
73868
|
+
const candidates2 = listing.runs.filter((run2) => {
|
|
73869
|
+
if (!run2.finished || run2.agent !== "review" || run2.parentInstanceId !== instance.id || run2.idempotencyKey !== reviewKeyPrefix && run2.idempotencyKey?.startsWith(`${reviewKeyPrefix}/a`) !== true || run2.userId !== instance.userId || run2.repo?.toLowerCase() !== instance.repo.toLowerCase() || run2.threadKey !== reviewThreadKey)
|
|
73870
|
+
return false;
|
|
73871
|
+
if (run2.reviewHead !== reviewedHead) return false;
|
|
73872
|
+
if (kind === "review") return run2.verdict === void 0 && run2.reviewPost === void 0;
|
|
73873
|
+
return run2.verdict?.verdict === "request_changes" && run2.reviewHead === reviewedHead && run2.reviewPost?.posted === true && run2.reviewPost.head === reviewedHead && run2.reviewPost.verdict === "request_changes" && run2.reviewPost.target.repo.toLowerCase() === instance.repo.toLowerCase() && run2.reviewPost.target.number === pr.number;
|
|
73874
|
+
});
|
|
73875
|
+
if (candidates2.length !== 1) return json3(409, { ok: false, error: "recovery_review_evidence_ambiguous", at });
|
|
73876
|
+
reviewRunId = candidates2[0].id;
|
|
73877
|
+
reviewKey = candidates2[0].idempotencyKey;
|
|
73878
|
+
try {
|
|
73879
|
+
facts2 = await deps.fetchPrFacts({ repo: instance.repo, number: pr.number });
|
|
73880
|
+
} catch (err2) {
|
|
73881
|
+
return json3(502, { ok: false, error: "github_unavailable", message: describe12(err2), at });
|
|
73882
|
+
}
|
|
73883
|
+
if (facts2 === void 0) return json3(502, { ok: false, error: "github_unavailable", at });
|
|
73884
|
+
const verified2 = facts2.verifiedHead;
|
|
73885
|
+
if (facts2.state !== "open" || facts2.sameRepoHead !== true || facts2.headBranchExists !== true || facts2.headRef !== row2.branch || facts2.baseRef !== base || !fullHead(facts2.headSha) || verified2 === void 0 || verified2.repo.toLowerCase() !== instance.repo.toLowerCase() || verified2.ref !== row2.branch || verified2.sha !== facts2.headSha)
|
|
73886
|
+
return json3(409, { ok: false, error: "recovery_facts_mismatch", at });
|
|
73887
|
+
const findingsStep = `${segmentPrefix}/${boundary.index}/findings/pr-check`;
|
|
73888
|
+
const findingsPrefix = `${instance.id}:${segmentPrefix}/${boundary.index}/findings`;
|
|
73889
|
+
const headMoved = facts2.headSha !== reviewedHead || expectedHead !== reviewedHead;
|
|
73890
|
+
const completedFindings = unitListing.runs.filter(
|
|
73891
|
+
(run2) => boundary.outcome === "request_changes" && row2.ending?.cause === "step_threw" && row2.ending.step === findingsStep && row2.ending.round === boundary.index && (expectedHead === reviewedHead || expectedHead === facts2.headSha) && run2.finished && run2.status === "completed" && run2.agent === "coding" && run2.parentInstanceId === instance.id && run2.userId === instance.userId && run2.repo?.toLowerCase() === instance.repo.toLowerCase() && run2.threadKey === unitThreadKey && (run2.idempotencyKey === findingsPrefix || run2.idempotencyKey?.startsWith(`${findingsPrefix}/a`) === true) && run2.headSha === facts2.headSha && (!headMoved || run2.pushed?.some((push) => push.ref === row2.branch && push.sha === facts2.headSha) === true) && run2.startedAt >= (candidates2[0].finishedAt ?? candidates2[0].startedAt) && (run2.finishedAt === void 0 || run2.finishedAt <= row2.ending.at)
|
|
73892
|
+
);
|
|
73893
|
+
if (completedFindings.length > 1 || headMoved && completedFindings.length !== 1)
|
|
73894
|
+
return json3(409, { ok: false, error: "recovery_head_moved", at });
|
|
73895
|
+
if (completedFindings.length === 1) {
|
|
73896
|
+
findingsRunId = completedFindings[0].id;
|
|
73897
|
+
findingsKey = completedFindings[0].idempotencyKey;
|
|
73898
|
+
expectedHead = facts2.headSha;
|
|
73899
|
+
kind = "review";
|
|
73900
|
+
round2 = boundary.index + 1;
|
|
73901
|
+
claimRow = {
|
|
73902
|
+
...row2,
|
|
73903
|
+
lastPush: expectedHead,
|
|
73904
|
+
publication: { ...publication, expectedHeadSha: expectedHead }
|
|
73905
|
+
};
|
|
73906
|
+
} else expectedHead = reviewedHead;
|
|
73907
|
+
const floor = minutesToMs(leaseMinimum(kind === "findings" ? "fix" : "review"));
|
|
73908
|
+
if (!Number.isFinite(remainingMs) || remainingMs < floor)
|
|
73909
|
+
return json3(409, { ok: false, error: "recovery_wall_clock_exhausted", at });
|
|
73910
|
+
stepName = `${row2.unit}/recovery/${round2}/${kind}`;
|
|
73911
|
+
workflowId = `recovery-${findingsRunId ?? reviewRunId}`;
|
|
73912
|
+
deadlineAt = at + remainingMs;
|
|
73913
|
+
findings = kind === "findings" ? candidates2[0].verdict?.findings : void 0;
|
|
73914
|
+
if (row2.recoveryReceipt?.reviewRunId === reviewRunId)
|
|
73915
|
+
return json3(409, { ok: false, error: "recovery_already_completed", at });
|
|
73916
|
+
originalRow = row2;
|
|
73917
|
+
}
|
|
73918
|
+
const expiredExistingClaim = existingClaim !== void 0 && at >= deadlineAt;
|
|
73919
|
+
if (requestedWorkflowId !== void 0) {
|
|
73920
|
+
if (existingClaim === void 0 || requestedWorkflowId !== workflowId)
|
|
73921
|
+
return json3(409, { ok: false, error: "recovery_claim_mismatch", at });
|
|
73922
|
+
}
|
|
73923
|
+
const fence = deps.runnerOwnership;
|
|
73924
|
+
if (fence === void 0 || fence.reserve === void 0 || fence.transferReservation === void 0 || fence.releaseReservation === void 0)
|
|
73925
|
+
return json3(503, { ok: false, error: "publication_ownership_unknown", at });
|
|
73926
|
+
const releaseReservation = fence.releaseReservation.bind(fence);
|
|
73927
|
+
if (existingClaim !== void 0) {
|
|
73928
|
+
try {
|
|
73929
|
+
if (!fence.claim(instance.repo, pr.number, originalOwner))
|
|
73930
|
+
return json3(409, { ok: false, error: "publication_ownership_changed", at });
|
|
73931
|
+
} catch (err2) {
|
|
73932
|
+
return json3(503, { ok: false, error: "publication_ownership_unknown", message: describe12(err2), at });
|
|
73933
|
+
}
|
|
73934
|
+
}
|
|
73935
|
+
let claimedRow = row2;
|
|
73936
|
+
let token2;
|
|
73937
|
+
let transferred = existingClaim !== void 0;
|
|
73938
|
+
const rollback = async (error, consumed = false) => {
|
|
73939
|
+
if (originalRow !== void 0) {
|
|
73940
|
+
const replacement = consumed ? {
|
|
73941
|
+
...originalRow,
|
|
73942
|
+
recoveryReceipt: { reviewRunId, workflowId, at }
|
|
73943
|
+
} : originalRow;
|
|
73944
|
+
const restored = await deps.instances.compareAndReplaceUnit(claimedRow, replacement).catch(() => void 0);
|
|
73945
|
+
if (restored?.ok !== true)
|
|
73946
|
+
return json3(500, {
|
|
73947
|
+
ok: false,
|
|
73948
|
+
error: "recovery_rollback_failed",
|
|
73949
|
+
cause: error,
|
|
73950
|
+
reason: restored?.reason ?? "unavailable",
|
|
73951
|
+
at
|
|
73952
|
+
});
|
|
73953
|
+
}
|
|
73954
|
+
const released = transferred ? fence.release(instance.repo, pr.number, originalOwner) : token2 !== void 0 ? releaseReservation(instance.repo, pr.number, token2) : true;
|
|
73955
|
+
if (!released) return json3(500, { ok: false, error: "recovery_cleanup_failed", cause: error, at });
|
|
73956
|
+
return json3(409, { ok: false, error, at });
|
|
73957
|
+
};
|
|
73958
|
+
if (requestedWorkflowId === void 0 && expiredExistingClaim) {
|
|
73959
|
+
const status3 = await deps.recoveryStatus?.(workflowId).catch((err2) => ({
|
|
73960
|
+
kind: "unanswered",
|
|
73961
|
+
reason: describe12(err2)
|
|
73962
|
+
}));
|
|
73963
|
+
if (status3 === void 0 || status3.kind === "unanswered")
|
|
73964
|
+
return json3(503, {
|
|
73965
|
+
ok: false,
|
|
73966
|
+
error: "recovery_status_unanswered",
|
|
73967
|
+
...status3?.kind === "unanswered" ? { message: status3.reason } : {},
|
|
73968
|
+
workflowId,
|
|
73969
|
+
at
|
|
73970
|
+
});
|
|
73971
|
+
if (status3.kind === "status" && !["complete", "errored", "terminated"].includes(status3.status))
|
|
73972
|
+
return json3(409, { ok: false, error: "recovery_wall_clock_exhausted", workflowId, at });
|
|
73973
|
+
return rollback("recovery_wall_clock_exhausted", status3.kind !== "absent");
|
|
73974
|
+
}
|
|
73975
|
+
if (facts2 === void 0)
|
|
73976
|
+
try {
|
|
73977
|
+
facts2 = await deps.fetchPrFacts({ repo: instance.repo, number: pr.number });
|
|
73978
|
+
} catch (err2) {
|
|
73979
|
+
return json3(502, { ok: false, error: "github_unavailable", message: describe12(err2), at });
|
|
73980
|
+
}
|
|
73981
|
+
if (facts2 === void 0) return json3(502, { ok: false, error: "github_unavailable", at });
|
|
73982
|
+
const verified = facts2.verifiedHead;
|
|
73983
|
+
if (facts2.state !== "open" || facts2.sameRepoHead !== true || facts2.headBranchExists !== true || facts2.headRef !== row2.branch || facts2.baseRef !== base || !fullHead(facts2.headSha) || facts2.headSha !== expectedHead || verified === void 0 || verified.repo.toLowerCase() !== instance.repo.toLowerCase() || verified.ref !== row2.branch || verified.sha !== expectedHead) {
|
|
73984
|
+
const error = fullHead(facts2.headSha) && facts2.headSha !== expectedHead ? "recovery_head_moved" : "recovery_facts_mismatch";
|
|
73985
|
+
return requestedWorkflowId === void 0 ? json3(409, { ok: false, error, at }) : rollback(error);
|
|
73986
|
+
}
|
|
73987
|
+
if (existingClaim === void 0) {
|
|
73988
|
+
try {
|
|
73989
|
+
token2 = fence.reserve(instance.repo, pr.number, originalOwner);
|
|
73990
|
+
} catch (err2) {
|
|
73991
|
+
return json3(503, { ok: false, error: "publication_ownership_unknown", message: describe12(err2), at });
|
|
73992
|
+
}
|
|
73993
|
+
if (token2 === void 0) return json3(409, { ok: false, error: "publication_ownership_changed", at });
|
|
73994
|
+
const { ending: _ending, ...withoutEnding } = claimRow;
|
|
73995
|
+
claimedRow = {
|
|
73996
|
+
...withoutEnding,
|
|
73997
|
+
recovery: {
|
|
73998
|
+
kind,
|
|
73999
|
+
round: round2,
|
|
74000
|
+
expectedHeadSha: expectedHead,
|
|
74001
|
+
remainingMs,
|
|
74002
|
+
claimedAt: at,
|
|
74003
|
+
step: stepName,
|
|
74004
|
+
reviewRunId,
|
|
74005
|
+
...findingsRunId !== void 0 ? { findingsRunId } : {},
|
|
74006
|
+
...findingsKey !== void 0 ? { findingsKey } : {},
|
|
74007
|
+
...findings !== void 0 ? { findings } : {},
|
|
74008
|
+
previousEnding: row2.ending,
|
|
74009
|
+
workflowId,
|
|
74010
|
+
deadlineAt,
|
|
74011
|
+
reviewKey
|
|
74012
|
+
}
|
|
74013
|
+
};
|
|
74014
|
+
let replaced;
|
|
74015
|
+
try {
|
|
74016
|
+
replaced = await deps.instances.compareAndReplaceUnit(row2, claimedRow);
|
|
74017
|
+
} catch (err2) {
|
|
74018
|
+
const reread = await deps.instances.listUnits(instance.id).catch(() => void 0);
|
|
74019
|
+
const current = reread?.filter((candidate) => candidate.unit === row2.unit);
|
|
74020
|
+
if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(claimedRow)) replaced = { ok: true };
|
|
74021
|
+
else if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(row2)) {
|
|
74022
|
+
const released = releaseReservation(instance.repo, pr.number, token2);
|
|
74023
|
+
if (!released)
|
|
74024
|
+
return json3(500, { ok: false, error: "recovery_cleanup_failed", cause: "recovery_store_unavailable", at });
|
|
74025
|
+
return json3(503, { ok: false, error: "recovery_store_unavailable", message: describe12(err2), at });
|
|
74026
|
+
} else return json3(503, { ok: false, error: "recovery_claim_unanswered", message: describe12(err2), at });
|
|
74027
|
+
}
|
|
74028
|
+
if (replaced.ok !== true) {
|
|
74029
|
+
const error = replaced.reason === "stale" ? "recovery_claim_stale" : "recovery_store_unavailable";
|
|
74030
|
+
const released = releaseReservation(instance.repo, pr.number, token2);
|
|
74031
|
+
if (!released) return json3(500, { ok: false, error: "recovery_cleanup_failed", cause: error, at });
|
|
74032
|
+
return json3(409, { ok: false, error, at });
|
|
74033
|
+
}
|
|
74034
|
+
if (!fence.transferReservation(instance.repo, pr.number, token2, originalOwner, originalOwner))
|
|
74035
|
+
return rollback("publication_ownership_changed");
|
|
74036
|
+
token2 = void 0;
|
|
74037
|
+
transferred = true;
|
|
74038
|
+
}
|
|
74039
|
+
if (requestedWorkflowId !== void 0)
|
|
74040
|
+
return json3(200, { ok: true, parentInstanceId: instance.id, unit: row2.unit, workflowId, at });
|
|
74041
|
+
const startRecovery = deps.startRecovery;
|
|
74042
|
+
if (startRecovery === void 0) return rollback("recovery_workflow_unavailable");
|
|
74043
|
+
const started = await startRecovery(workflowId, {
|
|
74044
|
+
kind: "recover-original-unit",
|
|
74045
|
+
parentInstanceId: instance.id,
|
|
74046
|
+
unit: row2.unit
|
|
74047
|
+
}).catch((err2) => ({ kind: "unanswered", reason: describe12(err2) }));
|
|
74048
|
+
if (started.kind !== "created" && started.kind !== "duplicate") {
|
|
74049
|
+
if (started.kind === "failed") return rollback("recovery_workflow_failed");
|
|
74050
|
+
return json3(200, {
|
|
74051
|
+
ok: true,
|
|
74052
|
+
outcome: "indeterminate",
|
|
74053
|
+
workflowId,
|
|
74054
|
+
parentInstanceId: instance.id,
|
|
74055
|
+
unit: row2.unit,
|
|
74056
|
+
message: started.reason,
|
|
74057
|
+
at
|
|
74058
|
+
});
|
|
74059
|
+
}
|
|
74060
|
+
if (started.kind === "duplicate" && started.status !== void 0 && ["complete", "errored", "terminated"].includes(started.status))
|
|
74061
|
+
return rollback("recovery_workflow_terminal", true);
|
|
74062
|
+
return json3(200, {
|
|
74063
|
+
ok: true,
|
|
74064
|
+
outcome: started.kind === "duplicate" ? "already_started" : "started",
|
|
74065
|
+
workflowId,
|
|
74066
|
+
parentInstanceId: instance.id,
|
|
74067
|
+
unit: row2.unit,
|
|
74068
|
+
at
|
|
74069
|
+
});
|
|
74070
|
+
}
|
|
73437
74071
|
async function prCheck(body, deps) {
|
|
73438
74072
|
const id = parseInstanceId(body.parentInstanceId);
|
|
73439
74073
|
if (!id.ok) return json3(400, { ok: false, error: id.error });
|
|
@@ -73466,7 +74100,8 @@ async function prCheck(body, deps) {
|
|
|
73466
74100
|
await rememberTerminal(followedPr);
|
|
73467
74101
|
return json3(200, { ok: true, ...state2, at });
|
|
73468
74102
|
}
|
|
73469
|
-
await
|
|
74103
|
+
const bindingRow = unit.row !== void 0 && unit.row.recovery !== void 0 && recover !== void 0 ? await advanceRecoveryPublication(deps, instance, unit.row, facts2, recover.runId) : unit.row;
|
|
74104
|
+
await bindOpenPullRequest(deps, instance, bindingRow, followedPr, facts2);
|
|
73470
74105
|
const prRef = { repo: instance.repo, number: follow };
|
|
73471
74106
|
const headSha = facts2.headSha;
|
|
73472
74107
|
const approved = entry && headSha !== void 0 ? await reviewPostedAt(deps, prRef, "approve", headSha) : void 0;
|
|
@@ -73854,19 +74489,43 @@ async function round(body, deps) {
|
|
|
73854
74489
|
const instance = await deps.instances.get(id.value);
|
|
73855
74490
|
if (!instance) return json3(404, { ok: false, error: "unknown_instance" });
|
|
73856
74491
|
if (namesForeignRun(instance, body)) return json3(404, { ok: false, error: "not_found" });
|
|
73857
|
-
const host = await hostRunOf(deps, instance);
|
|
73858
|
-
if (host.kind === "not_host") return json3(409, { ok: false, error: "not_host", at });
|
|
73859
74492
|
const units = await deps.instances.listUnits(instance.id);
|
|
73860
74493
|
const row2 = units.find((u) => u.unit === body.unit);
|
|
73861
74494
|
if (!row2) return json3(404, { ok: false, error: "unit_not_found", unit: body.unit });
|
|
74495
|
+
const recoveryWorkflowId = typeof body.recoveryWorkflowId === "string" && INSTANCE_ID_PATTERN.test(body.recoveryWorkflowId) ? body.recoveryWorkflowId : void 0;
|
|
74496
|
+
if (body.recoveryWorkflowId !== void 0 && recoveryWorkflowId === void 0)
|
|
74497
|
+
return json3(400, { ok: false, error: "recoveryWorkflowId must be a Workflow instance id", at });
|
|
74498
|
+
if (row2.recovery !== void 0 && recoveryWorkflowId !== row2.recovery.workflowId)
|
|
74499
|
+
return json3(409, { ok: false, error: "recovery_claim_mismatch", at });
|
|
74500
|
+
const host = row2.recovery !== void 0 ? { kind: "not_host" } : await hostRunOf(deps, instance);
|
|
74501
|
+
if (host.kind === "not_host" && row2.recovery === void 0) return json3(409, { ok: false, error: "not_host", at });
|
|
74502
|
+
const note = { index: body.index, agent: body.agent, outcome: body.outcome, at, ...gate ? { gate } : {} };
|
|
74503
|
+
const previous = row2.rounds.at(-1);
|
|
74504
|
+
if (row2.recovery !== void 0 && previous?.index === note.index && previous.agent === note.agent && previous.outcome === note.outcome && JSON.stringify(previous.gate) === JSON.stringify(note.gate))
|
|
74505
|
+
return json3(200, { ok: true, at: previous.at });
|
|
73862
74506
|
const updated = {
|
|
73863
74507
|
...row2,
|
|
73864
|
-
rounds: [
|
|
73865
|
-
...row2.rounds,
|
|
73866
|
-
{ index: body.index, agent: body.agent, outcome: body.outcome, at, ...gate ? { gate } : {} }
|
|
73867
|
-
]
|
|
74508
|
+
rounds: [...row2.rounds, note]
|
|
73868
74509
|
};
|
|
73869
|
-
|
|
74510
|
+
if (row2.recovery !== void 0) {
|
|
74511
|
+
let replaced;
|
|
74512
|
+
try {
|
|
74513
|
+
replaced = await deps.instances.compareAndReplaceUnit(row2, updated);
|
|
74514
|
+
} catch {
|
|
74515
|
+
const reread = await deps.instances.listUnits(instance.id).catch(() => void 0);
|
|
74516
|
+
const current = reread?.filter((candidate) => candidate.unit === row2.unit);
|
|
74517
|
+
if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(updated)) replaced = { ok: true };
|
|
74518
|
+
else if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(row2))
|
|
74519
|
+
return json3(503, { ok: false, error: "recovery_store_unavailable", at });
|
|
74520
|
+
else return json3(409, { ok: false, error: "recovery_claim_stale", at });
|
|
74521
|
+
}
|
|
74522
|
+
if (replaced?.ok !== true)
|
|
74523
|
+
return json3(409, {
|
|
74524
|
+
ok: false,
|
|
74525
|
+
error: replaced?.reason === "stale" ? "recovery_claim_stale" : "recovery_store_unavailable",
|
|
74526
|
+
at
|
|
74527
|
+
});
|
|
74528
|
+
} else await deps.instances.putUnits([updated]);
|
|
73870
74529
|
if (host.kind === "host") {
|
|
73871
74530
|
const thread = unitThread(instance, updated, units.length);
|
|
73872
74531
|
await hostPublish(
|
|
@@ -73946,11 +74605,11 @@ async function unitWake(body, deps) {
|
|
|
73946
74605
|
const instance = await deps.instances.get(id.value);
|
|
73947
74606
|
if (!instance) return json3(404, { ok: false, error: "unknown_instance" });
|
|
73948
74607
|
if (namesForeignRun(instance, body)) return json3(404, { ok: false, error: "not_found" });
|
|
73949
|
-
const host = await hostRunOf(deps, instance);
|
|
73950
|
-
if (host.kind === "not_host") return json3(409, { ok: false, error: "not_host", at });
|
|
73951
74608
|
const units = await deps.instances.listUnits(instance.id);
|
|
73952
74609
|
const row2 = units.find((u) => u.unit === body.unit);
|
|
73953
74610
|
if (!row2) return json3(404, { ok: false, error: "unit_not_found", unit: body.unit });
|
|
74611
|
+
const host = await hostRunOf(deps, instance);
|
|
74612
|
+
if (host.kind === "not_host") return json3(409, { ok: false, error: "not_host", at });
|
|
73954
74613
|
const stored = row2.wakes?.[body.waitId];
|
|
73955
74614
|
if (stored !== void 0) return json3(200, { ok: true, answer: stored, at });
|
|
73956
74615
|
if (!row2.idle) {
|
|
@@ -74103,11 +74762,25 @@ async function unitEnd(body, deps) {
|
|
|
74103
74762
|
const instance = await deps.instances.get(id.value);
|
|
74104
74763
|
if (!instance) return json3(404, { ok: false, error: "unknown_instance" });
|
|
74105
74764
|
if (namesForeignRun(instance, body)) return json3(404, { ok: false, error: "not_found" });
|
|
74106
|
-
const host = await hostRunOf(deps, instance);
|
|
74107
|
-
if (host.kind === "not_host") return json3(409, { ok: false, error: "not_host", at });
|
|
74108
74765
|
const units = await deps.instances.listUnits(instance.id);
|
|
74109
74766
|
const row2 = units.find((u) => u.unit === body.unit);
|
|
74110
74767
|
if (!row2) return json3(404, { ok: false, error: "unit_not_found", unit: body.unit });
|
|
74768
|
+
const recoveryWorkflowId = typeof body.recoveryWorkflowId === "string" && INSTANCE_ID_PATTERN.test(body.recoveryWorkflowId) ? body.recoveryWorkflowId : void 0;
|
|
74769
|
+
if (body.recoveryWorkflowId !== void 0 && recoveryWorkflowId === void 0)
|
|
74770
|
+
return json3(400, { ok: false, error: "recoveryWorkflowId must be a Workflow instance id", at });
|
|
74771
|
+
if (recoveryWorkflowId !== void 0 && row2.recoveryReceipt?.workflowId === recoveryWorkflowId) {
|
|
74772
|
+
if (row2.pr !== void 0) {
|
|
74773
|
+
const owner = { instanceId: instance.id, unit: row2.unit };
|
|
74774
|
+
const current = deps.runnerOwnership?.owner(instance.repo, row2.pr.number);
|
|
74775
|
+
if (current?.instanceId === owner.instanceId && current.unit === owner.unit)
|
|
74776
|
+
deps.runnerOwnership?.release(instance.repo, row2.pr.number, owner);
|
|
74777
|
+
}
|
|
74778
|
+
return json3(200, { ok: true, alreadySettled: true, at: row2.recoveryReceipt.at });
|
|
74779
|
+
}
|
|
74780
|
+
if (row2.recovery !== void 0 && recoveryWorkflowId !== row2.recovery.workflowId)
|
|
74781
|
+
return json3(409, { ok: false, error: "recovery_claim_mismatch", at });
|
|
74782
|
+
const host = row2.recovery !== void 0 ? { kind: "not_host" } : await hostRunOf(deps, instance);
|
|
74783
|
+
if (host.kind === "not_host" && row2.recovery === void 0) return json3(409, { ok: false, error: "not_host", at });
|
|
74111
74784
|
const pr = body.pr;
|
|
74112
74785
|
const lastPush = normalizeHead(body.headSha);
|
|
74113
74786
|
const segment = ending.kind === "continued" ? parseSegment(body.segment) : void 0;
|
|
@@ -74117,9 +74790,30 @@ async function unitEnd(body, deps) {
|
|
|
74117
74790
|
if (ending.kind === "idle" && idle === void 0)
|
|
74118
74791
|
return json3(400, { ok: false, error: "an idle ending must carry its why and the renewals left" });
|
|
74119
74792
|
const segments2 = row2.segments ?? [];
|
|
74120
|
-
const {
|
|
74793
|
+
const recoveryHold = row2.recovery !== void 0 && ending.kind === "held" && isHumanGatePending(ending.humanGate) ? { cause: "human", gate: ending.humanGate } : row2.recovery !== void 0 && ending.kind === "held" && ending.holdCause === "draft" && pr !== void 0 && typeof pr.number === "number" && typeof pr.url === "string" ? { cause: "draft", pr: { number: pr.number, url: pr.url } } : void 0;
|
|
74794
|
+
if (row2.recovery !== void 0 && ending.kind === "held" && recoveryHold === void 0)
|
|
74795
|
+
return json3(400, { ok: false, error: "a recovered held ending must carry a typed hold cause", at });
|
|
74796
|
+
if (row2.recovery !== void 0 && (idle !== void 0 || segment !== void 0))
|
|
74797
|
+
return json3(409, { ok: false, error: "recovery_continuation_unsupported", at });
|
|
74798
|
+
if (row2.recovery !== void 0 && row2.pr !== void 0) {
|
|
74799
|
+
try {
|
|
74800
|
+
if (deps.runnerOwnership?.claim(instance.repo, row2.pr.number, { instanceId: instance.id, unit: row2.unit }) !== true)
|
|
74801
|
+
return json3(409, { ok: false, error: "publication_ownership_changed", at });
|
|
74802
|
+
} catch (err2) {
|
|
74803
|
+
return json3(409, { ok: false, error: "publication_ownership_unknown", message: describe12(err2), at });
|
|
74804
|
+
}
|
|
74805
|
+
}
|
|
74806
|
+
const { idle: _idle, recovery: _recovery, ...rowWithoutLifecycle } = row2;
|
|
74121
74807
|
const updated = {
|
|
74122
|
-
...idle !== void 0 || segment !== void 0 ? row2 :
|
|
74808
|
+
...idle !== void 0 || segment !== void 0 ? row2 : rowWithoutLifecycle,
|
|
74809
|
+
...row2.recovery !== void 0 ? {
|
|
74810
|
+
recoveryReceipt: {
|
|
74811
|
+
reviewRunId: row2.recovery.reviewRunId,
|
|
74812
|
+
workflowId: row2.recovery.workflowId,
|
|
74813
|
+
at
|
|
74814
|
+
}
|
|
74815
|
+
} : {},
|
|
74816
|
+
...recoveryHold !== void 0 ? { recoveryHold } : {},
|
|
74123
74817
|
...pr && typeof pr.number === "number" && typeof pr.url === "string" ? { pr: { number: pr.number, url: pr.url } } : {},
|
|
74124
74818
|
...lastPush !== void 0 ? { lastPush } : {},
|
|
74125
74819
|
...idle !== void 0 ? { idle } : segment !== void 0 ? { segments: segments2.some((s) => s.index === segment.index) ? segments2 : [...segments2, { ...segment, at }] } : {
|
|
@@ -74135,9 +74829,32 @@ async function unitEnd(body, deps) {
|
|
|
74135
74829
|
}
|
|
74136
74830
|
}
|
|
74137
74831
|
};
|
|
74138
|
-
|
|
74832
|
+
if (row2.recovery !== void 0) {
|
|
74833
|
+
let replaced;
|
|
74834
|
+
try {
|
|
74835
|
+
replaced = await deps.instances.compareAndReplaceUnit(row2, updated);
|
|
74836
|
+
} catch {
|
|
74837
|
+
const reread = await deps.instances.listUnits(instance.id).catch(() => void 0);
|
|
74838
|
+
const current = reread?.filter((candidate) => candidate.unit === row2.unit);
|
|
74839
|
+
if (current?.length === 1 && (JSON.stringify(current[0]) === JSON.stringify(updated) || current[0].recovery === void 0 && current[0].recoveryReceipt?.workflowId === row2.recovery.workflowId && current[0].recoveryReceipt?.reviewRunId === row2.recovery.reviewRunId))
|
|
74840
|
+
replaced = { ok: true };
|
|
74841
|
+
else if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(row2))
|
|
74842
|
+
return json3(503, { ok: false, error: "recovery_store_unavailable", at });
|
|
74843
|
+
else return json3(409, { ok: false, error: "recovery_claim_stale", at });
|
|
74844
|
+
}
|
|
74845
|
+
if (replaced?.ok !== true)
|
|
74846
|
+
return json3(409, {
|
|
74847
|
+
ok: false,
|
|
74848
|
+
error: replaced?.reason === "stale" ? "recovery_claim_stale" : "recovery_store_unavailable",
|
|
74849
|
+
at
|
|
74850
|
+
});
|
|
74851
|
+
} else await deps.instances.putUnits([updated]);
|
|
74139
74852
|
if (segment === void 0 && idle === void 0 && updated.pr !== void 0)
|
|
74140
|
-
deps.runnerOwnership?.release(
|
|
74853
|
+
deps.runnerOwnership?.release(
|
|
74854
|
+
instance.repo,
|
|
74855
|
+
updated.pr.number,
|
|
74856
|
+
row2.recovery !== void 0 ? { instanceId: instance.id, unit: row2.unit } : void 0
|
|
74857
|
+
);
|
|
74141
74858
|
const thread = unitThread(instance, updated, units.length);
|
|
74142
74859
|
if (host.kind === "host")
|
|
74143
74860
|
await hostPublish(
|
|
@@ -74158,7 +74875,7 @@ async function unitEnd(body, deps) {
|
|
|
74158
74875
|
at
|
|
74159
74876
|
);
|
|
74160
74877
|
const io = thread.threadKey !== void 0 ? deps.ioFor({ threadKey: thread.threadKey, userId: instance.userId }) : void 0;
|
|
74161
|
-
if (segment === void 0 && idle === void 0) {
|
|
74878
|
+
if (segment === void 0 && idle === void 0 && row2.recovery === void 0) {
|
|
74162
74879
|
const leftovers = await deps.instances.listEvents({ instanceId: instance.id, unit: row2.unit }, true).catch(() => []);
|
|
74163
74880
|
if (leftovers.length > 0 && io === void 0) {
|
|
74164
74881
|
(deps.log ?? console.warn)(
|
|
@@ -74197,6 +74914,13 @@ async function unitEnd(body, deps) {
|
|
|
74197
74914
|
);
|
|
74198
74915
|
}
|
|
74199
74916
|
}
|
|
74917
|
+
if (segment === void 0 && idle === void 0 && row2.recovery !== void 0) {
|
|
74918
|
+
const leftovers = await deps.instances.listEvents({ instanceId: instance.id, unit: row2.unit }, true).catch(() => []);
|
|
74919
|
+
if (leftovers.length > 0)
|
|
74920
|
+
(deps.log ?? console.warn)(
|
|
74921
|
+
`[coordinator] ${instance.id} ${row2.unit}: ${leftovers.length} recovery event(s) stay unconsumed \u2014 terminal recovery never redispatches a replacement pipeline`
|
|
74922
|
+
);
|
|
74923
|
+
}
|
|
74200
74924
|
let told = false;
|
|
74201
74925
|
if (io && threadReport.length === 0)
|
|
74202
74926
|
told = true;
|
|
@@ -74266,10 +74990,15 @@ async function rebaseStep(body, deps) {
|
|
|
74266
74990
|
if (!row2) return json3(404, { ok: false, error: "unknown_unit", at });
|
|
74267
74991
|
if (row2.pr?.number !== void 0 && row2.pr.number !== body.prNumber)
|
|
74268
74992
|
return json3(409, { ok: false, error: "pull_request_moved", at });
|
|
74269
|
-
|
|
74270
|
-
|
|
74271
|
-
|
|
74272
|
-
|
|
74993
|
+
try {
|
|
74994
|
+
if (deps.runnerOwnership?.claim(instance.repo, body.prNumber, {
|
|
74995
|
+
instanceId: instance.id,
|
|
74996
|
+
unit: row2.unit
|
|
74997
|
+
}) !== true)
|
|
74998
|
+
return json3(409, { ok: false, error: "publication_ownership_changed", at });
|
|
74999
|
+
} catch (err2) {
|
|
75000
|
+
return json3(503, { ok: false, error: "publication_ownership_unknown", message: describe12(err2), at });
|
|
75001
|
+
}
|
|
74273
75002
|
if (deps.runnerRebase === void 0)
|
|
74274
75003
|
return json3(200, { ok: true, outcome: "refused", reason: "the runner's rebase resolver is unavailable", at });
|
|
74275
75004
|
try {
|
|
@@ -74431,7 +75160,7 @@ async function merge(body, deps, subject2) {
|
|
|
74431
75160
|
});
|
|
74432
75161
|
if (checks.failed.length > 0) return refused7(`CI is red at \`${headSha.slice(0, 7)}\`: ${checks.failed.join(", ")}`);
|
|
74433
75162
|
if (checks.total === 0) {
|
|
74434
|
-
deps.noteMergeWait?.(headSha, id.value, at);
|
|
75163
|
+
deps.noteMergeWait?.(headSha, unit.row?.recovery?.workflowId ?? id.value, at);
|
|
74435
75164
|
return json3(200, {
|
|
74436
75165
|
ok: true,
|
|
74437
75166
|
outcome: "pending",
|
|
@@ -74440,7 +75169,7 @@ async function merge(body, deps, subject2) {
|
|
|
74440
75169
|
});
|
|
74441
75170
|
}
|
|
74442
75171
|
if (checks.pending.length > 0) {
|
|
74443
|
-
deps.noteMergeWait?.(headSha, id.value, at);
|
|
75172
|
+
deps.noteMergeWait?.(headSha, row2.recovery?.workflowId ?? id.value, at);
|
|
74444
75173
|
return json3(200, {
|
|
74445
75174
|
ok: true,
|
|
74446
75175
|
outcome: "pending",
|
|
@@ -74550,7 +75279,7 @@ async function checksStep(body, deps) {
|
|
|
74550
75279
|
};
|
|
74551
75280
|
}
|
|
74552
75281
|
if (checks === void 0 || checks.pending.length > 0 || (checks.expected?.length ?? 0) > 0 || checks.total === 0 || facts2?.draft === true)
|
|
74553
|
-
deps.noteMergeWait?.(headSha, id.value, at);
|
|
75282
|
+
deps.noteMergeWait?.(headSha, unit.row?.recovery?.workflowId ?? id.value, at);
|
|
74554
75283
|
if (checks !== void 0 && checks.failed.length > 0)
|
|
74555
75284
|
log(
|
|
74556
75285
|
`[coordinator] ${instance.id} ${body.unit}: CI red at ${headSha.slice(0, 7)} \u2014 ${checks.failed.map((f) => `${f.name} (${f.conclusion}${f.flakeSuspect === true ? ", suspected flake" : ""})`).join(", ")}`
|
|
@@ -74755,6 +75484,8 @@ async function answerCoordinatorStep(door, body, deps) {
|
|
|
74755
75484
|
return steerChild(parsed.value, deps);
|
|
74756
75485
|
case "pr-check":
|
|
74757
75486
|
return prCheck(parsed.value, deps);
|
|
75487
|
+
case "recover-unit":
|
|
75488
|
+
return recoverOriginalUnit(parsed.value, deps);
|
|
74758
75489
|
case "round":
|
|
74759
75490
|
return round(parsed.value, deps);
|
|
74760
75491
|
case "unit-end":
|
|
@@ -74800,7 +75531,7 @@ function createAdminCoordinatorHandler(deps) {
|
|
|
74800
75531
|
})();
|
|
74801
75532
|
};
|
|
74802
75533
|
}
|
|
74803
|
-
var COORDINATOR_ADMIN_PREFIX, MAX_SPAWN_PROMPT_CHARS, FINISHED_LOOKBACK_PAGES, MAX_ADMIN_BODY_BYTES, UNIT_ID, PRESETS_OF_KIND, invalid, json3, describe12, isGenerated2, EVERY_RUN, REVIEW_POSTED_CHECKS, REVIEW_POSTED_RECHECK_MS, githubSecondStart, CHILD_SUPERSESSION_REASONS, PublicationBindingRefusal, samePublicationOwner, ROUND_OUTCOMES, ENDING_ICON, STEPS;
|
|
75534
|
+
var COORDINATOR_ADMIN_PREFIX, MAX_SPAWN_PROMPT_CHARS, FINISHED_LOOKBACK_PAGES, MAX_ADMIN_BODY_BYTES, UNIT_ID, PRESETS_OF_KIND, invalid, json3, describe12, isGenerated2, EVERY_RUN, REVIEW_POSTED_CHECKS, REVIEW_POSTED_RECHECK_MS, githubSecondStart, CHILD_SUPERSESSION_REASONS, PublicationBindingRefusal, samePublicationOwner, fullHead, ROUND_OUTCOMES, ENDING_ICON, STEPS;
|
|
74804
75535
|
var init_adminCoordinator = __esm({
|
|
74805
75536
|
"../../src/channels/adminCoordinator.ts"() {
|
|
74806
75537
|
"use strict";
|
|
@@ -74864,6 +75595,7 @@ var init_adminCoordinator = __esm({
|
|
|
74864
75595
|
reason;
|
|
74865
75596
|
};
|
|
74866
75597
|
samePublicationOwner = (left, right) => left?.instanceId === right.instanceId && left.unit === right.unit;
|
|
75598
|
+
fullHead = (value) => typeof value === "string" && /^[0-9a-f]{40}$/i.test(value);
|
|
74867
75599
|
ROUND_OUTCOMES = [
|
|
74868
75600
|
"started",
|
|
74869
75601
|
"pr_opened",
|
|
@@ -74897,6 +75629,7 @@ var init_adminCoordinator = __esm({
|
|
|
74897
75629
|
"read-record",
|
|
74898
75630
|
"steer",
|
|
74899
75631
|
"pr-check",
|
|
75632
|
+
"recover-unit",
|
|
74900
75633
|
"round",
|
|
74901
75634
|
"unit-end",
|
|
74902
75635
|
"unit-wake",
|
|
@@ -76177,7 +76910,7 @@ async function runBot() {
|
|
|
76177
76910
|
workflow: checksWorkflow,
|
|
76178
76911
|
now: systemClock
|
|
76179
76912
|
});
|
|
76180
|
-
const
|
|
76913
|
+
const coordinatorDeps = {
|
|
76181
76914
|
tokens: processSecrets.get("SWITCHBOARD_INGRESS_TOKENS"),
|
|
76182
76915
|
childAdmission: coordinatorChildAdmission,
|
|
76183
76916
|
grantsFor: (id) => config.grantsFor(id),
|
|
@@ -76287,8 +77020,20 @@ async function runBot() {
|
|
|
76287
77020
|
return classifyRoundChecks(runs, changed, required);
|
|
76288
77021
|
},
|
|
76289
77022
|
rerunFailedChecks: rerunFailedJobs,
|
|
76290
|
-
refirePullRequest: refirePullRequestEvent
|
|
76291
|
-
|
|
77023
|
+
refirePullRequest: refirePullRequestEvent,
|
|
77024
|
+
startRecovery: (id, params) => createInstanceViaShim(processShimOptions(), id, params),
|
|
77025
|
+
recoveryStatus: (id) => fetchInstanceStatusViaShim(processShimOptions(), id)
|
|
77026
|
+
};
|
|
77027
|
+
const coordinatorAdmin = createAdminCoordinatorHandler(coordinatorDeps);
|
|
77028
|
+
deps.recoverOriginalUnit = async (key, caller) => {
|
|
77029
|
+
const answer = await recoverOriginalUnit(
|
|
77030
|
+
{ parentInstanceId: key.instanceId, unit: key.unit },
|
|
77031
|
+
coordinatorDeps,
|
|
77032
|
+
caller
|
|
77033
|
+
);
|
|
77034
|
+
const body = typeof answer.body === "object" && answer.body !== null && !Array.isArray(answer.body) ? answer.body : { error: "original-unit recovery returned an unreadable answer" };
|
|
77035
|
+
return { status: answer.status, body };
|
|
77036
|
+
};
|
|
76292
77037
|
const cronArmed = Object.values(auth.tokens).some((id) => id.subject === "cron");
|
|
76293
77038
|
const schedulesState = `${SCHEDULES.length} schedule(s) on /runs (${capabilities.schedules ? `firings from ${config.config.schedules?.worker?.baseUrl}` : "no firing store"}; cron identity ${cronArmed ? "armed" : "NOT in SWITCHBOARD_INGRESS_TOKENS \u2014 scheduled runs fail closed"})`;
|
|
76294
77039
|
const webAssets = loadWebAssets(webDistDir(publicEnv(), PACKAGE_ROOT));
|