@coreplane/switchboard 1.261.3 → 1.262.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/deploy/cloudflare/coordinator.ts +18 -1
- package/dist/assets/deploy/cloudflare/worker.ts +4 -1
- package/dist/assets/deploy/cloudflare-memory/worker.ts +58 -10
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +1 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/core/coordinator/contract.ts +137 -4
- package/dist/assets/src/core/coordinator/driver.ts +192 -40
- package/dist/assets/src/core/refusal.ts +1 -0
- package/dist/assets/src/core/runEvents.ts +2 -0
- package/dist/assets/src/core/ship/coordinator.ts +91 -15
- package/dist/cli.js +894 -75
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -3967,7 +3967,7 @@ function matchDispositions(findings, dispositions) {
|
|
|
3967
3967
|
const dropped = [...new Set(dispositions.filter((d) => !issued.has(d.findingId)).map((d) => d.findingId))];
|
|
3968
3968
|
return { matched, dropped };
|
|
3969
3969
|
}
|
|
3970
|
-
var MIN, PLAN_ID_PATTERN, PLAN_BRANCH, UNIT_REF, SLUG_MAX, INSTANCE_ID_MAX, WAIT_MARGIN_MS, WAIT_CHUNK_MS, MERGE_WAIT_CHUNK_MS, BUSY_RETRY_MS;
|
|
3970
|
+
var MIN, PLAN_ID_PATTERN, PLAN_BRANCH, UNIT_REF, SLUG_MAX, INSTANCE_ID_MAX, WAIT_MARGIN_MS, WAIT_CHUNK_MS, MERGE_WAIT_CHUNK_MS, BUSY_RETRY_MS, stepPrefixOf;
|
|
3971
3971
|
var init_coordinator = __esm({
|
|
3972
3972
|
"../../src/core/ship/coordinator.ts"() {
|
|
3973
3973
|
"use strict";
|
|
@@ -3987,6 +3987,10 @@ var init_coordinator = __esm({
|
|
|
3987
3987
|
WAIT_CHUNK_MS = SHIP_WAIT.chunkMinutes * MIN;
|
|
3988
3988
|
MERGE_WAIT_CHUNK_MS = SHIP_WAIT.mergeChunkMinutes * MIN;
|
|
3989
3989
|
BUSY_RETRY_MS = SHIP_WAIT.busyRetryMinutes * MIN;
|
|
3990
|
+
stepPrefixOf = (unit, session) => {
|
|
3991
|
+
const segment = session !== void 0 && session.segment > 1 ? `${unit}/s${session.segment}` : unit;
|
|
3992
|
+
return session?.resume !== void 0 ? `${segment}/r${session.resume.attempt}` : segment;
|
|
3993
|
+
};
|
|
3990
3994
|
}
|
|
3991
3995
|
});
|
|
3992
3996
|
|
|
@@ -5804,6 +5808,13 @@ function isCoordinatorUnit(v) {
|
|
|
5804
5808
|
(x) => isObject(x) && isFinite(x.index) && isText(x.agent) && isText(x.outcome) && isFinite(x.at) && (x.gate === void 0 || isRoundGate(x.gate))
|
|
5805
5809
|
))
|
|
5806
5810
|
return false;
|
|
5811
|
+
if (r.recovery !== void 0 && !(isObject(r.recovery) && (r.recovery.kind === "findings" || r.recovery.kind === "review") && typeof r.recovery.round === "number" && Number.isInteger(r.recovery.round) && r.recovery.round >= 1 && typeof r.recovery.expectedHeadSha === "string" && /^[0-9a-f]{40}$/i.test(r.recovery.expectedHeadSha) && isFinite(r.recovery.remainingMs) && r.recovery.remainingMs > 0 && isFinite(r.recovery.claimedAt) && typeof r.recovery.step === "string" && STEP_NAME_PATTERN.test(r.recovery.step) && isText(r.recovery.reviewRunId) && (r.recovery.findingsRunId === void 0 || isText(r.recovery.findingsRunId)) && (r.recovery.findingsKey === void 0 || isText(r.recovery.findingsKey)) && (r.recovery.findingsRunId === void 0 && r.recovery.findingsKey === void 0 || r.recovery.kind === "review" && r.recovery.findingsRunId !== void 0 && r.recovery.findingsKey !== void 0) && (r.recovery.findings === void 0 || Array.isArray(r.recovery.findings) && r.recovery.findings.every(isFindingShape)) && isObject(r.recovery.previousEnding) && isText(r.recovery.previousEnding.kind) && typeof r.recovery.previousEnding.report === "string" && isFinite(r.recovery.previousEnding.at) && typeof r.recovery.workflowId === "string" && INSTANCE_ID_PATTERN.test(r.recovery.workflowId) && isFinite(r.recovery.deadlineAt) && isText(r.recovery.reviewKey)))
|
|
5812
|
+
return false;
|
|
5813
|
+
if (r.recoveryReceipt !== void 0 && (!isObject(r.recoveryReceipt) || !isText(r.recoveryReceipt.reviewRunId) || typeof r.recoveryReceipt.workflowId !== "string" || !INSTANCE_ID_PATTERN.test(r.recoveryReceipt.workflowId) || !isFinite(r.recoveryReceipt.at)))
|
|
5814
|
+
return false;
|
|
5815
|
+
if (r.recoveryHold !== void 0 && (!isObject(r.recoveryHold) || !(r.recoveryHold.cause === "human" && isHumanGatePending(r.recoveryHold.gate) || r.recoveryHold.cause === "draft" && isObject(r.recoveryHold.pr) && Number.isInteger(r.recoveryHold.pr.number) && r.recoveryHold.pr.number > 0 && isText(r.recoveryHold.pr.url))))
|
|
5816
|
+
return false;
|
|
5817
|
+
if (r.recovery !== void 0 && (r.idle !== void 0 || r.ending !== void 0)) return false;
|
|
5807
5818
|
if (r.ending !== void 0 && !(isObject(r.ending) && isText(r.ending.kind) && isText(r.ending.report, MAX_REPORT) && isFinite(r.ending.at) && (r.ending.cause === void 0 || isText(r.ending.cause, 64)) && (r.ending.step === void 0 || typeof r.ending.step === "string" && STEP_NAME_PATTERN.test(r.ending.step)) && (r.ending.round === void 0 || typeof r.ending.round === "number" && Number.isInteger(r.ending.round) && r.ending.round >= 0)))
|
|
5808
5819
|
return false;
|
|
5809
5820
|
if (r.startedAt !== void 0 && !isFinite(r.startedAt)) return false;
|
|
@@ -5850,7 +5861,7 @@ async function sendChildSignal(workflow, signal2) {
|
|
|
5850
5861
|
if (!workflow) return { kind: "no-binding", instance };
|
|
5851
5862
|
const type = kind === "interrupted" ? childInterruptedEventType(runId3) : childResumedEventType(runId3);
|
|
5852
5863
|
try {
|
|
5853
|
-
const handle2 = await workflow.get(instance);
|
|
5864
|
+
const handle2 = await workflow.get(signal2.transportWorkflowId ?? instance);
|
|
5854
5865
|
await handle2.sendEvent({ type, payload: { runId: runId3, kind, reason: reason2, at, parentInstanceId: instance } });
|
|
5855
5866
|
return { kind: "sent", instance, type };
|
|
5856
5867
|
} catch (err2) {
|
|
@@ -15534,6 +15545,7 @@ var init_refusal = __esm({
|
|
|
15534
15545
|
// one pipeline per thread (record 0060): the host key's claim answered thread-live
|
|
15535
15546
|
ship_thread_live: "request",
|
|
15536
15547
|
setup_failed: "system",
|
|
15548
|
+
publication_ownership_unknown: "system",
|
|
15537
15549
|
// the click on a confirmation (confirm.ts)
|
|
15538
15550
|
confirmation_used: "request",
|
|
15539
15551
|
confirmation_expired: "request",
|
|
@@ -34687,12 +34699,15 @@ var init_instanceStore = __esm({
|
|
|
34687
34699
|
async putUnits(_units) {
|
|
34688
34700
|
return { ok: false, reason: "unavailable" };
|
|
34689
34701
|
}
|
|
34690
|
-
async
|
|
34702
|
+
async compareAndReplaceUnit(_expected, _replacement) {
|
|
34691
34703
|
return { ok: false, reason: "unavailable" };
|
|
34692
34704
|
}
|
|
34693
34705
|
async listUnits(_instanceId) {
|
|
34694
34706
|
return [];
|
|
34695
34707
|
}
|
|
34708
|
+
async listActiveRecoveries() {
|
|
34709
|
+
return [];
|
|
34710
|
+
}
|
|
34696
34711
|
async reserveDecisionRecord(_repo, _taskKey, _claimed, _existing) {
|
|
34697
34712
|
return { ok: false, reason: "unavailable" };
|
|
34698
34713
|
}
|
|
@@ -34764,8 +34779,11 @@ var init_instanceStore = __esm({
|
|
|
34764
34779
|
if (d.ok === true) return { ok: true };
|
|
34765
34780
|
throw new Error(`coordinator store /runs/coordinator/units/put: unexpected answer (HTTP ${r.status})`);
|
|
34766
34781
|
}
|
|
34767
|
-
async
|
|
34768
|
-
const r = await this.post("/runs/coordinator/units/claim-legacy-continuation", {
|
|
34782
|
+
async compareAndReplaceUnit(expected, replacement) {
|
|
34783
|
+
const r = await this.post("/runs/coordinator/units/claim-legacy-continuation", {
|
|
34784
|
+
expected,
|
|
34785
|
+
recovered: replacement
|
|
34786
|
+
});
|
|
34769
34787
|
const d = r.data;
|
|
34770
34788
|
if (r.status === 409 && d.reason === "stale") return { ok: false, reason: "stale" };
|
|
34771
34789
|
if (d.ok === true) return { ok: true };
|
|
@@ -34780,6 +34798,15 @@ var init_instanceStore = __esm({
|
|
|
34780
34798
|
throw new Error("coordinator store /runs/coordinator/units/list: the answer is not a list of unit rows");
|
|
34781
34799
|
return d.units;
|
|
34782
34800
|
}
|
|
34801
|
+
async listActiveRecoveries() {
|
|
34802
|
+
const r = await this.post("/runs/coordinator/units/list-active-recoveries", {});
|
|
34803
|
+
const d = r.data;
|
|
34804
|
+
if (!Array.isArray(d.units) || !d.units.every(isCoordinatorUnit) || d.units.some((unit) => unit.recovery === void 0))
|
|
34805
|
+
throw new Error(
|
|
34806
|
+
"coordinator store /runs/coordinator/units/list-active-recoveries: the answer is not a recovery list"
|
|
34807
|
+
);
|
|
34808
|
+
return d.units;
|
|
34809
|
+
}
|
|
34783
34810
|
async reserveDecisionRecord(repo, taskKey, claimed2, existing) {
|
|
34784
34811
|
const r = await this.post("/runs/decision-record/reserve", {
|
|
34785
34812
|
repo,
|
|
@@ -46181,6 +46208,8 @@ async function claimRun(deps, ctx) {
|
|
|
46181
46208
|
type: "coordinator_tag",
|
|
46182
46209
|
parentInstanceId: coordinator.parentInstanceId,
|
|
46183
46210
|
...unit !== void 0 ? { unit } : {},
|
|
46211
|
+
...coordinator.transportWorkflowId !== void 0 ? { transportWorkflowId: coordinator.transportWorkflowId } : {},
|
|
46212
|
+
...coordinator.recovery !== void 0 ? { recovery: coordinator.recovery } : {},
|
|
46184
46213
|
...coordinator.base !== void 0 ? { base: coordinator.base } : {},
|
|
46185
46214
|
...coordinator.publication !== void 0 ? { publication: coordinator.publication } : {},
|
|
46186
46215
|
at: clock4()
|
|
@@ -54981,7 +55010,7 @@ function shimAddress(opts) {
|
|
|
54981
55010
|
};
|
|
54982
55011
|
return { base, bearer };
|
|
54983
55012
|
}
|
|
54984
|
-
async function createInstanceViaShim(opts, id) {
|
|
55013
|
+
async function createInstanceViaShim(opts, id, params = {}) {
|
|
54985
55014
|
const at = shimAddress(opts);
|
|
54986
55015
|
if ("reason" in at) return { kind: "unanswered", reason: at.reason };
|
|
54987
55016
|
const fetchImpl = opts.fetch ?? fetch;
|
|
@@ -54989,7 +55018,7 @@ async function createInstanceViaShim(opts, id) {
|
|
|
54989
55018
|
const res = await fetchImpl(`${at.base}${COORDINATOR_INSTANCES_PATH}`, {
|
|
54990
55019
|
method: "POST",
|
|
54991
55020
|
headers: { authorization: `Bearer ${at.bearer}`, "content-type": "application/json" },
|
|
54992
|
-
body: JSON.stringify({ id, params
|
|
55021
|
+
body: JSON.stringify({ id, params }),
|
|
54993
55022
|
signal: AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS)
|
|
54994
55023
|
});
|
|
54995
55024
|
return readCreateInstanceAnswer(res.status, await res.text().catch(() => ""));
|
|
@@ -55049,11 +55078,51 @@ var init_instancesClient = __esm({
|
|
|
55049
55078
|
});
|
|
55050
55079
|
|
|
55051
55080
|
// ../../src/core/dispatch/ship.ts
|
|
55081
|
+
function parseOriginalUnitRecoveryRequest(text) {
|
|
55082
|
+
const match = /^recover\s+unit\s+(\S+)\s*$/i.exec(text.trim());
|
|
55083
|
+
return match ? parseUnitKey(match[1]) : void 0;
|
|
55084
|
+
}
|
|
55052
55085
|
async function runShipBranch(deps, msg, io, ctx) {
|
|
55053
55086
|
const { agent, profile, card, directives, history, repoCtx, label, ending, trace, closeLines, refuse: refuse3, doneLines } = ctx;
|
|
55054
55087
|
const root = trace.root;
|
|
55055
55088
|
const clock4 = deps.clock ?? systemClock;
|
|
55056
55089
|
const shell = createCardShell({ label, startedAt: ctx.startedAt, now: clock4 });
|
|
55090
|
+
const recovery = parseOriginalUnitRecoveryRequest(directives.text);
|
|
55091
|
+
if (recovery === void 0 && namesOriginalUnitRecovery(directives.text)) {
|
|
55092
|
+
const reason2 = "original-unit recovery must be `recover unit <instanceId>:<unit>`";
|
|
55093
|
+
await refuse3(
|
|
55094
|
+
refusalOf("setup_failed", reason2),
|
|
55095
|
+
() => card.done(shell.close({ kind: "refused", icon: "\u{1F6AB}", reason: reason2, ...closeLines(clock4(), false) }))
|
|
55096
|
+
);
|
|
55097
|
+
return { hostedLive: false };
|
|
55098
|
+
}
|
|
55099
|
+
if (recovery !== void 0) {
|
|
55100
|
+
const answer = await root.span(
|
|
55101
|
+
"dispatch.ship_recover_original_unit",
|
|
55102
|
+
() => deps.recoverOriginalUnit === void 0 ? Promise.resolve({ status: 503, body: { error: "original-unit recovery is unavailable" } }) : deps.recoverOriginalUnit(recovery, { userId: msg.userId, threadKey: msg.threadKey })
|
|
55103
|
+
);
|
|
55104
|
+
if (answer.status !== 200) {
|
|
55105
|
+
const reason2 = typeof answer.body.error === "string" ? answer.body.error : "original-unit recovery was refused";
|
|
55106
|
+
await refuse3(
|
|
55107
|
+
refusalOf("setup_failed", reason2),
|
|
55108
|
+
() => card.done(shell.close({ kind: "refused", icon: "\u{1F6AB}", reason: reason2, ...closeLines(clock4(), false) }))
|
|
55109
|
+
);
|
|
55110
|
+
return { hostedLive: false };
|
|
55111
|
+
}
|
|
55112
|
+
const workflowId = typeof answer.body.workflowId === "string" ? answer.body.workflowId : "unknown";
|
|
55113
|
+
const outcome2 = answer.body.outcome === "already_started" ? "already running" : answer.body.outcome === "indeterminate" ? "has an indeterminate start; its claim remains held for same-checkpoint replay" : "started";
|
|
55114
|
+
const text = `Original unit \`${recovery.instanceId}:${recovery.unit}\` recovery ${outcome2} as durable checkpoint \`${workflowId}\`.`;
|
|
55115
|
+
await replyAck(io, ctx.verbosity, text);
|
|
55116
|
+
await card.done(
|
|
55117
|
+
shell.close({
|
|
55118
|
+
kind: "done",
|
|
55119
|
+
icon: answer.body.outcome === "indeterminate" ? "\u26A0\uFE0F" : "\u2705",
|
|
55120
|
+
detail: text,
|
|
55121
|
+
...closeLines(clock4(), true)
|
|
55122
|
+
})
|
|
55123
|
+
);
|
|
55124
|
+
return { hostedLive: false };
|
|
55125
|
+
}
|
|
55057
55126
|
const listSlugs = deps.residentSlugs ?? residentSlugsLister(deps.config.config.execution?.resident);
|
|
55058
55127
|
const repoCandidates = repoCtx.repo ? void 0 : await listSlugs?.().catch(() => void 0);
|
|
55059
55128
|
const pre = await root.span(
|
|
@@ -55462,6 +55531,7 @@ function handOffAck(outcome, verbosity2) {
|
|
|
55462
55531
|
return outcome.instanceId !== void 0 && shows(verbosity2, "debug") ? `${outcome.reply}
|
|
55463
55532
|
\u2022 runner instance \`${outcome.instanceId}\`` : outcome.reply;
|
|
55464
55533
|
}
|
|
55534
|
+
var namesOriginalUnitRecovery;
|
|
55465
55535
|
var init_ship = __esm({
|
|
55466
55536
|
"../../src/core/dispatch/ship.ts"() {
|
|
55467
55537
|
"use strict";
|
|
@@ -55474,6 +55544,7 @@ var init_ship = __esm({
|
|
|
55474
55544
|
init_budgets();
|
|
55475
55545
|
init_instancesClient();
|
|
55476
55546
|
init_instanceStore();
|
|
55547
|
+
init_contract2();
|
|
55477
55548
|
init_shipPipeline();
|
|
55478
55549
|
init_preflight();
|
|
55479
55550
|
init_runEvents();
|
|
@@ -55492,6 +55563,7 @@ var init_ship = __esm({
|
|
|
55492
55563
|
init_reply();
|
|
55493
55564
|
init_verbosity();
|
|
55494
55565
|
init_reply();
|
|
55566
|
+
namesOriginalUnitRecovery = (text) => /^recover\s+unit(?:\s|$)/i.test(text.trim());
|
|
55495
55567
|
}
|
|
55496
55568
|
});
|
|
55497
55569
|
|
|
@@ -55589,9 +55661,13 @@ function carriedCoordinatorTag(row2, events) {
|
|
|
55589
55661
|
const tag = events.find((e) => e.type === "coordinator_tag");
|
|
55590
55662
|
const base = tag?.type === "coordinator_tag" ? tag.base : void 0;
|
|
55591
55663
|
const publication = tag?.type === "coordinator_tag" ? tag.publication : void 0;
|
|
55664
|
+
const transportWorkflowId = tag?.type === "coordinator_tag" ? tag.transportWorkflowId : void 0;
|
|
55665
|
+
const recovery = tag?.type === "coordinator_tag" ? tag.recovery : void 0;
|
|
55592
55666
|
return {
|
|
55593
55667
|
parentInstanceId,
|
|
55594
55668
|
idempotencyKey,
|
|
55669
|
+
...transportWorkflowId !== void 0 ? { transportWorkflowId } : {},
|
|
55670
|
+
...recovery !== void 0 ? { recovery } : {},
|
|
55595
55671
|
...base !== void 0 ? { base } : {},
|
|
55596
55672
|
...publication !== void 0 ? { publication } : {}
|
|
55597
55673
|
};
|
|
@@ -55614,6 +55690,7 @@ async function announceChildRoll(ctx) {
|
|
|
55614
55690
|
const sent = await sendChildSignal(workflow, {
|
|
55615
55691
|
runId: runId3,
|
|
55616
55692
|
parentInstanceId: coordinator.parentInstanceId,
|
|
55693
|
+
...coordinator.transportWorkflowId !== void 0 ? { transportWorkflowId: coordinator.transportWorkflowId } : {},
|
|
55617
55694
|
kind,
|
|
55618
55695
|
reason: reason2,
|
|
55619
55696
|
at
|
|
@@ -55686,6 +55763,7 @@ async function recordRestartDeath(ctx) {
|
|
|
55686
55763
|
const sent = await sendChildSignal(workflow, {
|
|
55687
55764
|
runId: closed.id,
|
|
55688
55765
|
parentInstanceId: coordinator.parentInstanceId,
|
|
55766
|
+
...coordinator.transportWorkflowId !== void 0 ? { transportWorkflowId: coordinator.transportWorkflowId } : {},
|
|
55689
55767
|
kind: "interrupted",
|
|
55690
55768
|
reason: summary,
|
|
55691
55769
|
at
|
|
@@ -56067,14 +56145,49 @@ async function answerUnitOwnedThread(deps, ctx) {
|
|
|
56067
56145
|
if (!appended.ok) return "route-fresh";
|
|
56068
56146
|
try {
|
|
56069
56147
|
if (deps.workflow === void 0) throw new Error("no workflow sender in this process");
|
|
56070
|
-
const
|
|
56148
|
+
const workflowId = owner.unit.recovery?.workflowId ?? owner.instanceId;
|
|
56149
|
+
const handle2 = await deps.workflow.get(workflowId);
|
|
56071
56150
|
await handle2.sendEvent({ type: unitNudgeEventType(key), payload: {} });
|
|
56072
56151
|
} catch (err2) {
|
|
56073
56152
|
const readStatus = deps.fetchCoordinatorInstanceStatus ?? ((id) => fetchInstanceStatusViaShim(processShimOptions(), id));
|
|
56074
|
-
const
|
|
56153
|
+
const workflowId = owner.unit.recovery?.workflowId ?? owner.instanceId;
|
|
56154
|
+
const status3 = await readStatus(workflowId);
|
|
56075
56155
|
const gone = status3.kind === "absent" || status3.kind === "status" && ["complete", "errored", "terminated"].includes(status3.status);
|
|
56076
56156
|
if (gone) {
|
|
56077
56157
|
const why = status3.kind === "absent" ? "no such instance" : status3.status;
|
|
56158
|
+
if (owner.unit.recovery !== void 0) {
|
|
56159
|
+
const recovery = owner.unit.recovery;
|
|
56160
|
+
const recoveryInstance = await store.get(owner.instanceId).catch(() => null);
|
|
56161
|
+
const { recovery: _recovery, ...withoutRecovery } = owner.unit;
|
|
56162
|
+
const closed = {
|
|
56163
|
+
...withoutRecovery,
|
|
56164
|
+
recoveryReceipt: { reviewRunId: recovery.reviewRunId, workflowId: recovery.workflowId, at },
|
|
56165
|
+
ending: {
|
|
56166
|
+
kind: "terminated",
|
|
56167
|
+
report: `the original-unit recovery checkpoint is gone (${why}); no replacement pipeline was started`,
|
|
56168
|
+
at
|
|
56169
|
+
}
|
|
56170
|
+
};
|
|
56171
|
+
const replaced = await store.compareAndReplaceUnit(owner.unit, closed).catch(() => void 0);
|
|
56172
|
+
if (replaced?.ok !== true) {
|
|
56173
|
+
await replyAck(
|
|
56174
|
+
io,
|
|
56175
|
+
ctx.verbosity,
|
|
56176
|
+
`\u{1F4CC} Noted for unit ${owner.unit.unit} (\`${unitKey}\`): its recovery checkpoint is gone, but the durable claim could not be closed safely \u2014 your message remains queued and no replacement ran.`
|
|
56177
|
+
);
|
|
56178
|
+
return "acked";
|
|
56179
|
+
}
|
|
56180
|
+
if (recoveryInstance !== null && owner.unit.pr !== void 0)
|
|
56181
|
+
deps.runnerOwnership?.release(recoveryInstance.repo, owner.unit.pr.number, {
|
|
56182
|
+
instanceId: owner.instanceId,
|
|
56183
|
+
unit: owner.unit.unit
|
|
56184
|
+
});
|
|
56185
|
+
await store.markConsumed(key, [appended.seq], `recovery-terminal:${recovery.workflowId}`).catch(() => void 0);
|
|
56186
|
+
await io.reply(
|
|
56187
|
+
`\u26A0\uFE0F Unit ${owner.unit.unit}'s original-unit recovery checkpoint \`${recovery.workflowId}\` is gone (${why}). The claim is closed; this message did not start replacement work.`
|
|
56188
|
+
);
|
|
56189
|
+
return "acked";
|
|
56190
|
+
}
|
|
56078
56191
|
await store.putUnits([
|
|
56079
56192
|
{
|
|
56080
56193
|
...owner.unit,
|
|
@@ -56132,16 +56245,22 @@ function continuationBudgetOf(instance, unit) {
|
|
|
56132
56245
|
}
|
|
56133
56246
|
async function legacyReviewedHeadOf(runs, instance, unit, prNumber) {
|
|
56134
56247
|
const listed = await runs.listUnitRuns(unitKeyOf({ instanceId: instance.id, unit: unit.unit }), { kind: "all" }).catch(() => void 0);
|
|
56135
|
-
if (listed === void 0 || !listed.ok || listed.value.unit !== unitKeyOf({ instanceId: instance.id, unit: unit.unit }) || listed.value.instanceId !== instance.id || listed.value.id !== unit.unit || listed.value.instance.id !== instance.id || listed.value.instance.repo.toLowerCase() !== instance.repo.toLowerCase())
|
|
56248
|
+
if (listed === void 0 || !listed.ok || listed.value.unit !== unitKeyOf({ instanceId: instance.id, unit: unit.unit }) || listed.value.instanceId !== instance.id || listed.value.id !== unit.unit || listed.value.branch !== unit.branch || listed.value.pr?.number !== prNumber || listed.value.instance.id !== instance.id || listed.value.instance.repo.toLowerCase() !== instance.repo.toLowerCase())
|
|
56136
56249
|
return void 0;
|
|
56137
56250
|
const prefix = `${instance.id}:${unit.unit}/`;
|
|
56138
56251
|
const owned = listed.value.runs.filter(
|
|
56139
56252
|
(run2) => run2.finished && run2.status === "completed" && run2.parentInstanceId === instance.id && run2.idempotencyKey?.startsWith(prefix) === true && run2.userId === instance.userId && run2.repo?.toLowerCase() === instance.repo.toLowerCase()
|
|
56140
56253
|
);
|
|
56141
56254
|
const coding = new Set(
|
|
56142
|
-
owned.flatMap(
|
|
56143
|
-
(run2
|
|
56144
|
-
|
|
56255
|
+
owned.flatMap((run2) => {
|
|
56256
|
+
if (run2.agent !== "coding" || run2.pr?.number !== prNumber) return [];
|
|
56257
|
+
return [
|
|
56258
|
+
...typeof run2.headSha === "string" && FULL_SHA2.test(run2.headSha) ? [run2.headSha.toLowerCase()] : [],
|
|
56259
|
+
...(run2.pushed ?? []).flatMap(
|
|
56260
|
+
(pushed) => pushed.ref === unit.branch && FULL_SHA2.test(pushed.sha) ? [pushed.sha.toLowerCase()] : []
|
|
56261
|
+
)
|
|
56262
|
+
];
|
|
56263
|
+
})
|
|
56145
56264
|
);
|
|
56146
56265
|
const approved = new Set(
|
|
56147
56266
|
owned.flatMap((run2) => {
|
|
@@ -56157,6 +56276,8 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56157
56276
|
const ended = { status: "completed" };
|
|
56158
56277
|
const resume = opts.resume;
|
|
56159
56278
|
const restart = opts.restart;
|
|
56279
|
+
const coordinator = opts.coordinator ?? (resume ? carriedCoordinatorTag(resume.row, resume.events) : void 0);
|
|
56280
|
+
const recovery = opts.recovery ?? coordinator?.recovery;
|
|
56160
56281
|
const clock4 = deps.clock ?? systemClock;
|
|
56161
56282
|
const trace = opts.trace ?? startRequestRoot(deps, {
|
|
56162
56283
|
channel: channelOf(msg.channelId),
|
|
@@ -56413,6 +56534,13 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56413
56534
|
return ended;
|
|
56414
56535
|
}
|
|
56415
56536
|
if (owner.kind === "pipeline" && directives.agent === void 0) {
|
|
56537
|
+
if (owner.unit.recoveryReceipt !== void 0 || owner.unit.recoveryHold !== void 0) {
|
|
56538
|
+
await io.reply(
|
|
56539
|
+
`The original-unit recovery for \`${owner.instanceId}:${owner.unit.unit}\` is terminal and its evidence is consumed. This reply was not turned into replacement work.`
|
|
56540
|
+
);
|
|
56541
|
+
await recordPendingOperator();
|
|
56542
|
+
return ended;
|
|
56543
|
+
}
|
|
56416
56544
|
const instance = await deps.coordinatorInstances.get(owner.instanceId).catch(() => null);
|
|
56417
56545
|
if (instance === null) {
|
|
56418
56546
|
await io.reply(
|
|
@@ -56594,7 +56722,14 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56594
56722
|
return true;
|
|
56595
56723
|
};
|
|
56596
56724
|
reserveLegacyOwnership = () => {
|
|
56597
|
-
|
|
56725
|
+
try {
|
|
56726
|
+
token2 = runnerOwnership.reserve(instance.repo, recordedPr.number, reservationOwner);
|
|
56727
|
+
} catch {
|
|
56728
|
+
return refusalOf(
|
|
56729
|
+
"publication_ownership_unknown",
|
|
56730
|
+
`Runner ownership for ${instance.repo}#${recordedPr.number} is being rebuilt, so continuation did not start. Nothing else ran.`
|
|
56731
|
+
);
|
|
56732
|
+
}
|
|
56598
56733
|
if (token2 !== void 0) return void 0;
|
|
56599
56734
|
const competing = runnerOwnership.owner(instance.repo, recordedPr.number);
|
|
56600
56735
|
const sameContinuation = competing?.instanceId === reservationOwner.instanceId && competing.unit === reservationOwner.unit;
|
|
@@ -56613,7 +56748,7 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56613
56748
|
`Unit ${legacyUnit.unit}'s ownership reservation was lost, so continuation did not start. Nothing else ran.`
|
|
56614
56749
|
)
|
|
56615
56750
|
};
|
|
56616
|
-
const persisted = await coordinatorInstances.
|
|
56751
|
+
const persisted = await coordinatorInstances.compareAndReplaceUnit(legacyUnit, recoveredUnit).catch(() => void 0);
|
|
56617
56752
|
if (persisted?.ok !== true) {
|
|
56618
56753
|
const released = releaseLegacyOwnership?.() ?? true;
|
|
56619
56754
|
const reason2 = persisted?.reason === "stale" ? `Unit ${legacyUnit.unit} changed while its legacy continuation was being reserved, so continuation did not start.` : `Unit ${legacyUnit.unit}'s recovered continuation binding could not be stored, so continuation did not start.`;
|
|
@@ -56622,7 +56757,7 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56622
56757
|
}
|
|
56623
56758
|
const reservedOwner = runnerOwnership.owner(instance.repo, recordedPr.number);
|
|
56624
56759
|
if (reservedOwner?.instanceId !== reservationOwner.instanceId || reservedOwner.unit !== reservationOwner.unit) {
|
|
56625
|
-
const restored = await coordinatorInstances.
|
|
56760
|
+
const restored = await coordinatorInstances.compareAndReplaceUnit(recoveredUnit, legacyUnit).catch(() => void 0);
|
|
56626
56761
|
const released = releaseLegacyOwnership?.() ?? true;
|
|
56627
56762
|
const rollbackFailure = restored?.ok === true ? void 0 : restored?.reason ?? "unavailable";
|
|
56628
56763
|
const cleanupFailure = released ? "" : " and its ownership reservation could not be released";
|
|
@@ -56648,7 +56783,7 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56648
56783
|
transferredOwner = void 0;
|
|
56649
56784
|
},
|
|
56650
56785
|
abort: async () => {
|
|
56651
|
-
const restored = await coordinatorInstances.
|
|
56786
|
+
const restored = await coordinatorInstances.compareAndReplaceUnit(recoveredUnit, legacyUnit).catch(() => void 0);
|
|
56652
56787
|
const released = releaseLegacyOwnership?.() ?? true;
|
|
56653
56788
|
owner.unit = legacyUnit;
|
|
56654
56789
|
const failures = [
|
|
@@ -56697,6 +56832,8 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56697
56832
|
if ((await authorizeAgent(deps, { msg, io, refuse: refuse3, agentName: resolved.agentName })).kind === "refused")
|
|
56698
56833
|
return ended;
|
|
56699
56834
|
const agent = resolved.agentName === "ship" ? shipPresetFor(deps.config.config.ship) : getAgent(resolved.agentName);
|
|
56835
|
+
const recoveryRemainingMs = recovery === void 0 ? void 0 : Math.max(0, recovery.deadlineAt - clock4());
|
|
56836
|
+
const inheritedRemainingMs = parent?.remainingMs === void 0 ? recoveryRemainingMs : recoveryRemainingMs === void 0 ? parent.remainingMs : Math.min(parent.remainingMs, recoveryRemainingMs);
|
|
56700
56837
|
const profileGate = await authorizeProfile(deps, {
|
|
56701
56838
|
msg,
|
|
56702
56839
|
io,
|
|
@@ -56707,14 +56844,14 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56707
56844
|
resolved,
|
|
56708
56845
|
resume,
|
|
56709
56846
|
budget: directives.budget,
|
|
56710
|
-
...
|
|
56847
|
+
...inheritedRemainingMs !== void 0 ? { parentRemainingMs: inheritedRemainingMs } : {}
|
|
56711
56848
|
})
|
|
56712
56849
|
});
|
|
56713
56850
|
if (profileGate.kind === "refused") return ended;
|
|
56714
|
-
|
|
56851
|
+
let { profile } = profileGate;
|
|
56715
56852
|
const legacyOwnershipRefusal = reserveLegacyOwnership?.();
|
|
56716
56853
|
if (legacyOwnershipRefusal !== void 0) {
|
|
56717
|
-
await
|
|
56854
|
+
await refuse3(legacyOwnershipRefusal);
|
|
56718
56855
|
await recordPendingOperator();
|
|
56719
56856
|
return ended;
|
|
56720
56857
|
}
|
|
@@ -56827,6 +56964,26 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56827
56964
|
if (references.refused.length > 0)
|
|
56828
56965
|
await renderRefusal(refusalOf(referenceRefusalCode(references.refused[0]), REFERENCE_REFUSAL), io);
|
|
56829
56966
|
let repoCtx = await repoCtxP;
|
|
56967
|
+
if (recovery !== void 0 && (agent.name === "review" || agent.name === "coding") && (repoCtx.repo?.toLowerCase() !== recovery.repo.toLowerCase() || repoCtx.pr !== recovery.pr || repoCtx.ref !== recovery.headRef || repoCtx.baseRef !== recovery.baseRef || repoCtx.headSha !== recovery.expectedHeadSha)) {
|
|
56968
|
+
const reason2 = "the recovered pull request moved or no longer matches its durable target";
|
|
56969
|
+
await refuse3(
|
|
56970
|
+
refusalOf("setup_failed", reason2),
|
|
56971
|
+
() => card.done(shell.close({ kind: "refused", icon: "\u{1F6AB}", reason: reason2, ...closeLines(clock4(), false) }))
|
|
56972
|
+
);
|
|
56973
|
+
return ended;
|
|
56974
|
+
}
|
|
56975
|
+
if (recovery !== void 0) {
|
|
56976
|
+
const remainingMinutes = Math.floor((recovery.deadlineAt - clock4()) / 6e4);
|
|
56977
|
+
if (remainingMinutes <= 0) {
|
|
56978
|
+
const reason2 = "the original unit's absolute recovery deadline expired during setup";
|
|
56979
|
+
await refuse3(
|
|
56980
|
+
refusalOf("setup_failed", reason2),
|
|
56981
|
+
() => card.done(shell.close({ kind: "refused", icon: "\u{1F6AB}", reason: reason2, ...closeLines(clock4(), false) }))
|
|
56982
|
+
);
|
|
56983
|
+
return ended;
|
|
56984
|
+
}
|
|
56985
|
+
profile = { ...profile, minutes: Math.min(profile.minutes, remainingMinutes) };
|
|
56986
|
+
}
|
|
56830
56987
|
const contractBranch = opts.contract?.rebase.branch;
|
|
56831
56988
|
if (opts.coordinator !== void 0 && contractBranch !== void 0 && agent.name !== "review" && repoCtx.refFromPr === true && repoCtx.ref !== contractBranch) {
|
|
56832
56989
|
const { refFromPr: _refFromPr, headSha: _headSha, ...kept2 } = repoCtx;
|
|
@@ -56959,7 +57116,6 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
56959
57116
|
});
|
|
56960
57117
|
if (headPreflight.kind === "refused") return ended;
|
|
56961
57118
|
const parentRunId = parent?.runId;
|
|
56962
|
-
const coordinator = opts.coordinator ?? (resume ? carriedCoordinatorTag(resume.row, resume.events) : void 0);
|
|
56963
57119
|
const registration = await registerRun(deps, {
|
|
56964
57120
|
msg,
|
|
56965
57121
|
io,
|
|
@@ -57323,6 +57479,14 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
57323
57479
|
});
|
|
57324
57480
|
if (headGate.kind === "refused") return ended;
|
|
57325
57481
|
repoCtx = headGate.repoCtx;
|
|
57482
|
+
if (recovery !== void 0 && (agent.name === "review" || agent.name === "coding") && (repoCtx.repo?.toLowerCase() !== recovery.repo.toLowerCase() || repoCtx.pr !== recovery.pr || repoCtx.ref !== recovery.headRef || repoCtx.baseRef !== recovery.baseRef || repoCtx.headSha !== recovery.expectedHeadSha)) {
|
|
57483
|
+
const reason2 = "the recovered pull request moved during setup and no longer matches its durable target";
|
|
57484
|
+
await refuse3(
|
|
57485
|
+
refusalOf("setup_failed", reason2),
|
|
57486
|
+
() => card.done(shell.close({ kind: "refused", icon: "\u{1F6AB}", reason: reason2, ...closeLines(clock4(), false) }))
|
|
57487
|
+
);
|
|
57488
|
+
return ended;
|
|
57489
|
+
}
|
|
57326
57490
|
const verifiedAtAttach = headGate.verifiedAtAttach;
|
|
57327
57491
|
if (headGate.headAdopted) publishMeta(repoCtx);
|
|
57328
57492
|
const bound = round2.selection.binding;
|
|
@@ -57669,6 +57833,51 @@ async function dispatch(deps, msg, io, opts = {}) {
|
|
|
57669
57833
|
...restartRequest.coordinator !== void 0 ? { coordinator: restartRequest.coordinator } : {},
|
|
57670
57834
|
...deps.workflow !== void 0 ? { workflow: deps.workflow } : {}
|
|
57671
57835
|
});
|
|
57836
|
+
} else if (settled.kind === "handed-on" && recovery !== void 0 && coordinator !== void 0 && deps.coordinatorInstances !== void 0) {
|
|
57837
|
+
const unit = unitOfIdempotencyKey(coordinator.idempotencyKey);
|
|
57838
|
+
if (unit === void 0) {
|
|
57839
|
+
console.error(
|
|
57840
|
+
`[dispatch] ${msg.threadKey} could not retain ${settled.pending.length} recovery follow-up(s): the coordinator key names no unit`
|
|
57841
|
+
);
|
|
57842
|
+
} else {
|
|
57843
|
+
const key = { instanceId: coordinator.parentInstanceId, unit };
|
|
57844
|
+
let retained = 0;
|
|
57845
|
+
for (const pending2 of settled.pending) {
|
|
57846
|
+
const attachments = [...pending2.msg.images ?? [], ...pending2.msg.documents ?? []].map((asset) => ({
|
|
57847
|
+
mediaType: asset.mediaType,
|
|
57848
|
+
data: asset.data,
|
|
57849
|
+
...asset.name !== void 0 ? { name: asset.name } : {}
|
|
57850
|
+
}));
|
|
57851
|
+
try {
|
|
57852
|
+
const appended = await deps.coordinatorInstances.appendEvent(key, {
|
|
57853
|
+
...pending2.msg.messageId !== void 0 ? { id: pending2.msg.messageId } : {},
|
|
57854
|
+
sender: pending2.msg.userId,
|
|
57855
|
+
...pending2.msg.userName !== void 0 ? { senderName: pending2.msg.userName } : {},
|
|
57856
|
+
text: pending2.msg.text,
|
|
57857
|
+
...attachments.length > 0 ? { attachments } : {},
|
|
57858
|
+
mode: "steer",
|
|
57859
|
+
at: pending2.at
|
|
57860
|
+
});
|
|
57861
|
+
if (appended.ok) retained++;
|
|
57862
|
+
else {
|
|
57863
|
+
console.error(`[dispatch] ${msg.threadKey} could not retain a recovery follow-up: ${appended.reason}`);
|
|
57864
|
+
await pending2.io.reply(
|
|
57865
|
+
"Your reply could not be saved on the recovered unit. No replacement work was started; please retry the reply."
|
|
57866
|
+
).catch(() => void 0);
|
|
57867
|
+
}
|
|
57868
|
+
} catch (err2) {
|
|
57869
|
+
console.error(
|
|
57870
|
+
`[dispatch] ${msg.threadKey} could not retain a recovery follow-up: ${err2 instanceof Error ? err2.message : String(err2)}`
|
|
57871
|
+
);
|
|
57872
|
+
await pending2.io.reply(
|
|
57873
|
+
"Your reply could not be saved on the recovered unit. No replacement work was started; please retry the reply."
|
|
57874
|
+
).catch(() => void 0);
|
|
57875
|
+
}
|
|
57876
|
+
}
|
|
57877
|
+
console.log(
|
|
57878
|
+
`[dispatch] ${msg.threadKey} retained ${retained}/${settled.pending.length} recovery follow-up(s) on ${key.instanceId}:${key.unit}; no fresh turn ran`
|
|
57879
|
+
);
|
|
57880
|
+
}
|
|
57672
57881
|
} else if (settled.kind === "handed-on") {
|
|
57673
57882
|
const fresh = prepareFreshTurn(deps, { agent: settled.agent, pending: settled.pending, clock: clock4 });
|
|
57674
57883
|
await dispatch(deps, fresh.msg, fresh.io, fresh.opts).catch(
|
|
@@ -71732,6 +71941,12 @@ async function recoverRunnerOwnedPullOwners(instanceIds, instances) {
|
|
|
71732
71941
|
owned.set(runnerOwnedPullKey(instance.repo, unit.pr.number), { instanceId, unit: unit.unit });
|
|
71733
71942
|
}
|
|
71734
71943
|
}
|
|
71944
|
+
for (const unit of await instances.listActiveRecoveries()) {
|
|
71945
|
+
if (unit.pr === void 0) continue;
|
|
71946
|
+
const instance = await instances.get(unit.instanceId);
|
|
71947
|
+
if (instance !== null)
|
|
71948
|
+
owned.set(runnerOwnedPullKey(instance.repo, unit.pr.number), { instanceId: unit.instanceId, unit: unit.unit });
|
|
71949
|
+
}
|
|
71735
71950
|
return owned;
|
|
71736
71951
|
}
|
|
71737
71952
|
var runnerOwnedPullKey, RunnerOwnershipFence;
|
|
@@ -71754,6 +71969,7 @@ var init_runnerOwnership = __esm({
|
|
|
71754
71969
|
* synchronous, so a caller owns the process-local fence before its next
|
|
71755
71970
|
* awaited durable transition. Repeating the same owner's claim is idempotent. */
|
|
71756
71971
|
claim(repo, prNumber, owner) {
|
|
71972
|
+
if (!this.recoveryComplete) throw new Error("runner ownership recovery is still in progress");
|
|
71757
71973
|
const key = runnerOwnedPullKey(repo, prNumber);
|
|
71758
71974
|
const current = this.claimedOwners.get(key) ?? this.recoveredOwners.get(key);
|
|
71759
71975
|
const alreadyOwned = this.claimed.has(key) || this.recovered.has(key);
|
|
@@ -71768,6 +71984,7 @@ var init_runnerOwnership = __esm({
|
|
|
71768
71984
|
* transition. Even the same runner identity cannot reserve it twice: the
|
|
71769
71985
|
* returned token, not the owner fields, identifies the one caller. */
|
|
71770
71986
|
reserve(repo, prNumber, owner) {
|
|
71987
|
+
if (!this.recoveryComplete) throw new Error("runner ownership recovery is still in progress");
|
|
71771
71988
|
const key = runnerOwnedPullKey(repo, prNumber);
|
|
71772
71989
|
if (this.claimed.has(key) || this.recovered.has(key) || this.reservations.has(key)) return void 0;
|
|
71773
71990
|
const token2 = Symbol(key);
|
|
@@ -72564,6 +72781,8 @@ function parseBrief(v) {
|
|
|
72564
72781
|
if (!review.ok) return review;
|
|
72565
72782
|
const checks = checkRows();
|
|
72566
72783
|
if (!checks.ok) return checks;
|
|
72784
|
+
if (b.headSha !== void 0 && (typeof b.headSha !== "string" || !/^[0-9a-f]{40}$/i.test(b.headSha)))
|
|
72785
|
+
return invalid("brief.headSha must be a full commit sha");
|
|
72567
72786
|
return {
|
|
72568
72787
|
ok: true,
|
|
72569
72788
|
value: {
|
|
@@ -72571,6 +72790,7 @@ function parseBrief(v) {
|
|
|
72571
72790
|
unit: b.unit,
|
|
72572
72791
|
pr: n2.value,
|
|
72573
72792
|
reviewRunId: review.value,
|
|
72793
|
+
...b.headSha !== void 0 ? { headSha: b.headSha } : {},
|
|
72574
72794
|
...checks.value !== void 0 ? { checks: checks.value } : {}
|
|
72575
72795
|
}
|
|
72576
72796
|
};
|
|
@@ -72786,6 +73006,32 @@ async function spawn6(body, deps) {
|
|
|
72786
73006
|
if (live) return answerForLive(live, key, threadKey, at);
|
|
72787
73007
|
const done = await finishedWithKey(deps.runs, instance, threadKey, key);
|
|
72788
73008
|
if (done) return json3(200, { ok: true, runId: done.id, threadKey, alreadySpawned: true, at });
|
|
73009
|
+
if (row2?.recovery !== void 0) {
|
|
73010
|
+
const expectedKind = req.preset === "coding" ? "findings" : "review";
|
|
73011
|
+
const stepMatch = new RegExp(`^${row2.unit}/recovery/[1-9][0-9]*/${expectedKind}(?:/a[1-9][0-9]*)?$`).test(req.step);
|
|
73012
|
+
const briefHead = req.brief !== void 0 && "headSha" in req.brief && typeof req.brief.headSha === "string" ? req.brief.headSha : void 0;
|
|
73013
|
+
if (!stepMatch || row2.publication === void 0 || briefHead !== row2.publication.expectedHeadSha || typeof req.budget !== "number" || req.budget > Math.floor((row2.recovery.deadlineAt - at) / minutesToMs(1)))
|
|
73014
|
+
return json3(409, { ok: false, error: "recovery_claim_mismatch", at });
|
|
73015
|
+
let facts2;
|
|
73016
|
+
try {
|
|
73017
|
+
facts2 = await deps.fetchPrFacts({ repo: instance.repo, number: row2.publication.pr });
|
|
73018
|
+
} catch (err2) {
|
|
73019
|
+
return json3(502, { ok: false, error: "github_unavailable", message: describe12(err2), at });
|
|
73020
|
+
}
|
|
73021
|
+
const verified = facts2?.verifiedHead;
|
|
73022
|
+
if (facts2?.state !== "open" || facts2.sameRepoHead !== true || facts2.headBranchExists !== true || facts2.headRef !== row2.publication.headRef || facts2.baseRef !== row2.publication.baseRef || facts2.headSha !== row2.publication.expectedHeadSha || verified?.repo.toLowerCase() !== instance.repo.toLowerCase() || verified.ref !== row2.publication.publicationRef || verified.sha !== row2.publication.expectedHeadSha)
|
|
73023
|
+
return json3(409, {
|
|
73024
|
+
ok: false,
|
|
73025
|
+
error: fullHead(facts2?.headSha) && facts2.headSha !== row2.publication.expectedHeadSha ? "recovery_head_moved" : "recovery_facts_mismatch",
|
|
73026
|
+
at
|
|
73027
|
+
});
|
|
73028
|
+
try {
|
|
73029
|
+
if (deps.runnerOwnership?.claim(instance.repo, row2.publication.pr, row2.publication.owner) !== true)
|
|
73030
|
+
return json3(409, { ok: false, error: "publication_ownership_changed", at });
|
|
73031
|
+
} catch (err2) {
|
|
73032
|
+
return json3(409, { ok: false, error: "publication_ownership_unknown", message: describe12(err2), at });
|
|
73033
|
+
}
|
|
73034
|
+
}
|
|
72789
73035
|
const io = deps.ioFor({ threadKey, userId: instance.userId });
|
|
72790
73036
|
if (!io) return json3(503, { ok: false, error: "no_channel", at });
|
|
72791
73037
|
let turn;
|
|
@@ -72821,6 +73067,13 @@ ${foldThreadEvents(folded)}` };
|
|
|
72821
73067
|
at
|
|
72822
73068
|
});
|
|
72823
73069
|
try {
|
|
73070
|
+
if (row2.recovery !== void 0 && deps.runnerOwnership?.claim(instance.repo, row2.publication.pr, row2.publication.owner) !== true)
|
|
73071
|
+
return json3(409, {
|
|
73072
|
+
ok: false,
|
|
73073
|
+
error: "publication_ownership_changed",
|
|
73074
|
+
message: "the durable recovery claim no longer owns the pull request",
|
|
73075
|
+
at
|
|
73076
|
+
});
|
|
72824
73077
|
const owner = deps.runnerOwnership?.owner(instance.repo, row2.publication.pr);
|
|
72825
73078
|
if (owner === void 0 || owner.instanceId !== row2.publication.owner.instanceId || owner.unit !== row2.publication.owner.unit)
|
|
72826
73079
|
return json3(409, {
|
|
@@ -72864,6 +73117,7 @@ ${foldThreadEvents(folded)}` };
|
|
|
72864
73117
|
const releaseAdmission = leaveAdmission ?? (() => {
|
|
72865
73118
|
});
|
|
72866
73119
|
let startedId;
|
|
73120
|
+
let deadlineTimer;
|
|
72867
73121
|
let lastReply;
|
|
72868
73122
|
let resolveStarted;
|
|
72869
73123
|
const started = new Promise((resolve13) => {
|
|
@@ -72872,6 +73126,14 @@ ${foldThreadEvents(folded)}` };
|
|
|
72872
73126
|
const child = watched(io, {
|
|
72873
73127
|
started: (id) => {
|
|
72874
73128
|
startedId = id;
|
|
73129
|
+
if (row2?.recovery !== void 0) {
|
|
73130
|
+
const stopAtDeadline = () => {
|
|
73131
|
+
void deps.runs.stopRun(id, "hard", { kind: "access", id: "original-unit-recovery-deadline" });
|
|
73132
|
+
};
|
|
73133
|
+
const left = row2.recovery.deadlineAt - (deps.clock ?? systemClock)();
|
|
73134
|
+
if (left <= 0) stopAtDeadline();
|
|
73135
|
+
else deadlineTimer = setTimeout(stopAtDeadline, left);
|
|
73136
|
+
}
|
|
72875
73137
|
releaseAdmission();
|
|
72876
73138
|
resolveStarted(id);
|
|
72877
73139
|
},
|
|
@@ -72887,6 +73149,17 @@ ${foldThreadEvents(folded)}` };
|
|
|
72887
73149
|
const tag = {
|
|
72888
73150
|
parentInstanceId: instance.id,
|
|
72889
73151
|
idempotencyKey: key,
|
|
73152
|
+
...row2?.recovery !== void 0 ? { transportWorkflowId: row2.recovery.workflowId } : {},
|
|
73153
|
+
...row2?.recovery !== void 0 && row2.publication !== void 0 ? {
|
|
73154
|
+
recovery: {
|
|
73155
|
+
repo: row2.publication.repo,
|
|
73156
|
+
pr: row2.publication.pr,
|
|
73157
|
+
headRef: row2.publication.headRef,
|
|
73158
|
+
baseRef: row2.publication.baseRef,
|
|
73159
|
+
expectedHeadSha: row2.recovery.expectedHeadSha,
|
|
73160
|
+
deadlineAt: row2.recovery.deadlineAt
|
|
73161
|
+
}
|
|
73162
|
+
} : {},
|
|
72890
73163
|
...instance.base !== void 0 ? { base: instance.base } : {},
|
|
72891
73164
|
...publication !== void 0 ? { publication } : {}
|
|
72892
73165
|
};
|
|
@@ -72894,6 +73167,16 @@ ${foldThreadEvents(folded)}` };
|
|
|
72894
73167
|
try {
|
|
72895
73168
|
dispatching = deps.dispatch(msg, child, {
|
|
72896
73169
|
coordinator: tag,
|
|
73170
|
+
...row2?.recovery !== void 0 && row2.publication !== void 0 ? {
|
|
73171
|
+
recovery: {
|
|
73172
|
+
repo: row2.publication.repo,
|
|
73173
|
+
pr: row2.publication.pr,
|
|
73174
|
+
headRef: row2.publication.headRef,
|
|
73175
|
+
baseRef: row2.publication.baseRef,
|
|
73176
|
+
expectedHeadSha: row2.publication.expectedHeadSha,
|
|
73177
|
+
deadlineAt: row2.recovery.deadlineAt
|
|
73178
|
+
}
|
|
73179
|
+
} : {},
|
|
72897
73180
|
...turn.contract !== void 0 ? { contract: turn.contract } : {}
|
|
72898
73181
|
});
|
|
72899
73182
|
} catch (err2) {
|
|
@@ -72904,6 +73187,9 @@ ${foldThreadEvents(folded)}` };
|
|
|
72904
73187
|
(outcome) => ({ kind: "ended", outcome }),
|
|
72905
73188
|
(err2) => ({ kind: "threw", err: err2 })
|
|
72906
73189
|
);
|
|
73190
|
+
void settled.finally(() => {
|
|
73191
|
+
if (deadlineTimer !== void 0) clearTimeout(deadlineTimer);
|
|
73192
|
+
});
|
|
72907
73193
|
void settled.then((end) => {
|
|
72908
73194
|
releaseAdmission();
|
|
72909
73195
|
if (end.kind === "threw")
|
|
@@ -73363,6 +73649,425 @@ async function steerChild(body, deps) {
|
|
|
73363
73649
|
const steered = await deps.steerChild?.(body.runId, text, instance) ?? false;
|
|
73364
73650
|
return json3(200, { ok: true, outcome: steered ? "steered" : "not_live", at });
|
|
73365
73651
|
}
|
|
73652
|
+
async function advanceRecoveryPublication(deps, instance, row2, facts2, runId3) {
|
|
73653
|
+
if (row2.recovery === void 0 || row2.publication === void 0 || facts2.headSha === row2.publication.expectedHeadSha)
|
|
73654
|
+
return row2;
|
|
73655
|
+
const child = await deps.runs.getRun(runId3);
|
|
73656
|
+
const expectedPrefix = `${instance.id}:${row2.unit}/recovery/`;
|
|
73657
|
+
const owner = { instanceId: instance.id, unit: row2.unit };
|
|
73658
|
+
if (!child.ok || child.value.finished !== true || child.value.status !== "completed" || child.value.agent !== "coding" || child.value.parentInstanceId !== instance.id || child.value.userId !== instance.userId || child.value.repo?.toLowerCase() !== instance.repo.toLowerCase() || child.value.threadKey !== (row2.threadKey ?? instance.threadKey) || child.value.idempotencyKey?.startsWith(expectedPrefix) !== true || !/\/findings(?:\/a[1-9][0-9]*)?$/.test(child.value.idempotencyKey) || !fullHead(facts2.headSha) || child.value.headSha !== facts2.headSha || child.value.pushed?.some((push) => push.ref === row2.branch && push.sha === facts2.headSha) !== true || facts2.state !== "open" || facts2.sameRepoHead !== true || facts2.headBranchExists !== true || facts2.headRef !== row2.publication.headRef || facts2.baseRef !== row2.publication.baseRef || facts2.verifiedHead?.repo.toLowerCase() !== instance.repo.toLowerCase() || facts2.verifiedHead.ref !== row2.publication.publicationRef || facts2.verifiedHead.sha !== facts2.headSha || !samePublicationOwner(row2.publication.owner, owner))
|
|
73659
|
+
throw new PublicationBindingRefusal("publication_facts_mismatch");
|
|
73660
|
+
try {
|
|
73661
|
+
if (deps.runnerOwnership?.claim(instance.repo, row2.publication.pr, owner) !== true)
|
|
73662
|
+
throw new PublicationBindingRefusal("publication_ownership_changed");
|
|
73663
|
+
} catch (err2) {
|
|
73664
|
+
if (err2 instanceof PublicationBindingRefusal) throw err2;
|
|
73665
|
+
throw new PublicationBindingRefusal("publication_ownership_unknown");
|
|
73666
|
+
}
|
|
73667
|
+
const updated = {
|
|
73668
|
+
...row2,
|
|
73669
|
+
lastPush: facts2.headSha,
|
|
73670
|
+
publication: { ...row2.publication, expectedHeadSha: facts2.headSha },
|
|
73671
|
+
recovery: { ...row2.recovery, expectedHeadSha: facts2.headSha }
|
|
73672
|
+
};
|
|
73673
|
+
let replaced;
|
|
73674
|
+
try {
|
|
73675
|
+
replaced = await deps.instances.compareAndReplaceUnit(row2, updated);
|
|
73676
|
+
} catch {
|
|
73677
|
+
const reread = await deps.instances.listUnits(instance.id).catch(() => void 0);
|
|
73678
|
+
const current = reread?.filter((candidate) => candidate.unit === row2.unit);
|
|
73679
|
+
if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(updated)) replaced = { ok: true };
|
|
73680
|
+
}
|
|
73681
|
+
if (replaced?.ok !== true)
|
|
73682
|
+
throw new PublicationBindingRefusal(
|
|
73683
|
+
replaced?.reason === "stale" ? "publication_binding_stale" : "publication_store_unavailable"
|
|
73684
|
+
);
|
|
73685
|
+
return updated;
|
|
73686
|
+
}
|
|
73687
|
+
async function bindOpenPullRequest(deps, instance, row2, pr, facts2) {
|
|
73688
|
+
if (row2 === void 0) return;
|
|
73689
|
+
const verified = facts2.verifiedHead;
|
|
73690
|
+
if (instance.base === void 0 || facts2.state !== "open" || facts2.sameRepoHead !== true || facts2.headBranchExists !== true || facts2.headRef !== row2.branch || facts2.baseRef !== instance.base || verified === void 0 || verified.repo.toLowerCase() !== instance.repo.toLowerCase() || verified.ref !== row2.branch || !/^[0-9a-f]{40}$/i.test(verified.sha) || facts2.headSha !== verified.sha || row2.pr !== void 0 && row2.pr.number !== pr.number)
|
|
73691
|
+
throw new PublicationBindingRefusal("publication_facts_mismatch");
|
|
73692
|
+
const owner = { instanceId: instance.id, unit: row2.unit };
|
|
73693
|
+
const publication = {
|
|
73694
|
+
repo: instance.repo,
|
|
73695
|
+
pr: pr.number,
|
|
73696
|
+
headRef: row2.branch,
|
|
73697
|
+
baseRef: instance.base,
|
|
73698
|
+
expectedHeadSha: verified.sha,
|
|
73699
|
+
publicationRef: row2.branch,
|
|
73700
|
+
owner
|
|
73701
|
+
};
|
|
73702
|
+
const existing = row2.publication;
|
|
73703
|
+
if (existing !== void 0 && (existing.repo !== publication.repo || existing.pr !== publication.pr || existing.headRef !== publication.headRef || existing.baseRef !== publication.baseRef || existing.expectedHeadSha !== publication.expectedHeadSha || existing.publicationRef !== publication.publicationRef || !samePublicationOwner(existing.owner, owner)))
|
|
73704
|
+
throw new PublicationBindingRefusal("publication_facts_mismatch");
|
|
73705
|
+
const fence = deps.runnerOwnership;
|
|
73706
|
+
if (fence === void 0) throw new PublicationBindingRefusal("publication_ownership_unknown");
|
|
73707
|
+
let priorOwner;
|
|
73708
|
+
try {
|
|
73709
|
+
priorOwner = fence.owner(instance.repo, pr.number);
|
|
73710
|
+
} catch {
|
|
73711
|
+
throw new PublicationBindingRefusal("publication_ownership_unknown");
|
|
73712
|
+
}
|
|
73713
|
+
if (priorOwner !== void 0 && !samePublicationOwner(priorOwner, owner))
|
|
73714
|
+
throw new PublicationBindingRefusal("publication_ownership_changed");
|
|
73715
|
+
try {
|
|
73716
|
+
if (!fence.claim(instance.repo, pr.number, owner))
|
|
73717
|
+
throw new PublicationBindingRefusal("publication_ownership_changed");
|
|
73718
|
+
} catch (err2) {
|
|
73719
|
+
if (err2 instanceof PublicationBindingRefusal) throw err2;
|
|
73720
|
+
throw new PublicationBindingRefusal("publication_ownership_unknown");
|
|
73721
|
+
}
|
|
73722
|
+
let claimedHere = priorOwner === void 0;
|
|
73723
|
+
const releaseClaim = () => {
|
|
73724
|
+
if (!claimedHere) return;
|
|
73725
|
+
fence.release(instance.repo, pr.number, owner);
|
|
73726
|
+
claimedHere = false;
|
|
73727
|
+
};
|
|
73728
|
+
try {
|
|
73729
|
+
if (!samePublicationOwner(fence.owner(instance.repo, pr.number), owner)) {
|
|
73730
|
+
releaseClaim();
|
|
73731
|
+
throw new PublicationBindingRefusal("publication_ownership_unknown");
|
|
73732
|
+
}
|
|
73733
|
+
} catch (err2) {
|
|
73734
|
+
releaseClaim();
|
|
73735
|
+
if (err2 instanceof PublicationBindingRefusal) throw err2;
|
|
73736
|
+
throw new PublicationBindingRefusal("publication_ownership_unknown");
|
|
73737
|
+
}
|
|
73738
|
+
const replacement = { ...row2, pr, publication };
|
|
73739
|
+
if (JSON.stringify(replacement) === JSON.stringify(row2)) return;
|
|
73740
|
+
let replaced;
|
|
73741
|
+
try {
|
|
73742
|
+
replaced = await deps.instances.compareAndReplaceUnit(row2, replacement);
|
|
73743
|
+
} catch {
|
|
73744
|
+
releaseClaim();
|
|
73745
|
+
throw new PublicationBindingRefusal("publication_store_unavailable");
|
|
73746
|
+
}
|
|
73747
|
+
if (!replaced.ok) {
|
|
73748
|
+
releaseClaim();
|
|
73749
|
+
throw new PublicationBindingRefusal(
|
|
73750
|
+
replaced.reason === "stale" ? "publication_binding_stale" : "publication_store_unavailable"
|
|
73751
|
+
);
|
|
73752
|
+
}
|
|
73753
|
+
}
|
|
73754
|
+
async function recoverOriginalUnit(body, deps, caller) {
|
|
73755
|
+
const id = parseInstanceId(body.parentInstanceId);
|
|
73756
|
+
if (!id.ok) return json3(400, { ok: false, error: id.error });
|
|
73757
|
+
if (typeof body.unit !== "string" || !UNIT_ID.test(body.unit))
|
|
73758
|
+
return json3(400, { ok: false, error: "unit must be a unit id" });
|
|
73759
|
+
const at = (deps.clock ?? systemClock)();
|
|
73760
|
+
const instance = await deps.instances.get(id.value);
|
|
73761
|
+
if (instance === null) return json3(404, { ok: false, error: "unknown_instance", at });
|
|
73762
|
+
const requestedWorkflowId = typeof body.workflowId === "string" ? body.workflowId : void 0;
|
|
73763
|
+
if (caller === void 0 && requestedWorkflowId === void 0)
|
|
73764
|
+
return json3(403, { ok: false, error: "recovery_caller_required", at });
|
|
73765
|
+
if (caller !== void 0 && caller.userId !== instance.userId)
|
|
73766
|
+
return json3(403, { ok: false, error: "recovery_requester_mismatch", at });
|
|
73767
|
+
const rows = await deps.instances.listUnits(instance.id);
|
|
73768
|
+
const matches = rows.filter((candidate) => candidate.unit === body.unit);
|
|
73769
|
+
if (matches.length !== 1) return json3(409, { ok: false, error: "unit_evidence_ambiguous", at });
|
|
73770
|
+
const row2 = matches[0];
|
|
73771
|
+
if (caller !== void 0 && caller.threadKey !== (row2.threadKey ?? instance.threadKey))
|
|
73772
|
+
return json3(403, { ok: false, error: "recovery_requester_mismatch", at });
|
|
73773
|
+
if (row2.idle !== void 0 && row2.ending !== void 0)
|
|
73774
|
+
return json3(409, { ok: false, error: "unit_lifecycle_ambiguous", at });
|
|
73775
|
+
const originalOwner = { instanceId: instance.id, unit: row2.unit };
|
|
73776
|
+
const publication = row2.publication;
|
|
73777
|
+
const pr = row2.pr;
|
|
73778
|
+
const base = instance.base;
|
|
73779
|
+
let expectedHead = row2.lastPush;
|
|
73780
|
+
if (pr === void 0 || base === void 0 || !fullHead(expectedHead) || publication === void 0 || publication.repo.toLowerCase() !== instance.repo.toLowerCase() || publication.pr !== pr.number || publication.headRef !== row2.branch || publication.baseRef !== base || publication.publicationRef !== row2.branch || !fullHead(publication.expectedHeadSha) || !samePublicationOwner(publication.owner, originalOwner))
|
|
73781
|
+
return json3(409, { ok: false, error: "recovery_binding_mismatch", at });
|
|
73782
|
+
const existingClaim = row2.recovery;
|
|
73783
|
+
let kind;
|
|
73784
|
+
let round2;
|
|
73785
|
+
let remainingMs;
|
|
73786
|
+
let reviewRunId;
|
|
73787
|
+
let stepName;
|
|
73788
|
+
let workflowId;
|
|
73789
|
+
let reviewKey;
|
|
73790
|
+
let deadlineAt;
|
|
73791
|
+
let findings;
|
|
73792
|
+
let findingsRunId;
|
|
73793
|
+
let findingsKey;
|
|
73794
|
+
let claimRow = row2;
|
|
73795
|
+
let facts2;
|
|
73796
|
+
let originalRow;
|
|
73797
|
+
if (existingClaim !== void 0) {
|
|
73798
|
+
({
|
|
73799
|
+
kind,
|
|
73800
|
+
round: round2,
|
|
73801
|
+
remainingMs,
|
|
73802
|
+
reviewRunId,
|
|
73803
|
+
step: stepName,
|
|
73804
|
+
workflowId,
|
|
73805
|
+
reviewKey,
|
|
73806
|
+
deadlineAt,
|
|
73807
|
+
findings,
|
|
73808
|
+
findingsRunId,
|
|
73809
|
+
findingsKey
|
|
73810
|
+
} = existingClaim);
|
|
73811
|
+
if (existingClaim.expectedHeadSha !== expectedHead || publication.expectedHeadSha !== expectedHead)
|
|
73812
|
+
return json3(409, { ok: false, error: "recovery_binding_mismatch", at });
|
|
73813
|
+
const { recovery: _recovery, ...withoutRecovery } = row2;
|
|
73814
|
+
originalRow = { ...withoutRecovery, ending: existingClaim.previousEnding };
|
|
73815
|
+
} else {
|
|
73816
|
+
if (row2.idle !== void 0 || row2.ending === void 0)
|
|
73817
|
+
return json3(409, { ok: false, error: "unit_not_terminal", at });
|
|
73818
|
+
let boundaryPosition = -1;
|
|
73819
|
+
for (let index = row2.rounds.length - 1; index >= 0; index -= 1)
|
|
73820
|
+
if (row2.rounds[index].agent === "review" && row2.rounds[index].outcome !== "started") {
|
|
73821
|
+
boundaryPosition = index;
|
|
73822
|
+
break;
|
|
73823
|
+
}
|
|
73824
|
+
const boundary = boundaryPosition >= 0 ? row2.rounds[boundaryPosition] : void 0;
|
|
73825
|
+
if (boundary === void 0 || boundary.outcome !== "request_changes" && boundary.outcome !== "no_verdict")
|
|
73826
|
+
return json3(409, { ok: false, error: "recovery_ending_unsupported", at });
|
|
73827
|
+
if (row2.rounds.slice(boundaryPosition + 1).some((candidate) => candidate.agent === "review"))
|
|
73828
|
+
return json3(409, { ok: false, error: "recovery_stage_ambiguous", at });
|
|
73829
|
+
kind = boundary.outcome === "request_changes" ? "findings" : "review";
|
|
73830
|
+
if (kind === "review" && row2.ending.kind !== "no_verdict")
|
|
73831
|
+
return json3(409, { ok: false, error: "recovery_ending_mismatch", at });
|
|
73832
|
+
round2 = boundary.index;
|
|
73833
|
+
const caps = instance.caps;
|
|
73834
|
+
if (caps === void 0 || row2.startedAt === void 0 || !Number.isFinite(row2.ending.at))
|
|
73835
|
+
return json3(409, { ok: false, error: "recovery_budget_unknown", at });
|
|
73836
|
+
if (instance.grant?.costCapUsd !== void 0) return json3(409, { ok: false, error: "recovery_budget_unknown", at });
|
|
73837
|
+
if (round2 >= caps.maxRounds) return json3(409, { ok: false, error: "recovery_rounds_exhausted", at });
|
|
73838
|
+
const latestSegmentIndex = row2.segments?.reduce((latest, candidate) => Math.max(latest, candidate.index), 1) ?? 1;
|
|
73839
|
+
const latestSegments = row2.segments?.filter((candidate) => candidate.index === latestSegmentIndex) ?? [];
|
|
73840
|
+
if (latestSegments.length > 1) return json3(409, { ok: false, error: "recovery_budget_unknown", at });
|
|
73841
|
+
const segment = latestSegments[0];
|
|
73842
|
+
const leaseStartedAt = segment?.at ?? row2.startedAt;
|
|
73843
|
+
const resumedLeases = Object.values(row2.wakes ?? {}).filter(
|
|
73844
|
+
(answer) => answer.kind === "segment" && answer.index === latestSegmentIndex && answer.leaseMs !== void 0
|
|
73845
|
+
);
|
|
73846
|
+
if (resumedLeases.length > 0) return json3(409, { ok: false, error: "recovery_budget_unknown", at });
|
|
73847
|
+
remainingMs = minutesToMs(caps.maxMinutes) - (at - leaseStartedAt);
|
|
73848
|
+
const reviewThreadKey = row2.reviewThread?.threadKey ?? row2.threadKey ?? instance.threadKey;
|
|
73849
|
+
const unitThreadKey = row2.threadKey ?? instance.threadKey;
|
|
73850
|
+
const listing = await deps.runs.listRuns({
|
|
73851
|
+
status: "finished",
|
|
73852
|
+
visibleTo: EVERY_RUN,
|
|
73853
|
+
threadKey: reviewThreadKey,
|
|
73854
|
+
limit: RUN_LIST_MAX_LIMIT
|
|
73855
|
+
});
|
|
73856
|
+
const unitListing = unitThreadKey === reviewThreadKey ? listing : await deps.runs.listRuns({
|
|
73857
|
+
status: "finished",
|
|
73858
|
+
visibleTo: EVERY_RUN,
|
|
73859
|
+
threadKey: unitThreadKey,
|
|
73860
|
+
limit: RUN_LIST_MAX_LIMIT
|
|
73861
|
+
});
|
|
73862
|
+
const segmentPrefix = stepPrefixOf(
|
|
73863
|
+
row2.unit,
|
|
73864
|
+
latestSegmentIndex > 1 ? { segment: latestSegmentIndex, renewalsSpent: 0, spendUsd: null } : void 0
|
|
73865
|
+
);
|
|
73866
|
+
const reviewedHead = publication.expectedHeadSha;
|
|
73867
|
+
const reviewKeyPrefix = `${instance.id}:${segmentPrefix}/${round2}/review`;
|
|
73868
|
+
const candidates2 = listing.runs.filter((run2) => {
|
|
73869
|
+
if (!run2.finished || run2.agent !== "review" || run2.parentInstanceId !== instance.id || run2.idempotencyKey !== reviewKeyPrefix && run2.idempotencyKey?.startsWith(`${reviewKeyPrefix}/a`) !== true || run2.userId !== instance.userId || run2.repo?.toLowerCase() !== instance.repo.toLowerCase() || run2.threadKey !== reviewThreadKey)
|
|
73870
|
+
return false;
|
|
73871
|
+
if (run2.reviewHead !== reviewedHead) return false;
|
|
73872
|
+
if (kind === "review") return run2.verdict === void 0 && run2.reviewPost === void 0;
|
|
73873
|
+
return run2.verdict?.verdict === "request_changes" && run2.reviewHead === reviewedHead && run2.reviewPost?.posted === true && run2.reviewPost.head === reviewedHead && run2.reviewPost.verdict === "request_changes" && run2.reviewPost.target.repo.toLowerCase() === instance.repo.toLowerCase() && run2.reviewPost.target.number === pr.number;
|
|
73874
|
+
});
|
|
73875
|
+
if (candidates2.length !== 1) return json3(409, { ok: false, error: "recovery_review_evidence_ambiguous", at });
|
|
73876
|
+
reviewRunId = candidates2[0].id;
|
|
73877
|
+
reviewKey = candidates2[0].idempotencyKey;
|
|
73878
|
+
try {
|
|
73879
|
+
facts2 = await deps.fetchPrFacts({ repo: instance.repo, number: pr.number });
|
|
73880
|
+
} catch (err2) {
|
|
73881
|
+
return json3(502, { ok: false, error: "github_unavailable", message: describe12(err2), at });
|
|
73882
|
+
}
|
|
73883
|
+
if (facts2 === void 0) return json3(502, { ok: false, error: "github_unavailable", at });
|
|
73884
|
+
const verified2 = facts2.verifiedHead;
|
|
73885
|
+
if (facts2.state !== "open" || facts2.sameRepoHead !== true || facts2.headBranchExists !== true || facts2.headRef !== row2.branch || facts2.baseRef !== base || !fullHead(facts2.headSha) || verified2 === void 0 || verified2.repo.toLowerCase() !== instance.repo.toLowerCase() || verified2.ref !== row2.branch || verified2.sha !== facts2.headSha)
|
|
73886
|
+
return json3(409, { ok: false, error: "recovery_facts_mismatch", at });
|
|
73887
|
+
const findingsStep = `${segmentPrefix}/${boundary.index}/findings/pr-check`;
|
|
73888
|
+
const findingsPrefix = `${instance.id}:${segmentPrefix}/${boundary.index}/findings`;
|
|
73889
|
+
const headMoved = facts2.headSha !== reviewedHead || expectedHead !== reviewedHead;
|
|
73890
|
+
const completedFindings = unitListing.runs.filter(
|
|
73891
|
+
(run2) => boundary.outcome === "request_changes" && row2.ending?.cause === "step_threw" && row2.ending.step === findingsStep && row2.ending.round === boundary.index && (expectedHead === reviewedHead || expectedHead === facts2.headSha) && run2.finished && run2.status === "completed" && run2.agent === "coding" && run2.parentInstanceId === instance.id && run2.userId === instance.userId && run2.repo?.toLowerCase() === instance.repo.toLowerCase() && run2.threadKey === unitThreadKey && (run2.idempotencyKey === findingsPrefix || run2.idempotencyKey?.startsWith(`${findingsPrefix}/a`) === true) && run2.headSha === facts2.headSha && (!headMoved || run2.pushed?.some((push) => push.ref === row2.branch && push.sha === facts2.headSha) === true) && run2.startedAt >= (candidates2[0].finishedAt ?? candidates2[0].startedAt) && (run2.finishedAt === void 0 || run2.finishedAt <= row2.ending.at)
|
|
73892
|
+
);
|
|
73893
|
+
if (completedFindings.length > 1 || headMoved && completedFindings.length !== 1)
|
|
73894
|
+
return json3(409, { ok: false, error: "recovery_head_moved", at });
|
|
73895
|
+
if (completedFindings.length === 1) {
|
|
73896
|
+
findingsRunId = completedFindings[0].id;
|
|
73897
|
+
findingsKey = completedFindings[0].idempotencyKey;
|
|
73898
|
+
expectedHead = facts2.headSha;
|
|
73899
|
+
kind = "review";
|
|
73900
|
+
round2 = boundary.index + 1;
|
|
73901
|
+
claimRow = {
|
|
73902
|
+
...row2,
|
|
73903
|
+
lastPush: expectedHead,
|
|
73904
|
+
publication: { ...publication, expectedHeadSha: expectedHead }
|
|
73905
|
+
};
|
|
73906
|
+
} else expectedHead = reviewedHead;
|
|
73907
|
+
const floor = minutesToMs(leaseMinimum(kind === "findings" ? "fix" : "review"));
|
|
73908
|
+
if (!Number.isFinite(remainingMs) || remainingMs < floor)
|
|
73909
|
+
return json3(409, { ok: false, error: "recovery_wall_clock_exhausted", at });
|
|
73910
|
+
stepName = `${row2.unit}/recovery/${round2}/${kind}`;
|
|
73911
|
+
workflowId = `recovery-${findingsRunId ?? reviewRunId}`;
|
|
73912
|
+
deadlineAt = at + remainingMs;
|
|
73913
|
+
findings = kind === "findings" ? candidates2[0].verdict?.findings : void 0;
|
|
73914
|
+
if (row2.recoveryReceipt?.reviewRunId === reviewRunId)
|
|
73915
|
+
return json3(409, { ok: false, error: "recovery_already_completed", at });
|
|
73916
|
+
originalRow = row2;
|
|
73917
|
+
}
|
|
73918
|
+
const expiredExistingClaim = existingClaim !== void 0 && at >= deadlineAt;
|
|
73919
|
+
if (requestedWorkflowId !== void 0) {
|
|
73920
|
+
if (existingClaim === void 0 || requestedWorkflowId !== workflowId)
|
|
73921
|
+
return json3(409, { ok: false, error: "recovery_claim_mismatch", at });
|
|
73922
|
+
}
|
|
73923
|
+
const fence = deps.runnerOwnership;
|
|
73924
|
+
if (fence === void 0 || fence.reserve === void 0 || fence.transferReservation === void 0 || fence.releaseReservation === void 0)
|
|
73925
|
+
return json3(503, { ok: false, error: "publication_ownership_unknown", at });
|
|
73926
|
+
const releaseReservation = fence.releaseReservation.bind(fence);
|
|
73927
|
+
if (existingClaim !== void 0) {
|
|
73928
|
+
try {
|
|
73929
|
+
if (!fence.claim(instance.repo, pr.number, originalOwner))
|
|
73930
|
+
return json3(409, { ok: false, error: "publication_ownership_changed", at });
|
|
73931
|
+
} catch (err2) {
|
|
73932
|
+
return json3(503, { ok: false, error: "publication_ownership_unknown", message: describe12(err2), at });
|
|
73933
|
+
}
|
|
73934
|
+
}
|
|
73935
|
+
let claimedRow = row2;
|
|
73936
|
+
let token2;
|
|
73937
|
+
let transferred = existingClaim !== void 0;
|
|
73938
|
+
const rollback = async (error, consumed = false) => {
|
|
73939
|
+
if (originalRow !== void 0) {
|
|
73940
|
+
const replacement = consumed ? {
|
|
73941
|
+
...originalRow,
|
|
73942
|
+
recoveryReceipt: { reviewRunId, workflowId, at }
|
|
73943
|
+
} : originalRow;
|
|
73944
|
+
const restored = await deps.instances.compareAndReplaceUnit(claimedRow, replacement).catch(() => void 0);
|
|
73945
|
+
if (restored?.ok !== true)
|
|
73946
|
+
return json3(500, {
|
|
73947
|
+
ok: false,
|
|
73948
|
+
error: "recovery_rollback_failed",
|
|
73949
|
+
cause: error,
|
|
73950
|
+
reason: restored?.reason ?? "unavailable",
|
|
73951
|
+
at
|
|
73952
|
+
});
|
|
73953
|
+
}
|
|
73954
|
+
const released = transferred ? fence.release(instance.repo, pr.number, originalOwner) : token2 !== void 0 ? releaseReservation(instance.repo, pr.number, token2) : true;
|
|
73955
|
+
if (!released) return json3(500, { ok: false, error: "recovery_cleanup_failed", cause: error, at });
|
|
73956
|
+
return json3(409, { ok: false, error, at });
|
|
73957
|
+
};
|
|
73958
|
+
if (requestedWorkflowId === void 0 && expiredExistingClaim) {
|
|
73959
|
+
const status3 = await deps.recoveryStatus?.(workflowId).catch((err2) => ({
|
|
73960
|
+
kind: "unanswered",
|
|
73961
|
+
reason: describe12(err2)
|
|
73962
|
+
}));
|
|
73963
|
+
if (status3 === void 0 || status3.kind === "unanswered")
|
|
73964
|
+
return json3(503, {
|
|
73965
|
+
ok: false,
|
|
73966
|
+
error: "recovery_status_unanswered",
|
|
73967
|
+
...status3?.kind === "unanswered" ? { message: status3.reason } : {},
|
|
73968
|
+
workflowId,
|
|
73969
|
+
at
|
|
73970
|
+
});
|
|
73971
|
+
if (status3.kind === "status" && !["complete", "errored", "terminated"].includes(status3.status))
|
|
73972
|
+
return json3(409, { ok: false, error: "recovery_wall_clock_exhausted", workflowId, at });
|
|
73973
|
+
return rollback("recovery_wall_clock_exhausted", status3.kind !== "absent");
|
|
73974
|
+
}
|
|
73975
|
+
if (facts2 === void 0)
|
|
73976
|
+
try {
|
|
73977
|
+
facts2 = await deps.fetchPrFacts({ repo: instance.repo, number: pr.number });
|
|
73978
|
+
} catch (err2) {
|
|
73979
|
+
return json3(502, { ok: false, error: "github_unavailable", message: describe12(err2), at });
|
|
73980
|
+
}
|
|
73981
|
+
if (facts2 === void 0) return json3(502, { ok: false, error: "github_unavailable", at });
|
|
73982
|
+
const verified = facts2.verifiedHead;
|
|
73983
|
+
if (facts2.state !== "open" || facts2.sameRepoHead !== true || facts2.headBranchExists !== true || facts2.headRef !== row2.branch || facts2.baseRef !== base || !fullHead(facts2.headSha) || facts2.headSha !== expectedHead || verified === void 0 || verified.repo.toLowerCase() !== instance.repo.toLowerCase() || verified.ref !== row2.branch || verified.sha !== expectedHead) {
|
|
73984
|
+
const error = fullHead(facts2.headSha) && facts2.headSha !== expectedHead ? "recovery_head_moved" : "recovery_facts_mismatch";
|
|
73985
|
+
return requestedWorkflowId === void 0 ? json3(409, { ok: false, error, at }) : rollback(error);
|
|
73986
|
+
}
|
|
73987
|
+
if (existingClaim === void 0) {
|
|
73988
|
+
try {
|
|
73989
|
+
token2 = fence.reserve(instance.repo, pr.number, originalOwner);
|
|
73990
|
+
} catch (err2) {
|
|
73991
|
+
return json3(503, { ok: false, error: "publication_ownership_unknown", message: describe12(err2), at });
|
|
73992
|
+
}
|
|
73993
|
+
if (token2 === void 0) return json3(409, { ok: false, error: "publication_ownership_changed", at });
|
|
73994
|
+
const { ending: _ending, ...withoutEnding } = claimRow;
|
|
73995
|
+
claimedRow = {
|
|
73996
|
+
...withoutEnding,
|
|
73997
|
+
recovery: {
|
|
73998
|
+
kind,
|
|
73999
|
+
round: round2,
|
|
74000
|
+
expectedHeadSha: expectedHead,
|
|
74001
|
+
remainingMs,
|
|
74002
|
+
claimedAt: at,
|
|
74003
|
+
step: stepName,
|
|
74004
|
+
reviewRunId,
|
|
74005
|
+
...findingsRunId !== void 0 ? { findingsRunId } : {},
|
|
74006
|
+
...findingsKey !== void 0 ? { findingsKey } : {},
|
|
74007
|
+
...findings !== void 0 ? { findings } : {},
|
|
74008
|
+
previousEnding: row2.ending,
|
|
74009
|
+
workflowId,
|
|
74010
|
+
deadlineAt,
|
|
74011
|
+
reviewKey
|
|
74012
|
+
}
|
|
74013
|
+
};
|
|
74014
|
+
let replaced;
|
|
74015
|
+
try {
|
|
74016
|
+
replaced = await deps.instances.compareAndReplaceUnit(row2, claimedRow);
|
|
74017
|
+
} catch (err2) {
|
|
74018
|
+
const reread = await deps.instances.listUnits(instance.id).catch(() => void 0);
|
|
74019
|
+
const current = reread?.filter((candidate) => candidate.unit === row2.unit);
|
|
74020
|
+
if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(claimedRow)) replaced = { ok: true };
|
|
74021
|
+
else if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(row2)) {
|
|
74022
|
+
const released = releaseReservation(instance.repo, pr.number, token2);
|
|
74023
|
+
if (!released)
|
|
74024
|
+
return json3(500, { ok: false, error: "recovery_cleanup_failed", cause: "recovery_store_unavailable", at });
|
|
74025
|
+
return json3(503, { ok: false, error: "recovery_store_unavailable", message: describe12(err2), at });
|
|
74026
|
+
} else return json3(503, { ok: false, error: "recovery_claim_unanswered", message: describe12(err2), at });
|
|
74027
|
+
}
|
|
74028
|
+
if (replaced.ok !== true) {
|
|
74029
|
+
const error = replaced.reason === "stale" ? "recovery_claim_stale" : "recovery_store_unavailable";
|
|
74030
|
+
const released = releaseReservation(instance.repo, pr.number, token2);
|
|
74031
|
+
if (!released) return json3(500, { ok: false, error: "recovery_cleanup_failed", cause: error, at });
|
|
74032
|
+
return json3(409, { ok: false, error, at });
|
|
74033
|
+
}
|
|
74034
|
+
if (!fence.transferReservation(instance.repo, pr.number, token2, originalOwner, originalOwner))
|
|
74035
|
+
return rollback("publication_ownership_changed");
|
|
74036
|
+
token2 = void 0;
|
|
74037
|
+
transferred = true;
|
|
74038
|
+
}
|
|
74039
|
+
if (requestedWorkflowId !== void 0)
|
|
74040
|
+
return json3(200, { ok: true, parentInstanceId: instance.id, unit: row2.unit, workflowId, at });
|
|
74041
|
+
const startRecovery = deps.startRecovery;
|
|
74042
|
+
if (startRecovery === void 0) return rollback("recovery_workflow_unavailable");
|
|
74043
|
+
const started = await startRecovery(workflowId, {
|
|
74044
|
+
kind: "recover-original-unit",
|
|
74045
|
+
parentInstanceId: instance.id,
|
|
74046
|
+
unit: row2.unit
|
|
74047
|
+
}).catch((err2) => ({ kind: "unanswered", reason: describe12(err2) }));
|
|
74048
|
+
if (started.kind !== "created" && started.kind !== "duplicate") {
|
|
74049
|
+
if (started.kind === "failed") return rollback("recovery_workflow_failed");
|
|
74050
|
+
return json3(200, {
|
|
74051
|
+
ok: true,
|
|
74052
|
+
outcome: "indeterminate",
|
|
74053
|
+
workflowId,
|
|
74054
|
+
parentInstanceId: instance.id,
|
|
74055
|
+
unit: row2.unit,
|
|
74056
|
+
message: started.reason,
|
|
74057
|
+
at
|
|
74058
|
+
});
|
|
74059
|
+
}
|
|
74060
|
+
if (started.kind === "duplicate" && started.status !== void 0 && ["complete", "errored", "terminated"].includes(started.status))
|
|
74061
|
+
return rollback("recovery_workflow_terminal", true);
|
|
74062
|
+
return json3(200, {
|
|
74063
|
+
ok: true,
|
|
74064
|
+
outcome: started.kind === "duplicate" ? "already_started" : "started",
|
|
74065
|
+
workflowId,
|
|
74066
|
+
parentInstanceId: instance.id,
|
|
74067
|
+
unit: row2.unit,
|
|
74068
|
+
at
|
|
74069
|
+
});
|
|
74070
|
+
}
|
|
73366
74071
|
async function prCheck(body, deps) {
|
|
73367
74072
|
const id = parseInstanceId(body.parentInstanceId);
|
|
73368
74073
|
if (!id.ok) return json3(400, { ok: false, error: id.error });
|
|
@@ -73379,19 +74084,9 @@ async function prCheck(body, deps) {
|
|
|
73379
74084
|
const unit = await unitRowOf(deps, instance, body.unit);
|
|
73380
74085
|
if (!unit.ok) return unit.response;
|
|
73381
74086
|
const branch2 = unit.row?.branch ?? instance.branch;
|
|
73382
|
-
const
|
|
73383
|
-
if (!unit.row) return;
|
|
73384
|
-
|
|
73385
|
-
repo: instance.repo,
|
|
73386
|
-
pr: pr.number,
|
|
73387
|
-
headRef: facts2.headRef,
|
|
73388
|
-
baseRef: facts2.baseRef,
|
|
73389
|
-
expectedHeadSha: facts2.headSha,
|
|
73390
|
-
publicationRef: facts2.headRef,
|
|
73391
|
-
owner: { instanceId: instance.id, unit: unit.row.unit }
|
|
73392
|
-
} : void 0);
|
|
73393
|
-
if (unit.row.pr?.number !== pr.number || unit.row.pr.url !== pr.url || unit.row.publication === void 0 && publication !== void 0)
|
|
73394
|
-
await deps.instances.putUnits([{ ...unit.row, pr, ...publication !== void 0 ? { publication } : {} }]);
|
|
74087
|
+
const rememberTerminal = async (pr) => {
|
|
74088
|
+
if (!unit.row || unit.row.pr?.number === pr.number && unit.row.pr.url === pr.url) return;
|
|
74089
|
+
await deps.instances.putUnits([{ ...unit.row, pr }]);
|
|
73395
74090
|
};
|
|
73396
74091
|
try {
|
|
73397
74092
|
if (follow !== void 0) {
|
|
@@ -73400,12 +74095,13 @@ async function prCheck(body, deps) {
|
|
|
73400
74095
|
const fallbackUrl = unit.row?.pr?.url ?? `https://github.com/${instance.repo}/pull/${follow}`;
|
|
73401
74096
|
const state2 = pullRequestState(follow, fallbackUrl, facts2);
|
|
73402
74097
|
if (state2 === void 0) throw new Error(headBranchStateError(instance.repo, follow));
|
|
73403
|
-
|
|
73404
|
-
if (state2.state !== "open")
|
|
73405
|
-
|
|
73406
|
-
|
|
73407
|
-
|
|
73408
|
-
|
|
74098
|
+
const followedPr = { number: follow, url: String(state2.url) };
|
|
74099
|
+
if (state2.state !== "open") {
|
|
74100
|
+
await rememberTerminal(followedPr);
|
|
74101
|
+
return json3(200, { ok: true, ...state2, at });
|
|
74102
|
+
}
|
|
74103
|
+
const bindingRow = unit.row !== void 0 && unit.row.recovery !== void 0 && recover !== void 0 ? await advanceRecoveryPublication(deps, instance, unit.row, facts2, recover.runId) : unit.row;
|
|
74104
|
+
await bindOpenPullRequest(deps, instance, bindingRow, followedPr, facts2);
|
|
73409
74105
|
const prRef = { repo: instance.repo, number: follow };
|
|
73410
74106
|
const headSha = facts2.headSha;
|
|
73411
74107
|
const approved = entry && headSha !== void 0 ? await reviewPostedAt(deps, prRef, "approve", headSha) : void 0;
|
|
@@ -73436,16 +74132,16 @@ async function prCheck(body, deps) {
|
|
|
73436
74132
|
}
|
|
73437
74133
|
const open2 = await deps.findOpenPrByHead(instance.repo, branch2);
|
|
73438
74134
|
if (open2) {
|
|
73439
|
-
deps.runnerOwnership?.claim(instance.repo, open2.number, {
|
|
73440
|
-
instanceId: instance.id,
|
|
73441
|
-
unit: unit.row?.unit ?? body.unit
|
|
73442
|
-
});
|
|
73443
74135
|
const liveFacts = await deps.fetchPrFacts({ repo: instance.repo, number: open2.number });
|
|
73444
74136
|
if (liveFacts === void 0) throw new Error(`could not read ${instance.repo}#${open2.number}`);
|
|
73445
|
-
await remember({ number: open2.number, url: open2.htmlUrl }, liveFacts);
|
|
73446
74137
|
const current = pullRequestState(open2.number, open2.htmlUrl, liveFacts);
|
|
73447
74138
|
if (current === void 0) throw new Error(headBranchStateError(instance.repo, open2.number));
|
|
73448
|
-
|
|
74139
|
+
const discoveredPr = { number: open2.number, url: open2.htmlUrl };
|
|
74140
|
+
if (current.state !== "open") {
|
|
74141
|
+
await rememberTerminal(discoveredPr);
|
|
74142
|
+
return json3(200, { ok: true, ...current, at });
|
|
74143
|
+
}
|
|
74144
|
+
await bindOpenPullRequest(deps, instance, unit.row, discoveredPr, liveFacts);
|
|
73449
74145
|
const entryFacts = entry ? liveFacts : void 0;
|
|
73450
74146
|
const branchHead = entryFacts?.headSha;
|
|
73451
74147
|
const entryHead = branchHead ?? open2.headSha;
|
|
@@ -73496,22 +74192,27 @@ async function prCheck(body, deps) {
|
|
|
73496
74192
|
}
|
|
73497
74193
|
const recovered = await recoverPushedBranch(deps, instance, unit.row, branch2, recover.runId);
|
|
73498
74194
|
if (recovered.kind === "opened") {
|
|
73499
|
-
await remember({ number: recovered.pr.number, url: recovered.pr.htmlUrl });
|
|
73500
74195
|
const recoveredFacts = await deps.fetchPrFacts({ repo: instance.repo, number: recovered.pr.number });
|
|
73501
74196
|
if (recoveredFacts === void 0) throw new Error(`could not read ${instance.repo}#${recovered.pr.number}`);
|
|
73502
74197
|
const recoveredState = pullRequestState(recovered.pr.number, recovered.pr.htmlUrl, recoveredFacts);
|
|
73503
74198
|
if (recoveredState === void 0) throw new Error(headBranchStateError(instance.repo, recovered.pr.number));
|
|
74199
|
+
const recoveredPr = { number: recovered.pr.number, url: recovered.pr.htmlUrl };
|
|
74200
|
+
if (recoveredState.state === "open")
|
|
74201
|
+
await bindOpenPullRequest(deps, instance, unit.row, recoveredPr, recoveredFacts);
|
|
74202
|
+
else await rememberTerminal(recoveredPr);
|
|
73504
74203
|
return json3(200, { ok: true, ...recoveredState, at });
|
|
73505
74204
|
}
|
|
73506
74205
|
return json3(200, { ok: true, state: "none", unrecovered: recovered.why, at });
|
|
73507
74206
|
}
|
|
73508
|
-
await remember({ number: merged.number, url: merged.htmlUrl });
|
|
73509
74207
|
const mergedFacts = await deps.fetchPrFacts({ repo: instance.repo, number: merged.number });
|
|
73510
74208
|
if (mergedFacts === void 0) throw new Error(`could not read ${instance.repo}#${merged.number}`);
|
|
73511
74209
|
const state = pullRequestState(merged.number, merged.htmlUrl, mergedFacts);
|
|
73512
74210
|
if (state?.state !== "merged") throw new Error(`${instance.repo}#${merged.number} no longer reads merged`);
|
|
74211
|
+
await rememberTerminal({ number: merged.number, url: merged.htmlUrl });
|
|
73513
74212
|
return json3(200, { ok: true, ...state, at });
|
|
73514
74213
|
} catch (err2) {
|
|
74214
|
+
if (err2 instanceof PublicationBindingRefusal)
|
|
74215
|
+
return json3(409, { ok: false, error: err2.reason, message: "the open pull request was not durably bound", at });
|
|
73515
74216
|
return json3(502, { ok: false, error: "github_unavailable", message: describe12(err2), at });
|
|
73516
74217
|
}
|
|
73517
74218
|
}
|
|
@@ -73788,19 +74489,43 @@ async function round(body, deps) {
|
|
|
73788
74489
|
const instance = await deps.instances.get(id.value);
|
|
73789
74490
|
if (!instance) return json3(404, { ok: false, error: "unknown_instance" });
|
|
73790
74491
|
if (namesForeignRun(instance, body)) return json3(404, { ok: false, error: "not_found" });
|
|
73791
|
-
const host = await hostRunOf(deps, instance);
|
|
73792
|
-
if (host.kind === "not_host") return json3(409, { ok: false, error: "not_host", at });
|
|
73793
74492
|
const units = await deps.instances.listUnits(instance.id);
|
|
73794
74493
|
const row2 = units.find((u) => u.unit === body.unit);
|
|
73795
74494
|
if (!row2) return json3(404, { ok: false, error: "unit_not_found", unit: body.unit });
|
|
74495
|
+
const recoveryWorkflowId = typeof body.recoveryWorkflowId === "string" && INSTANCE_ID_PATTERN.test(body.recoveryWorkflowId) ? body.recoveryWorkflowId : void 0;
|
|
74496
|
+
if (body.recoveryWorkflowId !== void 0 && recoveryWorkflowId === void 0)
|
|
74497
|
+
return json3(400, { ok: false, error: "recoveryWorkflowId must be a Workflow instance id", at });
|
|
74498
|
+
if (row2.recovery !== void 0 && recoveryWorkflowId !== row2.recovery.workflowId)
|
|
74499
|
+
return json3(409, { ok: false, error: "recovery_claim_mismatch", at });
|
|
74500
|
+
const host = row2.recovery !== void 0 ? { kind: "not_host" } : await hostRunOf(deps, instance);
|
|
74501
|
+
if (host.kind === "not_host" && row2.recovery === void 0) return json3(409, { ok: false, error: "not_host", at });
|
|
74502
|
+
const note = { index: body.index, agent: body.agent, outcome: body.outcome, at, ...gate ? { gate } : {} };
|
|
74503
|
+
const previous = row2.rounds.at(-1);
|
|
74504
|
+
if (row2.recovery !== void 0 && previous?.index === note.index && previous.agent === note.agent && previous.outcome === note.outcome && JSON.stringify(previous.gate) === JSON.stringify(note.gate))
|
|
74505
|
+
return json3(200, { ok: true, at: previous.at });
|
|
73796
74506
|
const updated = {
|
|
73797
74507
|
...row2,
|
|
73798
|
-
rounds: [
|
|
73799
|
-
...row2.rounds,
|
|
73800
|
-
{ index: body.index, agent: body.agent, outcome: body.outcome, at, ...gate ? { gate } : {} }
|
|
73801
|
-
]
|
|
74508
|
+
rounds: [...row2.rounds, note]
|
|
73802
74509
|
};
|
|
73803
|
-
|
|
74510
|
+
if (row2.recovery !== void 0) {
|
|
74511
|
+
let replaced;
|
|
74512
|
+
try {
|
|
74513
|
+
replaced = await deps.instances.compareAndReplaceUnit(row2, updated);
|
|
74514
|
+
} catch {
|
|
74515
|
+
const reread = await deps.instances.listUnits(instance.id).catch(() => void 0);
|
|
74516
|
+
const current = reread?.filter((candidate) => candidate.unit === row2.unit);
|
|
74517
|
+
if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(updated)) replaced = { ok: true };
|
|
74518
|
+
else if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(row2))
|
|
74519
|
+
return json3(503, { ok: false, error: "recovery_store_unavailable", at });
|
|
74520
|
+
else return json3(409, { ok: false, error: "recovery_claim_stale", at });
|
|
74521
|
+
}
|
|
74522
|
+
if (replaced?.ok !== true)
|
|
74523
|
+
return json3(409, {
|
|
74524
|
+
ok: false,
|
|
74525
|
+
error: replaced?.reason === "stale" ? "recovery_claim_stale" : "recovery_store_unavailable",
|
|
74526
|
+
at
|
|
74527
|
+
});
|
|
74528
|
+
} else await deps.instances.putUnits([updated]);
|
|
73804
74529
|
if (host.kind === "host") {
|
|
73805
74530
|
const thread = unitThread(instance, updated, units.length);
|
|
73806
74531
|
await hostPublish(
|
|
@@ -73880,11 +74605,11 @@ async function unitWake(body, deps) {
|
|
|
73880
74605
|
const instance = await deps.instances.get(id.value);
|
|
73881
74606
|
if (!instance) return json3(404, { ok: false, error: "unknown_instance" });
|
|
73882
74607
|
if (namesForeignRun(instance, body)) return json3(404, { ok: false, error: "not_found" });
|
|
73883
|
-
const host = await hostRunOf(deps, instance);
|
|
73884
|
-
if (host.kind === "not_host") return json3(409, { ok: false, error: "not_host", at });
|
|
73885
74608
|
const units = await deps.instances.listUnits(instance.id);
|
|
73886
74609
|
const row2 = units.find((u) => u.unit === body.unit);
|
|
73887
74610
|
if (!row2) return json3(404, { ok: false, error: "unit_not_found", unit: body.unit });
|
|
74611
|
+
const host = await hostRunOf(deps, instance);
|
|
74612
|
+
if (host.kind === "not_host") return json3(409, { ok: false, error: "not_host", at });
|
|
73888
74613
|
const stored = row2.wakes?.[body.waitId];
|
|
73889
74614
|
if (stored !== void 0) return json3(200, { ok: true, answer: stored, at });
|
|
73890
74615
|
if (!row2.idle) {
|
|
@@ -74037,11 +74762,25 @@ async function unitEnd(body, deps) {
|
|
|
74037
74762
|
const instance = await deps.instances.get(id.value);
|
|
74038
74763
|
if (!instance) return json3(404, { ok: false, error: "unknown_instance" });
|
|
74039
74764
|
if (namesForeignRun(instance, body)) return json3(404, { ok: false, error: "not_found" });
|
|
74040
|
-
const host = await hostRunOf(deps, instance);
|
|
74041
|
-
if (host.kind === "not_host") return json3(409, { ok: false, error: "not_host", at });
|
|
74042
74765
|
const units = await deps.instances.listUnits(instance.id);
|
|
74043
74766
|
const row2 = units.find((u) => u.unit === body.unit);
|
|
74044
74767
|
if (!row2) return json3(404, { ok: false, error: "unit_not_found", unit: body.unit });
|
|
74768
|
+
const recoveryWorkflowId = typeof body.recoveryWorkflowId === "string" && INSTANCE_ID_PATTERN.test(body.recoveryWorkflowId) ? body.recoveryWorkflowId : void 0;
|
|
74769
|
+
if (body.recoveryWorkflowId !== void 0 && recoveryWorkflowId === void 0)
|
|
74770
|
+
return json3(400, { ok: false, error: "recoveryWorkflowId must be a Workflow instance id", at });
|
|
74771
|
+
if (recoveryWorkflowId !== void 0 && row2.recoveryReceipt?.workflowId === recoveryWorkflowId) {
|
|
74772
|
+
if (row2.pr !== void 0) {
|
|
74773
|
+
const owner = { instanceId: instance.id, unit: row2.unit };
|
|
74774
|
+
const current = deps.runnerOwnership?.owner(instance.repo, row2.pr.number);
|
|
74775
|
+
if (current?.instanceId === owner.instanceId && current.unit === owner.unit)
|
|
74776
|
+
deps.runnerOwnership?.release(instance.repo, row2.pr.number, owner);
|
|
74777
|
+
}
|
|
74778
|
+
return json3(200, { ok: true, alreadySettled: true, at: row2.recoveryReceipt.at });
|
|
74779
|
+
}
|
|
74780
|
+
if (row2.recovery !== void 0 && recoveryWorkflowId !== row2.recovery.workflowId)
|
|
74781
|
+
return json3(409, { ok: false, error: "recovery_claim_mismatch", at });
|
|
74782
|
+
const host = row2.recovery !== void 0 ? { kind: "not_host" } : await hostRunOf(deps, instance);
|
|
74783
|
+
if (host.kind === "not_host" && row2.recovery === void 0) return json3(409, { ok: false, error: "not_host", at });
|
|
74045
74784
|
const pr = body.pr;
|
|
74046
74785
|
const lastPush = normalizeHead(body.headSha);
|
|
74047
74786
|
const segment = ending.kind === "continued" ? parseSegment(body.segment) : void 0;
|
|
@@ -74051,9 +74790,30 @@ async function unitEnd(body, deps) {
|
|
|
74051
74790
|
if (ending.kind === "idle" && idle === void 0)
|
|
74052
74791
|
return json3(400, { ok: false, error: "an idle ending must carry its why and the renewals left" });
|
|
74053
74792
|
const segments2 = row2.segments ?? [];
|
|
74054
|
-
const {
|
|
74793
|
+
const recoveryHold = row2.recovery !== void 0 && ending.kind === "held" && isHumanGatePending(ending.humanGate) ? { cause: "human", gate: ending.humanGate } : row2.recovery !== void 0 && ending.kind === "held" && ending.holdCause === "draft" && pr !== void 0 && typeof pr.number === "number" && typeof pr.url === "string" ? { cause: "draft", pr: { number: pr.number, url: pr.url } } : void 0;
|
|
74794
|
+
if (row2.recovery !== void 0 && ending.kind === "held" && recoveryHold === void 0)
|
|
74795
|
+
return json3(400, { ok: false, error: "a recovered held ending must carry a typed hold cause", at });
|
|
74796
|
+
if (row2.recovery !== void 0 && (idle !== void 0 || segment !== void 0))
|
|
74797
|
+
return json3(409, { ok: false, error: "recovery_continuation_unsupported", at });
|
|
74798
|
+
if (row2.recovery !== void 0 && row2.pr !== void 0) {
|
|
74799
|
+
try {
|
|
74800
|
+
if (deps.runnerOwnership?.claim(instance.repo, row2.pr.number, { instanceId: instance.id, unit: row2.unit }) !== true)
|
|
74801
|
+
return json3(409, { ok: false, error: "publication_ownership_changed", at });
|
|
74802
|
+
} catch (err2) {
|
|
74803
|
+
return json3(409, { ok: false, error: "publication_ownership_unknown", message: describe12(err2), at });
|
|
74804
|
+
}
|
|
74805
|
+
}
|
|
74806
|
+
const { idle: _idle, recovery: _recovery, ...rowWithoutLifecycle } = row2;
|
|
74055
74807
|
const updated = {
|
|
74056
|
-
...idle !== void 0 || segment !== void 0 ? row2 :
|
|
74808
|
+
...idle !== void 0 || segment !== void 0 ? row2 : rowWithoutLifecycle,
|
|
74809
|
+
...row2.recovery !== void 0 ? {
|
|
74810
|
+
recoveryReceipt: {
|
|
74811
|
+
reviewRunId: row2.recovery.reviewRunId,
|
|
74812
|
+
workflowId: row2.recovery.workflowId,
|
|
74813
|
+
at
|
|
74814
|
+
}
|
|
74815
|
+
} : {},
|
|
74816
|
+
...recoveryHold !== void 0 ? { recoveryHold } : {},
|
|
74057
74817
|
...pr && typeof pr.number === "number" && typeof pr.url === "string" ? { pr: { number: pr.number, url: pr.url } } : {},
|
|
74058
74818
|
...lastPush !== void 0 ? { lastPush } : {},
|
|
74059
74819
|
...idle !== void 0 ? { idle } : segment !== void 0 ? { segments: segments2.some((s) => s.index === segment.index) ? segments2 : [...segments2, { ...segment, at }] } : {
|
|
@@ -74069,9 +74829,32 @@ async function unitEnd(body, deps) {
|
|
|
74069
74829
|
}
|
|
74070
74830
|
}
|
|
74071
74831
|
};
|
|
74072
|
-
|
|
74832
|
+
if (row2.recovery !== void 0) {
|
|
74833
|
+
let replaced;
|
|
74834
|
+
try {
|
|
74835
|
+
replaced = await deps.instances.compareAndReplaceUnit(row2, updated);
|
|
74836
|
+
} catch {
|
|
74837
|
+
const reread = await deps.instances.listUnits(instance.id).catch(() => void 0);
|
|
74838
|
+
const current = reread?.filter((candidate) => candidate.unit === row2.unit);
|
|
74839
|
+
if (current?.length === 1 && (JSON.stringify(current[0]) === JSON.stringify(updated) || current[0].recovery === void 0 && current[0].recoveryReceipt?.workflowId === row2.recovery.workflowId && current[0].recoveryReceipt?.reviewRunId === row2.recovery.reviewRunId))
|
|
74840
|
+
replaced = { ok: true };
|
|
74841
|
+
else if (current?.length === 1 && JSON.stringify(current[0]) === JSON.stringify(row2))
|
|
74842
|
+
return json3(503, { ok: false, error: "recovery_store_unavailable", at });
|
|
74843
|
+
else return json3(409, { ok: false, error: "recovery_claim_stale", at });
|
|
74844
|
+
}
|
|
74845
|
+
if (replaced?.ok !== true)
|
|
74846
|
+
return json3(409, {
|
|
74847
|
+
ok: false,
|
|
74848
|
+
error: replaced?.reason === "stale" ? "recovery_claim_stale" : "recovery_store_unavailable",
|
|
74849
|
+
at
|
|
74850
|
+
});
|
|
74851
|
+
} else await deps.instances.putUnits([updated]);
|
|
74073
74852
|
if (segment === void 0 && idle === void 0 && updated.pr !== void 0)
|
|
74074
|
-
deps.runnerOwnership?.release(
|
|
74853
|
+
deps.runnerOwnership?.release(
|
|
74854
|
+
instance.repo,
|
|
74855
|
+
updated.pr.number,
|
|
74856
|
+
row2.recovery !== void 0 ? { instanceId: instance.id, unit: row2.unit } : void 0
|
|
74857
|
+
);
|
|
74075
74858
|
const thread = unitThread(instance, updated, units.length);
|
|
74076
74859
|
if (host.kind === "host")
|
|
74077
74860
|
await hostPublish(
|
|
@@ -74092,7 +74875,7 @@ async function unitEnd(body, deps) {
|
|
|
74092
74875
|
at
|
|
74093
74876
|
);
|
|
74094
74877
|
const io = thread.threadKey !== void 0 ? deps.ioFor({ threadKey: thread.threadKey, userId: instance.userId }) : void 0;
|
|
74095
|
-
if (segment === void 0 && idle === void 0) {
|
|
74878
|
+
if (segment === void 0 && idle === void 0 && row2.recovery === void 0) {
|
|
74096
74879
|
const leftovers = await deps.instances.listEvents({ instanceId: instance.id, unit: row2.unit }, true).catch(() => []);
|
|
74097
74880
|
if (leftovers.length > 0 && io === void 0) {
|
|
74098
74881
|
(deps.log ?? console.warn)(
|
|
@@ -74131,6 +74914,13 @@ async function unitEnd(body, deps) {
|
|
|
74131
74914
|
);
|
|
74132
74915
|
}
|
|
74133
74916
|
}
|
|
74917
|
+
if (segment === void 0 && idle === void 0 && row2.recovery !== void 0) {
|
|
74918
|
+
const leftovers = await deps.instances.listEvents({ instanceId: instance.id, unit: row2.unit }, true).catch(() => []);
|
|
74919
|
+
if (leftovers.length > 0)
|
|
74920
|
+
(deps.log ?? console.warn)(
|
|
74921
|
+
`[coordinator] ${instance.id} ${row2.unit}: ${leftovers.length} recovery event(s) stay unconsumed \u2014 terminal recovery never redispatches a replacement pipeline`
|
|
74922
|
+
);
|
|
74923
|
+
}
|
|
74134
74924
|
let told = false;
|
|
74135
74925
|
if (io && threadReport.length === 0)
|
|
74136
74926
|
told = true;
|
|
@@ -74200,10 +74990,15 @@ async function rebaseStep(body, deps) {
|
|
|
74200
74990
|
if (!row2) return json3(404, { ok: false, error: "unknown_unit", at });
|
|
74201
74991
|
if (row2.pr?.number !== void 0 && row2.pr.number !== body.prNumber)
|
|
74202
74992
|
return json3(409, { ok: false, error: "pull_request_moved", at });
|
|
74203
|
-
|
|
74204
|
-
|
|
74205
|
-
|
|
74206
|
-
|
|
74993
|
+
try {
|
|
74994
|
+
if (deps.runnerOwnership?.claim(instance.repo, body.prNumber, {
|
|
74995
|
+
instanceId: instance.id,
|
|
74996
|
+
unit: row2.unit
|
|
74997
|
+
}) !== true)
|
|
74998
|
+
return json3(409, { ok: false, error: "publication_ownership_changed", at });
|
|
74999
|
+
} catch (err2) {
|
|
75000
|
+
return json3(503, { ok: false, error: "publication_ownership_unknown", message: describe12(err2), at });
|
|
75001
|
+
}
|
|
74207
75002
|
if (deps.runnerRebase === void 0)
|
|
74208
75003
|
return json3(200, { ok: true, outcome: "refused", reason: "the runner's rebase resolver is unavailable", at });
|
|
74209
75004
|
try {
|
|
@@ -74365,7 +75160,7 @@ async function merge(body, deps, subject2) {
|
|
|
74365
75160
|
});
|
|
74366
75161
|
if (checks.failed.length > 0) return refused7(`CI is red at \`${headSha.slice(0, 7)}\`: ${checks.failed.join(", ")}`);
|
|
74367
75162
|
if (checks.total === 0) {
|
|
74368
|
-
deps.noteMergeWait?.(headSha, id.value, at);
|
|
75163
|
+
deps.noteMergeWait?.(headSha, unit.row?.recovery?.workflowId ?? id.value, at);
|
|
74369
75164
|
return json3(200, {
|
|
74370
75165
|
ok: true,
|
|
74371
75166
|
outcome: "pending",
|
|
@@ -74374,7 +75169,7 @@ async function merge(body, deps, subject2) {
|
|
|
74374
75169
|
});
|
|
74375
75170
|
}
|
|
74376
75171
|
if (checks.pending.length > 0) {
|
|
74377
|
-
deps.noteMergeWait?.(headSha, id.value, at);
|
|
75172
|
+
deps.noteMergeWait?.(headSha, row2.recovery?.workflowId ?? id.value, at);
|
|
74378
75173
|
return json3(200, {
|
|
74379
75174
|
ok: true,
|
|
74380
75175
|
outcome: "pending",
|
|
@@ -74484,7 +75279,7 @@ async function checksStep(body, deps) {
|
|
|
74484
75279
|
};
|
|
74485
75280
|
}
|
|
74486
75281
|
if (checks === void 0 || checks.pending.length > 0 || (checks.expected?.length ?? 0) > 0 || checks.total === 0 || facts2?.draft === true)
|
|
74487
|
-
deps.noteMergeWait?.(headSha, id.value, at);
|
|
75282
|
+
deps.noteMergeWait?.(headSha, unit.row?.recovery?.workflowId ?? id.value, at);
|
|
74488
75283
|
if (checks !== void 0 && checks.failed.length > 0)
|
|
74489
75284
|
log(
|
|
74490
75285
|
`[coordinator] ${instance.id} ${body.unit}: CI red at ${headSha.slice(0, 7)} \u2014 ${checks.failed.map((f) => `${f.name} (${f.conclusion}${f.flakeSuspect === true ? ", suspected flake" : ""})`).join(", ")}`
|
|
@@ -74689,6 +75484,8 @@ async function answerCoordinatorStep(door, body, deps) {
|
|
|
74689
75484
|
return steerChild(parsed.value, deps);
|
|
74690
75485
|
case "pr-check":
|
|
74691
75486
|
return prCheck(parsed.value, deps);
|
|
75487
|
+
case "recover-unit":
|
|
75488
|
+
return recoverOriginalUnit(parsed.value, deps);
|
|
74692
75489
|
case "round":
|
|
74693
75490
|
return round(parsed.value, deps);
|
|
74694
75491
|
case "unit-end":
|
|
@@ -74734,7 +75531,7 @@ function createAdminCoordinatorHandler(deps) {
|
|
|
74734
75531
|
})();
|
|
74735
75532
|
};
|
|
74736
75533
|
}
|
|
74737
|
-
var COORDINATOR_ADMIN_PREFIX, MAX_SPAWN_PROMPT_CHARS, FINISHED_LOOKBACK_PAGES, MAX_ADMIN_BODY_BYTES, UNIT_ID, PRESETS_OF_KIND, invalid, json3, describe12, isGenerated2, EVERY_RUN, REVIEW_POSTED_CHECKS, REVIEW_POSTED_RECHECK_MS, githubSecondStart, CHILD_SUPERSESSION_REASONS, ROUND_OUTCOMES, ENDING_ICON, STEPS;
|
|
75534
|
+
var COORDINATOR_ADMIN_PREFIX, MAX_SPAWN_PROMPT_CHARS, FINISHED_LOOKBACK_PAGES, MAX_ADMIN_BODY_BYTES, UNIT_ID, PRESETS_OF_KIND, invalid, json3, describe12, isGenerated2, EVERY_RUN, REVIEW_POSTED_CHECKS, REVIEW_POSTED_RECHECK_MS, githubSecondStart, CHILD_SUPERSESSION_REASONS, PublicationBindingRefusal, samePublicationOwner, fullHead, ROUND_OUTCOMES, ENDING_ICON, STEPS;
|
|
74738
75535
|
var init_adminCoordinator = __esm({
|
|
74739
75536
|
"../../src/channels/adminCoordinator.ts"() {
|
|
74740
75537
|
"use strict";
|
|
@@ -74790,6 +75587,15 @@ var init_adminCoordinator = __esm({
|
|
|
74790
75587
|
REVIEW_POSTED_RECHECK_MS = 2e3;
|
|
74791
75588
|
githubSecondStart = (at) => Math.floor(at / SECOND_MS) * SECOND_MS;
|
|
74792
75589
|
CHILD_SUPERSESSION_REASONS = /* @__PURE__ */ new Set(["merged", "closed", "head_moved", "branch_deleted"]);
|
|
75590
|
+
PublicationBindingRefusal = class extends Error {
|
|
75591
|
+
constructor(reason2) {
|
|
75592
|
+
super(reason2);
|
|
75593
|
+
this.reason = reason2;
|
|
75594
|
+
}
|
|
75595
|
+
reason;
|
|
75596
|
+
};
|
|
75597
|
+
samePublicationOwner = (left, right) => left?.instanceId === right.instanceId && left.unit === right.unit;
|
|
75598
|
+
fullHead = (value) => typeof value === "string" && /^[0-9a-f]{40}$/i.test(value);
|
|
74793
75599
|
ROUND_OUTCOMES = [
|
|
74794
75600
|
"started",
|
|
74795
75601
|
"pr_opened",
|
|
@@ -74823,6 +75629,7 @@ var init_adminCoordinator = __esm({
|
|
|
74823
75629
|
"read-record",
|
|
74824
75630
|
"steer",
|
|
74825
75631
|
"pr-check",
|
|
75632
|
+
"recover-unit",
|
|
74826
75633
|
"round",
|
|
74827
75634
|
"unit-end",
|
|
74828
75635
|
"unit-wake",
|
|
@@ -76103,7 +76910,7 @@ async function runBot() {
|
|
|
76103
76910
|
workflow: checksWorkflow,
|
|
76104
76911
|
now: systemClock
|
|
76105
76912
|
});
|
|
76106
|
-
const
|
|
76913
|
+
const coordinatorDeps = {
|
|
76107
76914
|
tokens: processSecrets.get("SWITCHBOARD_INGRESS_TOKENS"),
|
|
76108
76915
|
childAdmission: coordinatorChildAdmission,
|
|
76109
76916
|
grantsFor: (id) => config.grantsFor(id),
|
|
@@ -76213,8 +77020,20 @@ async function runBot() {
|
|
|
76213
77020
|
return classifyRoundChecks(runs, changed, required);
|
|
76214
77021
|
},
|
|
76215
77022
|
rerunFailedChecks: rerunFailedJobs,
|
|
76216
|
-
refirePullRequest: refirePullRequestEvent
|
|
76217
|
-
|
|
77023
|
+
refirePullRequest: refirePullRequestEvent,
|
|
77024
|
+
startRecovery: (id, params) => createInstanceViaShim(processShimOptions(), id, params),
|
|
77025
|
+
recoveryStatus: (id) => fetchInstanceStatusViaShim(processShimOptions(), id)
|
|
77026
|
+
};
|
|
77027
|
+
const coordinatorAdmin = createAdminCoordinatorHandler(coordinatorDeps);
|
|
77028
|
+
deps.recoverOriginalUnit = async (key, caller) => {
|
|
77029
|
+
const answer = await recoverOriginalUnit(
|
|
77030
|
+
{ parentInstanceId: key.instanceId, unit: key.unit },
|
|
77031
|
+
coordinatorDeps,
|
|
77032
|
+
caller
|
|
77033
|
+
);
|
|
77034
|
+
const body = typeof answer.body === "object" && answer.body !== null && !Array.isArray(answer.body) ? answer.body : { error: "original-unit recovery returned an unreadable answer" };
|
|
77035
|
+
return { status: answer.status, body };
|
|
77036
|
+
};
|
|
76218
77037
|
const cronArmed = Object.values(auth.tokens).some((id) => id.subject === "cron");
|
|
76219
77038
|
const schedulesState = `${SCHEDULES.length} schedule(s) on /runs (${capabilities.schedules ? `firings from ${config.config.schedules?.worker?.baseUrl}` : "no firing store"}; cron identity ${cronArmed ? "armed" : "NOT in SWITCHBOARD_INGRESS_TOKENS \u2014 scheduled runs fail closed"})`;
|
|
76220
77039
|
const webAssets = loadWebAssets(webDistDir(publicEnv(), PACKAGE_ROOT));
|