omp-conductor 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/package.json +1 -1
- package/src/approval-surface.ts +8 -3
- package/src/board.ts +134 -7
- package/src/briefs/orchestrator.md +16 -0
- package/src/cli.ts +95 -2
- package/src/daemon.ts +117 -7
- package/src/failure-class.ts +62 -0
- package/src/orchestrator-tick.ts +35 -17
- package/src/store.ts +14 -3
- package/src/tracker/github.ts +16 -0
- package/src/types.ts +12 -0
- package/src/unblock.ts +30 -1
- package/src/verbs/actions.ts +8 -0
- package/src/verbs/protocol.ts +29 -0
- package/src/verbs/server.ts +113 -10
- package/src/worktree.ts +24 -1
package/README.md
CHANGED
|
@@ -1918,6 +1918,7 @@ omp-conductor release-pane [--project NAME]
|
|
|
1918
1918
|
omp-conductor tail <issue> [--project NAME]
|
|
1919
1919
|
omp-conductor extend <issue> --turns N [--project NAME]
|
|
1920
1920
|
omp-conductor unblock <issue> [--force] [--project NAME]
|
|
1921
|
+
omp-conductor verb <conductor_*> [--project NAME] [--arg k=v ...]
|
|
1921
1922
|
omp-conductor friction <escalation-digest|report-noise|report-surprise> --detail TEXT [--issue N] [--project NAME]
|
|
1922
1923
|
omp-conductor report --text TEXT [--kind material|digest] [--project NAME]
|
|
1923
1924
|
omp-conductor decision open --question TEXT [--blocks TEXT] [--resolves-when COND] [--project NAME]
|
|
@@ -1949,6 +1950,7 @@ omp-conductor help
|
|
|
1949
1950
|
| `tail <issue>` | Follow the newest run for that issue: the worker's assistant text as `assistant: …` and each tool it calls as `tool: <name>`, printed as they land. Workers are omp sessions inside the daemon rather than terminals, so this is the only way to watch one live — a herdr pane running it becomes an observation window. Starts from the top of the transcript, not the end, so attaching to a run that is already ten turns in shows those ten turns. Exits `1` with `no run recorded for #N` when the issue has never been dispatched, or `no transcript yet (state: …)` when the attempt has not opened one. Otherwise it runs until `Ctrl-C`, or until the run has finished and its transcript has been silent for five seconds, and prints `run ended: <state>`. |
|
|
1950
1951
|
| `extend <issue> --turns N [--project NAME]` | Monotonically raise that live worker's effective turn ceiling through its owning daemon. The current omp session keeps running; no restart or continuation is created. The daemon persists the new ceiling for `status` and rejects missing, settled, cap-killed, equal, or lower requests instead of implying that an immutable session changed. |
|
|
1951
1952
|
| `unblock <issue> [--force]` | Remove that issue's `blocked` and `failed` labels so an answered escalation can be claimed again. `agent:in-progress` comes off too, but only when the newest recorded run is terminal — that row is the proof no worker still owns the issue, so a live run keeps the label, and so does an issue with no run row at all. Run history remains intact: blocks consume the independent continuation budget, not failed implementation attempts. The output reports both budgets and warns when either will make the next tick escalate instead of dispatch. **Refuses, clearing nothing and exiting `3`, when the newest attempt's work could not be committed and its worktree is the only copy** — re-claiming removes that tree. `--force` records the operator's acceptance on the run row and then clears; the salvage failure stays in history. Exits `2` when the issue number is missing or malformed. |
|
|
1953
|
+
| `verb <conductor_*> [--arg k=v ...]` | Run one [mediated verb](#the-mediated-verbs-126) as the orchestrator, from the CLI — the external-orchestrator half of the verb surface. Every argument goes in as a `--arg k=v` string; an orchestrator can merge (`conductor_pr_merge`), label (`conductor_label`), release (`conductor_release`), update a branch (`conductor_pr_update_branch`) or title/body (`conductor_pr_update`), or read PR state (`conductor_pr_status`). The daemon applies the same checks and writes the same ledger rows a session's call would; a missing `--arg` is refused exactly as a missing tool argument is, worker-only verbs (`conductor_push`, `conductor_pr_create`) are refused with `role-not-allowed`, and a refusal exits `3`. An unknown verb exits `2`. |
|
|
1952
1954
|
| `friction <kind> --detail TEXT [--issue N]` | Record one bounded judgment the daemon cannot infer: an escalation belonged in a digest, or a tick report was noise/surprising. The detail is limited to 160 characters. One event never changes policy; three observations inside seven days make the aggregate eligible for one Learning-loop prompt, followed by a seven-day cooldown. |
|
|
1953
1955
|
| `report --text TEXT [--kind material|digest]` | Hand a rendered report to the daemon's durable outbox. The text is persisted **before** anything is sent and the command prints a report id; the daemon then owns delivery, retries on a bounded backoff, and records the Telegram message id it actually got back. Delivery is [at-least-once](#report-delivery-the-outbox) — a crash mid-send is retried and the retry says it may be a repeat — so `delivered` is never proof of exactly one message. `--kind digest` is accepted at most once per local day, decided from the ledger rather than from what the session remembers sending; an unknown `--kind` exits `2` rather than quietly becoming a material report. Anything undelivered shows in `status` with its age. |
|
|
1954
1956
|
| `decision open --question TEXT [--blocks TEXT] [--resolves-when COND]` | Record a question the orchestrator has put to you, and print its id. A question that lives only in a session's context is lost at the next compaction — after which it is either asked twice or dropped silently. `--resolves-when` attaches a machine-checkable condition: `pr-merged:<https url>`, `issue-closed:<n>`, or `npm-version:<pkg>@<version>`; anything else exits `2` listing the three forms. See [The decision ledger](#the-decision-ledger-136). |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "omp-conductor",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",
|
package/src/approval-surface.ts
CHANGED
|
@@ -55,7 +55,12 @@ import { dirname, join } from "node:path";
|
|
|
55
55
|
* says whether it can be called must name the same thing. */
|
|
56
56
|
export const TELEGRAM_APPROVAL_TOOL = "telegram_ask";
|
|
57
57
|
|
|
58
|
-
export type ApprovalSurface =
|
|
58
|
+
export type ApprovalSurface =
|
|
59
|
+
| { kind: "ready"; /** `notifyMode` from access.json, when the surface read it
|
|
60
|
+
(#169): whether end-of-turn text reaches the operator's
|
|
61
|
+
chat, so the tick's delivery rule can say so truthfully. */
|
|
62
|
+
notifyMode?: string }
|
|
63
|
+
| { kind: "missing"; reason: string };
|
|
59
64
|
|
|
60
65
|
/** Whether the Telegram bridge is running the headless contract, or merely
|
|
61
66
|
* configured well enough to answer a question.
|
|
@@ -257,7 +262,7 @@ export function readApprovalSurface(path: string): ApprovalSurface {
|
|
|
257
262
|
"set notifyChat to the paired owner id",
|
|
258
263
|
};
|
|
259
264
|
}
|
|
260
|
-
return { kind: "ready" };
|
|
265
|
+
return { kind: "ready", notifyMode: mode === undefined ? undefined : String(mode) };
|
|
261
266
|
}
|
|
262
267
|
|
|
263
268
|
// A group destination needs a policy entry, and that policy must admit the
|
|
@@ -283,7 +288,7 @@ export function readApprovalSurface(path: string): ApprovalSurface {
|
|
|
283
288
|
`paired owner (${owner}) in its allowFrom, so their answer would be rejected`,
|
|
284
289
|
};
|
|
285
290
|
}
|
|
286
|
-
return { kind: "ready" };
|
|
291
|
+
return { kind: "ready", notifyMode: mode === undefined ? undefined : String(mode) };
|
|
287
292
|
}
|
|
288
293
|
|
|
289
294
|
/**
|
package/src/board.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { closeSync, openSync, readSync, statSync } from "node:fs";
|
|
1
|
+
import { closeSync, existsSync, openSync, readSync, statSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { emitKeypressEvents } from "node:readline";
|
|
4
4
|
import { findProject, loadConfig, resolveCaps } from "./config.ts";
|
|
@@ -16,10 +16,21 @@ import { dbPath, openStore } from "./store.ts";
|
|
|
16
16
|
import { formatTranscriptLine } from "./transcript.ts";
|
|
17
17
|
import { makeTracker } from "./tracker/github.ts";
|
|
18
18
|
import { planUsageBadge, readPlanUsage, sharedUsageSource } from "./usage.ts";
|
|
19
|
-
import type {
|
|
19
|
+
import type {
|
|
20
|
+
AdmissionHoldReason,
|
|
21
|
+
PrVerification,
|
|
22
|
+
ProjectConfig,
|
|
23
|
+
RunRecord,
|
|
24
|
+
RunState,
|
|
25
|
+
Store,
|
|
26
|
+
} from "./types.ts";
|
|
20
27
|
|
|
21
28
|
const REFRESH_MS = 1_000;
|
|
22
29
|
const HEALTH_REFRESH_MS = 10_000;
|
|
30
|
+
/** Cadence for re-verifying pushed rows against GitHub — deliberately slower
|
|
31
|
+
* than the board's health/label probe: it is one `gh` call per pushed row, and
|
|
32
|
+
* pull-request state does not move at 1 Hz. #173. */
|
|
33
|
+
const PR_REFRESH_MS = 60_000;
|
|
23
34
|
const MERGED_HISTORY_MS = 24 * 60 * 60_000;
|
|
24
35
|
const TRANSCRIPT_BYTES = 64 * 1024;
|
|
25
36
|
const MIN_WIDTH = 50;
|
|
@@ -53,6 +64,7 @@ const COLUMN_DEFS = [
|
|
|
53
64
|
{ key: "blocked", title: "BLOCKED" },
|
|
54
65
|
{ key: "failed", title: "FAILED" },
|
|
55
66
|
{ key: "orphaned", title: "ORPHANED" },
|
|
67
|
+
{ key: "parked", title: "PARKED" },
|
|
56
68
|
{ key: "merged", title: "MERGED" },
|
|
57
69
|
{ key: "history", title: "HISTORY" },
|
|
58
70
|
] as const satisfies readonly { key: string; title: string }[];
|
|
@@ -69,6 +81,23 @@ const LIVE_LANES: Partial<Record<RunState, BoardLane>> = {
|
|
|
69
81
|
"pushed-green": "green",
|
|
70
82
|
};
|
|
71
83
|
|
|
84
|
+
/** Terminal run states that park an issue nobody has acted on: a human cleared
|
|
85
|
+
* the state labels but the issue is still open, so its last run is the story
|
|
86
|
+
* that has to stay visible — #173. Differs from {@link LIVE_LANES} in that
|
|
87
|
+
* these are rows, never current work, and from MERGED in that they are not a
|
|
88
|
+
* happy resolution. */
|
|
89
|
+
const PARKED_STATES = new Set<RunState>(["blocked", "failed", "killed", "orphaned"]);
|
|
90
|
+
|
|
91
|
+
/** Whether a terminal blocked/failed run is wearing its own state label. When
|
|
92
|
+
* the label is absent the run row is the only record of what happened, so the
|
|
93
|
+
* card and detail header say "last run: <state>" rather than presenting it as
|
|
94
|
+
* current — #173, the same distinction that parks the lane. */
|
|
95
|
+
function isLastRun(run: RunRecord, labels: BoardLabels): boolean {
|
|
96
|
+
if (run.state === "blocked") return !labels.blocked.has(run.issue);
|
|
97
|
+
if (run.state === "failed") return !labels.failed.has(run.issue);
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
|
|
72
101
|
type DaemonBoardState = "stopped" | "ok" | "unreachable" | "other-project";
|
|
73
102
|
|
|
74
103
|
/**
|
|
@@ -114,6 +143,11 @@ export interface BoardLabels {
|
|
|
114
143
|
inProgress: ReadonlySet<number>;
|
|
115
144
|
blocked: ReadonlySet<number>;
|
|
116
145
|
failed: ReadonlySet<number>;
|
|
146
|
+
/** Issues whose newest run is terminal and unlabelled but that are still open
|
|
147
|
+
* on the tracker — the PARKED lane (#173), probed on the same 10s cadence as
|
|
148
|
+
* the label sets. An issue missing from this set (closed, or the probe could
|
|
149
|
+
* not tell) keeps today's HISTORY/undefined behaviour rather than parking. */
|
|
150
|
+
open: ReadonlySet<number>;
|
|
117
151
|
/** Epoch ms of the last read that succeeded; 0 when none has. */
|
|
118
152
|
readAt: number;
|
|
119
153
|
/** Message from the most recent failed read. The sets above are then the last
|
|
@@ -129,6 +163,7 @@ const UNREAD_LABELS: BoardLabels = {
|
|
|
129
163
|
inProgress: new Set<number>(),
|
|
130
164
|
blocked: new Set<number>(),
|
|
131
165
|
failed: new Set<number>(),
|
|
166
|
+
open: new Set<number>(),
|
|
132
167
|
readAt: 0,
|
|
133
168
|
};
|
|
134
169
|
|
|
@@ -137,10 +172,18 @@ export interface BoardSnapshot {
|
|
|
137
172
|
status: StatusSnapshot;
|
|
138
173
|
health: BoardHealth;
|
|
139
174
|
labels: BoardLabels;
|
|
175
|
+
/** Live re-verification of pushed rows (#173): run id → what the tracker says
|
|
176
|
+
* the PR looks like now. Rendered as a `now:` suffix on the card. */
|
|
177
|
+
pr: ReadonlyMap<string, { status: PrVerification["status"]; reason: string }>;
|
|
140
178
|
runs: RunRecord[];
|
|
141
179
|
now: number;
|
|
142
180
|
}
|
|
143
181
|
|
|
182
|
+
/** The card state a {@link PrVerification.status} reads as in the UI. */
|
|
183
|
+
function prLabel(status: PrVerification["status"]): string {
|
|
184
|
+
return status === "green" ? "green" : status === "failed" ? "red" : "pending";
|
|
185
|
+
}
|
|
186
|
+
|
|
144
187
|
export interface BoardCursor {
|
|
145
188
|
column: number;
|
|
146
189
|
card: number;
|
|
@@ -166,6 +209,7 @@ function ansiColor(key: BoardLane): string {
|
|
|
166
209
|
case "claimed":
|
|
167
210
|
case "blocked":
|
|
168
211
|
case "orphaned":
|
|
212
|
+
case "parked":
|
|
169
213
|
return YELLOW;
|
|
170
214
|
case "failed":
|
|
171
215
|
return RED;
|
|
@@ -266,6 +310,13 @@ function laneOf(
|
|
|
266
310
|
if (queued.has(issue)) return "queue";
|
|
267
311
|
if (run?.state === "merged") return "merged";
|
|
268
312
|
if (run === undefined) return undefined;
|
|
313
|
+
// PARKED (#173): a terminal run row with no state or queue label left on it,
|
|
314
|
+
// on an issue the tracker says is still open. A human cleared the labels but
|
|
315
|
+
// has not acted — the blank lane the four labels used to imply is a lie, and
|
|
316
|
+
// closed or unreadable issues keep the HISTORY/undefined fallthrough below.
|
|
317
|
+
// Deliberately not subject to MERGED_HISTORY_MS: parked work does not stop
|
|
318
|
+
// being the issue's last word after 24 hours.
|
|
319
|
+
if (PARKED_STATES.has(run.state) && labels.open.has(issue)) return "parked";
|
|
269
320
|
// What is left is a terminal run row with no current claim on it — a closed
|
|
270
321
|
// issue, or one a human already cleared. Recent rows stay readable under
|
|
271
322
|
// HISTORY for the same 24 hours MERGED uses; older ones are not news.
|
|
@@ -424,6 +475,21 @@ function runCardLines(run: RunRecord, snapshot: BoardSnapshot): string[] {
|
|
|
424
475
|
`attempt ${run.attempt} · ${run.turns}/${run.maxTurns}t`,
|
|
425
476
|
`$${run.spendUsd.toFixed(2)} · ${duration}`,
|
|
426
477
|
];
|
|
478
|
+
// #173: a live re-verification of a pushed row, when the 60s probe has one.
|
|
479
|
+
// Guarded on the pushed states: a stale entry must not follow a row that
|
|
480
|
+
// settled into another lane and keep claiming "now: green" about a PR that
|
|
481
|
+
// already merged or failed.
|
|
482
|
+
const live =
|
|
483
|
+
run.state === "pushed-green" || run.state === "pushed-pending" ? snapshot.pr.get(run.id) : undefined;
|
|
484
|
+
if (live !== undefined) {
|
|
485
|
+
const reason = live.reason.split(/\s+/, 1)[0] ?? "";
|
|
486
|
+
lines.push(
|
|
487
|
+
live.status === "failed" ? `now: red — ${reason === "" ? "failed" : reason}` : `now: ${prLabel(live.status)}`,
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
// #173: a terminal blocked/failed run whose state label is gone reads as the
|
|
491
|
+
// lane it sits in — which the label being absent made PARKED, not the truth.
|
|
492
|
+
if (isLastRun(run, snapshot.labels)) lines.push(`last run: ${run.state}`);
|
|
427
493
|
// Ordered by what needs a human first. An unsalvaged tree outranks even a
|
|
428
494
|
// last error: the error describes a run that is over, the tree is work that
|
|
429
495
|
// is still at risk and an issue that will not dispatch (#118).
|
|
@@ -567,6 +633,9 @@ function renderBoardColumns(snapshot: BoardSnapshot, cursor: BoardCursor, width:
|
|
|
567
633
|
|
|
568
634
|
function readTranscript(run: RunRecord): string[] {
|
|
569
635
|
if (run.sessionFile === undefined) return [`No transcript yet (state: ${run.state}).`];
|
|
636
|
+
// #172: a row name that never materialised is "not yet", not a read error —
|
|
637
|
+
// rendering it as `Transcript unavailable` would blame the file for existing.
|
|
638
|
+
if (!existsSync(run.sessionFile)) return [`No transcript yet (state: ${run.state}).`];
|
|
570
639
|
let fd: number | undefined;
|
|
571
640
|
try {
|
|
572
641
|
const size = statSync(run.sessionFile).size;
|
|
@@ -617,8 +686,11 @@ function renderDetail(snapshot: BoardSnapshot, cursor: BoardCursor, width: numbe
|
|
|
617
686
|
}
|
|
618
687
|
|
|
619
688
|
const run = card.run;
|
|
689
|
+
// #173: a blocked/failed run whose label is gone is the reason this card is
|
|
690
|
+
// parked; the header says so rather than presenting the state as current.
|
|
691
|
+
const stateShown = isLastRun(run, snapshot.labels) ? `last run: ${run.state}` : run.state;
|
|
620
692
|
const metadata = [
|
|
621
|
-
styledCell(` RUN #${run.issue} ${run.repo} ${
|
|
693
|
+
styledCell(` RUN #${run.issue} ${run.repo} ${stateShown} `, width, `${BOLD}${REVERSE}`),
|
|
622
694
|
styledCell(`attempt ${run.attempt} · ${run.turns}/${run.maxTurns} turns · $${run.spendUsd.toFixed(2)} · ${humanDuration((run.endedAt ?? snapshot.now) - run.startedAt)}`, width),
|
|
623
695
|
styledCell(`branch ${run.branch}`, width, DIM),
|
|
624
696
|
styledCell(`worktree ${run.worktree || "removed"}`, width, DIM),
|
|
@@ -739,7 +811,11 @@ async function probeBoardHealth(project: ProjectConfig): Promise<BoardHealth> {
|
|
|
739
811
|
* why, because blanking QUEUE and FAILED is a louder lie than showing them
|
|
740
812
|
* stale — {@link labelFact} puts the staleness in the header.
|
|
741
813
|
*/
|
|
742
|
-
async function probeBoardLabels(
|
|
814
|
+
async function probeBoardLabels(
|
|
815
|
+
project: ProjectConfig,
|
|
816
|
+
store: Store,
|
|
817
|
+
previous?: BoardLabels,
|
|
818
|
+
): Promise<BoardLabels> {
|
|
743
819
|
const openIssuesLabelled = async (label: string): Promise<Set<number>> =>
|
|
744
820
|
new Set((await makeTracker({ ...project, queueLabel: label }).listReady()).map((issue) => issue.number));
|
|
745
821
|
try {
|
|
@@ -749,7 +825,26 @@ async function probeBoardLabels(project: ProjectConfig, previous?: BoardLabels):
|
|
|
749
825
|
openIssuesLabelled(project.stateLabels.blocked),
|
|
750
826
|
openIssuesLabelled(project.stateLabels.failed),
|
|
751
827
|
]);
|
|
752
|
-
|
|
828
|
+
// PARKED candidates: the newest run per issue, terminal, non-merged, and
|
|
829
|
+
// wearing no state or queue label. Exactly the rows {@link laneOf} would
|
|
830
|
+
// otherwise fall through on. Bounded by `recentRuns`' own recentJobs limit,
|
|
831
|
+
// issued in parallel, and cached: a failed read keeps the last set.
|
|
832
|
+
const labelled = new Set([...queued, ...inProgress, ...blocked, ...failed]);
|
|
833
|
+
const newestRun = new Map<number, RunRecord>();
|
|
834
|
+
for (const run of store.recentRuns(project.name, Date.now() - MERGED_HISTORY_MS)) {
|
|
835
|
+
if (!newestRun.has(run.issue)) newestRun.set(run.issue, run);
|
|
836
|
+
}
|
|
837
|
+
const parkedCandidates = [...newestRun.values()].filter(
|
|
838
|
+
(run) => PARKED_STATES.has(run.state) && !labelled.has(run.issue),
|
|
839
|
+
);
|
|
840
|
+
const open = new Set<number>();
|
|
841
|
+
await Promise.all(
|
|
842
|
+
parkedCandidates.map(async (run) => {
|
|
843
|
+
const state = await makeTracker(project).issueState(run.issue);
|
|
844
|
+
if (state === "open") open.add(run.issue);
|
|
845
|
+
}),
|
|
846
|
+
);
|
|
847
|
+
return { queued, inProgress, blocked, failed, open, readAt: Date.now() };
|
|
753
848
|
} catch (err) {
|
|
754
849
|
return {
|
|
755
850
|
...(previous ?? UNREAD_LABELS),
|
|
@@ -827,11 +922,17 @@ export async function runBoard(projectName?: string): Promise<void> {
|
|
|
827
922
|
let notice = "";
|
|
828
923
|
let [health, labels, planUsage] = await Promise.all([
|
|
829
924
|
probeBoardHealth(project),
|
|
830
|
-
probeBoardLabels(project),
|
|
925
|
+
probeBoardLabels(project, store),
|
|
831
926
|
readPlanUsage(caps.planUsage, sharedUsageSource()),
|
|
832
927
|
]);
|
|
833
928
|
let healthAt = Date.now();
|
|
834
929
|
let healthRefresh: Promise<void> | undefined;
|
|
930
|
+
// Live re-verification of pushed rows (#173): cached per run id, refreshed on
|
|
931
|
+
// the slow PR cadence so a `gh` call per pushed row does not ride the 1 Hz
|
|
932
|
+
// repaint or the 10 s health gate.
|
|
933
|
+
const prProbe = new Map<string, { status: PrVerification["status"]; reason: string }>();
|
|
934
|
+
let prProbeAt = 0;
|
|
935
|
+
let prProbeRun: Promise<void> | undefined;
|
|
835
936
|
|
|
836
937
|
emitKeypressEvents(process.stdin);
|
|
837
938
|
const onKey = (_text: string, key: KeyInput): void => enqueue(queue, key, wake);
|
|
@@ -855,7 +956,7 @@ export async function runBoard(projectName?: string): Promise<void> {
|
|
|
855
956
|
healthAt = now;
|
|
856
957
|
healthRefresh = Promise.all([
|
|
857
958
|
probeBoardHealth(project),
|
|
858
|
-
probeBoardLabels(project, labels),
|
|
959
|
+
probeBoardLabels(project, store, labels),
|
|
859
960
|
// On the health cadence, not the 1s repaint: the provider read is a
|
|
860
961
|
// subprocess, and an allowance does not move at 1 Hz.
|
|
861
962
|
readPlanUsage(caps.planUsage, sharedUsageSource()),
|
|
@@ -873,11 +974,37 @@ export async function runBoard(projectName?: string): Promise<void> {
|
|
|
873
974
|
healthRefresh = undefined;
|
|
874
975
|
});
|
|
875
976
|
}
|
|
977
|
+
// #173: re-verify pushed rows against the tracker on their own slow
|
|
978
|
+
// cadence, so red-shown-as-green and a moved head surface without paying
|
|
979
|
+
// for the read every second. A failed probe leaves the last verdicts in
|
|
980
|
+
// place (blanking a card's `now:` is a quieter lie than nothing).
|
|
981
|
+
if (now - prProbeAt >= PR_REFRESH_MS && prProbeRun === undefined) {
|
|
982
|
+
prProbeAt = now;
|
|
983
|
+
const rows = store.recentRuns(project.name, now - MERGED_HISTORY_MS).filter(
|
|
984
|
+
(run) => (run.state === "pushed-green" || run.state === "pushed-pending") && run.prUrl !== undefined && run.headSha !== undefined,
|
|
985
|
+
);
|
|
986
|
+
prProbeRun = (async () => {
|
|
987
|
+
const tracker = makeTracker(project);
|
|
988
|
+
await Promise.all(
|
|
989
|
+
rows.map(async (run) => {
|
|
990
|
+
const verified = await tracker.verifyPr(run.prUrl!, run.headSha!);
|
|
991
|
+
if (verified !== undefined) prProbe.set(run.id, { status: verified.status, reason: verified.reason });
|
|
992
|
+
}),
|
|
993
|
+
);
|
|
994
|
+
})()
|
|
995
|
+
.catch((err: unknown) => {
|
|
996
|
+
notice = `PR probe failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
997
|
+
})
|
|
998
|
+
.finally(() => {
|
|
999
|
+
prProbeRun = undefined;
|
|
1000
|
+
});
|
|
1001
|
+
}
|
|
876
1002
|
const snapshot: BoardSnapshot = {
|
|
877
1003
|
project,
|
|
878
1004
|
status: statusSnapshotFromStore(project, caps, store, planUsage),
|
|
879
1005
|
health,
|
|
880
1006
|
labels,
|
|
1007
|
+
pr: prProbe,
|
|
881
1008
|
runs: store.recentRuns(project.name, now - MERGED_HISTORY_MS),
|
|
882
1009
|
now,
|
|
883
1010
|
};
|
|
@@ -290,6 +290,22 @@ what you asked for and what it decided — readable with `omp-conductor ledger`,
|
|
|
290
290
|
including the refusals. That record is the point: a merge or a release the
|
|
291
291
|
dispatcher did not perform is one nobody can account for afterwards.
|
|
292
292
|
|
|
293
|
+
An external orchestrator (a session outside the daemon) runs the same verbs, with
|
|
294
|
+
the same checks and the same ledger rows, through the CLI:
|
|
295
|
+
|
|
296
|
+
```
|
|
297
|
+
omp-conductor verb conductor_pr_merge --arg prUrl=<url> --arg headSha=<sha> --arg reason=<reason>
|
|
298
|
+
omp-conductor verb conductor_label --arg issueUrl=<url> --arg label=<name> --arg action=add --arg reason=<reason>
|
|
299
|
+
omp-conductor verb conductor_release --arg shape=git-tag --arg repo=<name> --arg reason=<reason> --arg tag=<tag>
|
|
300
|
+
omp-conductor verb conductor_pr_update_branch --arg prUrl=<url>
|
|
301
|
+
omp-conductor verb conductor_pr_update --arg prUrl=<url> --arg title=<title>
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
Each `--arg k=v` is one argument, read as a string. Worker-only verbs
|
|
305
|
+
(`conductor_push`, `conductor_pr_create`) are refused with `role-not-allowed` —
|
|
306
|
+
an external orchestrator never pushes or opens a PR, exactly as this floor's
|
|
307
|
+
sessions do not.
|
|
308
|
+
|
|
293
309
|
| Tool | Yours when | What the dispatcher checks before acting |
|
|
294
310
|
| --- | --- | --- |
|
|
295
311
|
| `conductor_pr_status` | always | Nothing to gate: it reads. |
|
package/src/cli.ts
CHANGED
|
@@ -23,7 +23,7 @@ import {
|
|
|
23
23
|
} from "./brief-upgrade.ts";
|
|
24
24
|
import { findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
|
|
25
25
|
import { CONDITION_FORMS, parseCondition } from "./decisions.ts";
|
|
26
|
-
import { runDaemon, setPaused } from "./daemon.ts";
|
|
26
|
+
import { isPaused, pausedAt, runDaemon, setPaused } from "./daemon.ts";
|
|
27
27
|
import {
|
|
28
28
|
armTicks,
|
|
29
29
|
clearPaneHalt,
|
|
@@ -58,7 +58,9 @@ import { dbPath, LIVE_STATES, openStore } from "./store.ts";
|
|
|
58
58
|
import { formatTranscriptLine } from "./transcript.ts";
|
|
59
59
|
import { formatVerbLedgerEntry } from "./verbs/ledger.ts";
|
|
60
60
|
import { makeTracker } from "./tracker/github.ts";
|
|
61
|
-
import {
|
|
61
|
+
import { githubVerbActions } from "./verbs/actions.ts";
|
|
62
|
+
import { handleVerbCall, type VerbChannel } from "./verbs/server.ts";
|
|
63
|
+
import { REPORT_KINDS, VERB_NAMES } from "./types.ts";
|
|
62
64
|
import type { ProjectConfig, ReportKind } from "./types.ts";
|
|
63
65
|
import { formatUnblock, unblockIssue } from "./unblock.ts";
|
|
64
66
|
import { upgradeConductor } from "./upgrade.ts";
|
|
@@ -100,6 +102,7 @@ usage:
|
|
|
100
102
|
omp-conductor tail <issue> [--project NAME]
|
|
101
103
|
omp-conductor extend <issue> --turns N [--project NAME]
|
|
102
104
|
omp-conductor unblock <issue> [--force] [--project NAME]
|
|
105
|
+
omp-conductor verb <conductor_*> [--project NAME] [--arg k=v ...]
|
|
103
106
|
omp-conductor daemon [--once] [--port N] [--project NAME]
|
|
104
107
|
omp-conductor pause
|
|
105
108
|
omp-conductor resume
|
|
@@ -168,6 +171,13 @@ usage:
|
|
|
168
171
|
Refuses when the newest attempt's work could not be committed and
|
|
169
172
|
its worktree is the only copy: re-claiming removes that tree. Use
|
|
170
173
|
--force once you have recovered it or accepted the loss.
|
|
174
|
+
verb run one conductor_* verb as the orchestrator, from the CLI: the same
|
|
175
|
+
checks and the same ledger rows a session's call would get. This is
|
|
176
|
+
how an external orchestrator merges, labels, releases or reads PR
|
|
177
|
+
state without a raw gh call that skips every gate. Arguments are
|
|
178
|
+
strings, one per --arg (e.g. --arg prUrl=https://x --arg headSha=y).
|
|
179
|
+
A refusal exits 3. See conductor_pr_merge/conductor_label/
|
|
180
|
+
conductor_release/conductor_pr_update_branch in the brief.
|
|
171
181
|
report hand a rendered report to the daemon's durable outbox. The report is
|
|
172
182
|
persisted before anything is sent, delivered by the daemon with
|
|
173
183
|
bounded retries, and shown by status until it lands. Delivery is
|
|
@@ -732,6 +742,89 @@ try {
|
|
|
732
742
|
break;
|
|
733
743
|
}
|
|
734
744
|
|
|
745
|
+
/**
|
|
746
|
+
* The external-orchestrator half of the verbs (#167): run any
|
|
747
|
+
* `conductor_*` verb from the CLI, through the daemon's own checks and
|
|
748
|
+
* ledger. An orchestrator that merges, labels or releases through this
|
|
749
|
+
* gets exactly the same refusals a session would and writes the same
|
|
750
|
+
* ledger rows — the raw `gh pr merge` a session reaches for instead is
|
|
751
|
+
* invisible to both.
|
|
752
|
+
*
|
|
753
|
+
* Worker-only verbs (conductor_push, conductor_pr_create) refuse with
|
|
754
|
+
* `role-not-allowed` — correct, and deliberately not special-cased here.
|
|
755
|
+
*
|
|
756
|
+
* Accepted limitation: the daemon's in-process one-merge-per-project slot
|
|
757
|
+
* does not span the daemon and a concurrent CLI merge. The execution-time
|
|
758
|
+
* head re-read (`--match-head-commit` in prMergeVerb) is the cross-process
|
|
759
|
+
* guard, so a CLI merge races a daemon merge exactly as two daemon merges
|
|
760
|
+
* would.
|
|
761
|
+
*/
|
|
762
|
+
case "verb": {
|
|
763
|
+
const name = argv[1];
|
|
764
|
+
if (name === undefined || !VERB_NAMES.some((n) => n === name)) {
|
|
765
|
+
process.stderr.write(
|
|
766
|
+
`omp-conductor: unknown verb "${name ?? ""}". Known verbs: ${VERB_NAMES.join(", ")}\n`,
|
|
767
|
+
);
|
|
768
|
+
process.exit(2);
|
|
769
|
+
}
|
|
770
|
+
const cfg = loadConfig();
|
|
771
|
+
const project = findProject(cfg, flag(argv, "project"));
|
|
772
|
+
const args: Record<string, string> = {};
|
|
773
|
+
for (let i = 2; i < argv.length; i++) {
|
|
774
|
+
const token = argv[i];
|
|
775
|
+
if (token === "--arg") {
|
|
776
|
+
const pair = argv[i + 1];
|
|
777
|
+
if (pair === undefined || !pair.includes("=")) {
|
|
778
|
+
process.stderr.write(`omp-conductor: verb --arg needs k=v, got "${pair ?? ""}"\n`);
|
|
779
|
+
process.exit(2);
|
|
780
|
+
}
|
|
781
|
+
const eq = pair.indexOf("=");
|
|
782
|
+
args[pair.slice(0, eq)] = pair.slice(eq + 1);
|
|
783
|
+
i++;
|
|
784
|
+
} else if (token === "--project" || token?.startsWith("--project=") === true) {
|
|
785
|
+
// Consumed by `flag(argv, "project")` above; both spellings skip here.
|
|
786
|
+
if (token === "--project") i++;
|
|
787
|
+
} else {
|
|
788
|
+
process.stderr.write(
|
|
789
|
+
`omp-conductor: verb: unexpected argument "${token ?? ""}" (pass verb arguments as --arg k=v)\n`,
|
|
790
|
+
);
|
|
791
|
+
process.exit(2);
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
const store = openStore(dbPath());
|
|
795
|
+
try {
|
|
796
|
+
const channel: VerbChannel = {
|
|
797
|
+
kind: "orchestrator",
|
|
798
|
+
path: "cli",
|
|
799
|
+
project: project.name,
|
|
800
|
+
role: "orchestrator",
|
|
801
|
+
};
|
|
802
|
+
const reply = await handleVerbCall(
|
|
803
|
+
{
|
|
804
|
+
project: () => findProject(loadConfig(), project.name),
|
|
805
|
+
store,
|
|
806
|
+
tracker: makeTracker(project),
|
|
807
|
+
actions: githubVerbActions(project),
|
|
808
|
+
fleetStop: () =>
|
|
809
|
+
isPaused()
|
|
810
|
+
? "claiming is paused for this fleet (omp-conductor pause, hold or halt)"
|
|
811
|
+
: undefined,
|
|
812
|
+
pausedAt,
|
|
813
|
+
log: (m) => process.stderr.write(`${m}\n`),
|
|
814
|
+
now: () => Date.now(),
|
|
815
|
+
},
|
|
816
|
+
channel,
|
|
817
|
+
{ verb: name, args },
|
|
818
|
+
);
|
|
819
|
+
process.stdout.write(`${reply.text}\n`);
|
|
820
|
+
// A refusal must not read as success to a script calling this.
|
|
821
|
+
if (!reply.ok) process.exitCode = 3;
|
|
822
|
+
} finally {
|
|
823
|
+
store.close();
|
|
824
|
+
}
|
|
825
|
+
break;
|
|
826
|
+
}
|
|
827
|
+
|
|
735
828
|
/**
|
|
736
829
|
* The handover point. Authorship stays with the model; from here the daemon
|
|
737
830
|
* owns delivery, so "I sent the report" stops being a claim the model makes
|
package/src/daemon.ts
CHANGED
|
@@ -102,6 +102,11 @@ const GRAPH_HEALTH_INTERVAL_MS = 60_000;
|
|
|
102
102
|
* report an operator is waiting on must not sit in the outbox for the length of
|
|
103
103
|
* a poll interval, and delivery is owed even while claiming is paused (#123). */
|
|
104
104
|
const REPORT_DELIVERY_INTERVAL_MS = 30_000;
|
|
105
|
+
/** A dispatch-infra run (turn-0 git failure) is requeued so the next tick
|
|
106
|
+
* retries — but only a bounded number of times. Three strikes for one issue
|
|
107
|
+
* means the mirror itself is broken, not unlucky, and the sweep escalates
|
|
108
|
+
* instead of burning a turn-0 run per tick forever (#168, #177). */
|
|
109
|
+
const DISPATCH_INFRA_MAX_STRIKES = 3;
|
|
105
110
|
const DEFAULT_PORT = 8787;
|
|
106
111
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
107
112
|
|
|
@@ -121,8 +126,13 @@ export interface DaemonOpts {
|
|
|
121
126
|
project?: string;
|
|
122
127
|
}
|
|
123
128
|
|
|
124
|
-
/** Everything one tick touches
|
|
125
|
-
*
|
|
129
|
+
/** Everything one tick touches. `project` and `caps` are re-resolved at each
|
|
130
|
+
* tick boundary so an operator's config edit applies on the next tick rather
|
|
131
|
+
* than the next daemon restart (#170); a tick and the runs it admits see one
|
|
132
|
+
* consistent snapshot, and a mid-run edit never changes a live run's labels,
|
|
133
|
+
* model or caps — `handleIssue` destructures them at dispatch time. The rest
|
|
134
|
+
* are resolved once at startup so a tick never re-reads config mid-flight and
|
|
135
|
+
* changes its own limits underneath itself. */
|
|
126
136
|
interface Deps {
|
|
127
137
|
project: ProjectConfig;
|
|
128
138
|
caps: Caps;
|
|
@@ -176,6 +186,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
176
186
|
isPaused()
|
|
177
187
|
? "claiming is paused for this fleet (omp-conductor pause, hold or halt)"
|
|
178
188
|
: undefined,
|
|
189
|
+
pausedAt,
|
|
179
190
|
log,
|
|
180
191
|
now: () => Date.now(),
|
|
181
192
|
};
|
|
@@ -288,6 +299,30 @@ export function isPaused(): boolean {
|
|
|
288
299
|
return existsSync(join(stateDir(), "paused"));
|
|
289
300
|
}
|
|
290
301
|
|
|
302
|
+
/**
|
|
303
|
+
* The epoch-ms timestamp at which the current pause began, read from the same
|
|
304
|
+
* sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
|
|
305
|
+
* `undefined` when the fleet is not paused, or when the file's first line does
|
|
306
|
+
* not parse as a date — a legacy/blank sentinel keeps today's refuse-everything
|
|
307
|
+
* behavior, because a run admitted before an *unknown* pause cannot be proven
|
|
308
|
+
* innocent. {@link isPaused} is the authority on *whether*; this answers
|
|
309
|
+
* *since when*.
|
|
310
|
+
*/
|
|
311
|
+
export function pausedAt(): number | undefined {
|
|
312
|
+
const f = join(stateDir(), "paused");
|
|
313
|
+
if (!existsSync(f)) return undefined;
|
|
314
|
+
try {
|
|
315
|
+
const first = readFileSync(f, "utf8").split("\n")[0]?.trim();
|
|
316
|
+
if (first === undefined || first === "") return undefined;
|
|
317
|
+
const t = Date.parse(first);
|
|
318
|
+
return Number.isNaN(t) ? undefined : t;
|
|
319
|
+
} catch {
|
|
320
|
+
// Unreadable sentinel (permissions, corruption): fail closed like an
|
|
321
|
+
// unparseable line — refuse mutations while the pause is unprovable.
|
|
322
|
+
return undefined;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
291
326
|
export function setPaused(v: boolean): void {
|
|
292
327
|
const f = join(stateDir(), "paused");
|
|
293
328
|
if (v) {
|
|
@@ -1709,8 +1744,19 @@ export async function admitCandidates(
|
|
|
1709
1744
|
slots: number,
|
|
1710
1745
|
): Promise<AdmissionPass> {
|
|
1711
1746
|
const { project, caps, tracker, store } = d;
|
|
1712
|
-
const
|
|
1747
|
+
const activeRuns = store.activeRuns(project.name);
|
|
1748
|
+
const busyIssues = activeRuns.map((r) => r.issue);
|
|
1713
1749
|
const busy = new Set(busyIssues);
|
|
1750
|
+
// issue -> its active run rows, for the pushed-green admission bypass (#175):
|
|
1751
|
+
// only a worker-free pushed-green row may be bypassed, and only when *every*
|
|
1752
|
+
// active run for the issue is worker-free. A live (claimed/running) row still
|
|
1753
|
+
// holds unconditionally.
|
|
1754
|
+
const activeByIssue = new Map<number, RunRecord[]>();
|
|
1755
|
+
for (const run of activeRuns) {
|
|
1756
|
+
const list = activeByIssue.get(run.issue);
|
|
1757
|
+
if (list === undefined) activeByIssue.set(run.issue, [run]);
|
|
1758
|
+
else list.push(run);
|
|
1759
|
+
}
|
|
1714
1760
|
const holds: AdmissionHold[] = [];
|
|
1715
1761
|
const hold = (issue: number, reason: AdmissionHoldReason): void => {
|
|
1716
1762
|
holds.push({ issue, reason });
|
|
@@ -1769,8 +1815,17 @@ export async function admitCandidates(
|
|
|
1769
1815
|
continue;
|
|
1770
1816
|
}
|
|
1771
1817
|
if (busy.has(issue)) {
|
|
1772
|
-
|
|
1773
|
-
|
|
1818
|
+
// A pushed-green row is worker-free by definition (it is not in
|
|
1819
|
+
// LIVE_STATES): its PR is live but no process is writing to its branch.
|
|
1820
|
+
// So an issue whose active runs are ALL pushed-green is not actually
|
|
1821
|
+
// occupied — the corrective attempt the operator unblocked may be
|
|
1822
|
+
// admitted as a continuation of that PR, and the open-PR gate below
|
|
1823
|
+
// decides the identity. Any live row still holds (#175).
|
|
1824
|
+
const allWorkerFree = (activeByIssue.get(issue) ?? []).every((r) => r.state === "pushed-green");
|
|
1825
|
+
if (!allWorkerFree) {
|
|
1826
|
+
hold(issue, "issue-active");
|
|
1827
|
+
continue;
|
|
1828
|
+
}
|
|
1774
1829
|
}
|
|
1775
1830
|
|
|
1776
1831
|
const priorRuns = store.attemptsFor(project.name, issue);
|
|
@@ -1888,7 +1943,8 @@ export async function admitCandidates(
|
|
|
1888
1943
|
latest?.state === "blocked" ||
|
|
1889
1944
|
latest?.state === "failed" ||
|
|
1890
1945
|
latest?.state === "killed" ||
|
|
1891
|
-
latest?.state === "orphaned"
|
|
1946
|
+
latest?.state === "orphaned" ||
|
|
1947
|
+
latest?.state === "pushed-green"
|
|
1892
1948
|
? latest
|
|
1893
1949
|
: undefined;
|
|
1894
1950
|
// The second half asks "is this open PR our retained work", and accepts
|
|
@@ -1966,6 +2022,24 @@ export async function dispatchAdmissions(
|
|
|
1966
2022
|
// ----------------------------------------------------------------------- a tick
|
|
1967
2023
|
|
|
1968
2024
|
export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
2025
|
+
// A config edit takes effect on the next tick, not the next daemon restart
|
|
2026
|
+
// (#170). Re-resolve the project and its caps at the tick boundary so a tick
|
|
2027
|
+
// and every run it admits see one consistent snapshot; a failed read keeps
|
|
2028
|
+
// the boot values rather than wedging the tick, and the next tick tries
|
|
2029
|
+
// again.
|
|
2030
|
+
try {
|
|
2031
|
+
const cfg = loadConfig();
|
|
2032
|
+
const fresh = findProject(cfg, d.project.name);
|
|
2033
|
+
const freshCaps = resolveCaps(fresh, cfg.defaults);
|
|
2034
|
+
if (JSON.stringify(fresh) !== JSON.stringify(d.project) || JSON.stringify(freshCaps) !== JSON.stringify(d.caps)) {
|
|
2035
|
+
log(`config reloaded: project ${d.project.name} changed on disk — applying from this tick`);
|
|
2036
|
+
}
|
|
2037
|
+
d.project = fresh;
|
|
2038
|
+
d.caps = freshCaps;
|
|
2039
|
+
} catch (err) {
|
|
2040
|
+
log(`config reload failed (${errText(err)}) — continuing with the values loaded at boot`);
|
|
2041
|
+
}
|
|
2042
|
+
|
|
1969
2043
|
// Before the pause check, deliberately. This one is not about dispatch: the
|
|
1970
2044
|
// orchestrator is a different process, and it can be wedged while this fleet
|
|
1971
2045
|
// is paused — which is exactly the state the reference fleet was in when the
|
|
@@ -2643,6 +2717,14 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
2643
2717
|
}
|
|
2644
2718
|
if (run.state === "failed" && facts.pr === "open") {
|
|
2645
2719
|
facts.checks = await tracker.checkConclusions(run.prUrl);
|
|
2720
|
+
// When a check failed with a reachable log, pull its tail so the
|
|
2721
|
+
// table can tell an infra outage (#177) from a real test failure by
|
|
2722
|
+
// the log's own words. First failure wins; a log that cannot be
|
|
2723
|
+
// fetched is left undefined and classification stays conservative.
|
|
2724
|
+
const firstFailure = facts.checks.find((c) => c.state === "failure" && c.link !== undefined);
|
|
2725
|
+
if (firstFailure?.link !== undefined) {
|
|
2726
|
+
facts.failingLog = await tracker.checkLog(firstFailure.link);
|
|
2727
|
+
}
|
|
2646
2728
|
}
|
|
2647
2729
|
}
|
|
2648
2730
|
} catch (err) {
|
|
@@ -2730,6 +2812,26 @@ async function recoverRun(
|
|
|
2730
2812
|
}
|
|
2731
2813
|
|
|
2732
2814
|
if (recovery === "requeue") {
|
|
2815
|
+
// A dispatch-infra requeue that keeps landing on the same issue means the
|
|
2816
|
+
// mirror for its repo is persistently broken — a ref-lock that retry already
|
|
2817
|
+
// exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
|
|
2818
|
+
// with no chance of success, so after a bounded number of strikes this
|
|
2819
|
+
// escalates to a human instead (#168, #177).
|
|
2820
|
+
if (cls === "dispatch-infra" && store.classCountFor(project.name, run.issue, "dispatch-infra") >= DISPATCH_INFRA_MAX_STRIKES) {
|
|
2821
|
+
await safeEscalate(d, {
|
|
2822
|
+
tier: 1,
|
|
2823
|
+
project: project.name,
|
|
2824
|
+
issue: run.issue,
|
|
2825
|
+
summary: `[dispatch-infra] #${run.issue}: the mirror for ${run.repo} is failing persistently — ${evidence}`,
|
|
2826
|
+
detail: [
|
|
2827
|
+
`The dispatcher could not provision a worktree for #${run.issue} ${DISPATCH_INFRA_MAX_STRIKES} times in a row, all before the worker's first turn.`,
|
|
2828
|
+
"The mirror on this host needs attention (check disk, SSH/HTTPS credentials, and the mirror root).",
|
|
2829
|
+
].join("\n"),
|
|
2830
|
+
});
|
|
2831
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
2832
|
+
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
2833
|
+
return;
|
|
2834
|
+
}
|
|
2733
2835
|
// Only when the tracker still shows this issue as ours to hand back. An
|
|
2734
2836
|
// issue that is closed, or has no state label, was resolved by another route
|
|
2735
2837
|
// and requeueing it would dispatch work nobody asked for.
|
|
@@ -2772,7 +2874,15 @@ async function recoverRun(
|
|
|
2772
2874
|
);
|
|
2773
2875
|
}
|
|
2774
2876
|
if (run.lastError !== undefined && cls !== "question") detail.push(run.lastError);
|
|
2775
|
-
|
|
2877
|
+
// #172: an unwritten transcript is "the run died before it flushed", not a
|
|
2878
|
+
// link to a file the operator will open and find missing.
|
|
2879
|
+
detail.push(
|
|
2880
|
+
run.sessionFile === undefined
|
|
2881
|
+
? "Session: (no transcript)"
|
|
2882
|
+
: existsSync(run.sessionFile)
|
|
2883
|
+
? `Session: ${run.sessionFile}`
|
|
2884
|
+
: `Session: ${run.sessionFile} (file was never written — the run died before its transcript was flushed)`,
|
|
2885
|
+
);
|
|
2776
2886
|
// The class and the run are in the summary, which is what the notifications
|
|
2777
2887
|
// ledger dedupes on — so one class escalates once per run rather than every
|
|
2778
2888
|
// five minutes.
|
package/src/failure-class.ts
CHANGED
|
@@ -22,6 +22,10 @@ export interface ClassifyFacts {
|
|
|
22
22
|
pr?: "open" | "merged" | "closed";
|
|
23
23
|
mergeable?: "conflicting" | "clean" | "unknown";
|
|
24
24
|
checks?: { name: string; state: string; link?: string }[];
|
|
25
|
+
/** Tail (ANSI-stripped) of the first failed check's log, when one was
|
|
26
|
+
* reachable. Lets the table tell an infrastructure outage (#177) from a
|
|
27
|
+
* deterministic test failure by the log's own words. */
|
|
28
|
+
failingLog?: string;
|
|
25
29
|
}
|
|
26
30
|
|
|
27
31
|
export interface Classification {
|
|
@@ -50,6 +54,24 @@ const INFRA_CHECK_STATES: Record<string, true> = {
|
|
|
50
54
|
/** States that mean "this check has a verdict and it is good". */
|
|
51
55
|
const SUCCESS_CHECK_STATES: Record<string, true> = { success: true, neutral: true };
|
|
52
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Substrings in a failed check's log that prove the failure was infrastructure,
|
|
59
|
+
* not the diff (#177). Each is a registry/docker/runner fault a worker cannot
|
|
60
|
+
* have introduced: a rate limit, an image-manifest resolution failure, a runner
|
|
61
|
+
* being torn down under the job, or a DNS failure. Matched lowercased against
|
|
62
|
+
* the log tail.
|
|
63
|
+
*
|
|
64
|
+
* Deliberately *not* matching bare `failed to solve:` — a docker build failure
|
|
65
|
+
* often prints it with a real resolution error, so the closing words carry the
|
|
66
|
+
* signal, not the phrase.
|
|
67
|
+
*/
|
|
68
|
+
const INFRA_LOG_SIGNATURES = [
|
|
69
|
+
"429 too many requests",
|
|
70
|
+
"failed to resolve source metadata for",
|
|
71
|
+
"the runner has received a shutdown signal",
|
|
72
|
+
"could not resolve host",
|
|
73
|
+
];
|
|
74
|
+
|
|
53
75
|
function normalise(state: string): string {
|
|
54
76
|
return state.trim().toLowerCase();
|
|
55
77
|
}
|
|
@@ -161,6 +183,28 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
|
|
|
161
183
|
}
|
|
162
184
|
}
|
|
163
185
|
|
|
186
|
+
// Dispatch infra: the conductor's own git path failed before the worker's
|
|
187
|
+
// first turn — a mirror ref-lock (#168), a dead mirror, a transiently
|
|
188
|
+
// unreachable host. The worker never touched the issue, so this should not
|
|
189
|
+
// charge an implementation attempt. `lastError` here is written only by the
|
|
190
|
+
// dispatch catch (`errText(err)` = the stack, which prefixes the thrown
|
|
191
|
+
// "git … exited N: …" message with "Error: ") and by state messages — never
|
|
192
|
+
// by a worker — so that shape is proof of the daemon's own git, not a
|
|
193
|
+
// worker's failed push.
|
|
194
|
+
if (
|
|
195
|
+
run.state === "failed" &&
|
|
196
|
+
run.turns === 0 &&
|
|
197
|
+
!hasArtifacts &&
|
|
198
|
+
run.lastError !== undefined &&
|
|
199
|
+
/^(?:Error: )?git .+ exited \d+/s.test(run.lastError)
|
|
200
|
+
) {
|
|
201
|
+
return {
|
|
202
|
+
cls: "dispatch-infra",
|
|
203
|
+
recovery: "requeue",
|
|
204
|
+
evidence: `dispatch failed in the conductor's own git path before turn 1: ${run.lastError.split("\n")[0]}`,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
164
208
|
if (run.state === "blocked") {
|
|
165
209
|
return {
|
|
166
210
|
cls: "question",
|
|
@@ -215,6 +259,24 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
|
|
|
215
259
|
const checks = facts.checks ?? [];
|
|
216
260
|
const unresolved = checks.filter((c) => !SUCCESS_CHECK_STATES[normalise(c.state)] === true);
|
|
217
261
|
if (checks.length > 0 && unresolved.length > 0) {
|
|
262
|
+
// A failed check whose *log* smells like infrastructure — a registry 429,
|
|
263
|
+
// a runner shutdown, a DNS failure (#177). The check has a verdict, so
|
|
264
|
+
// the check-state table above cannot call it infra; only the log can. It
|
|
265
|
+
// beats ci-deterministic because charging an implementation attempt for a
|
|
266
|
+
// rate limit is exactly the waste that class exists to prevent.
|
|
267
|
+
if (facts.failingLog !== undefined) {
|
|
268
|
+
const lower = facts.failingLog.toLowerCase();
|
|
269
|
+
for (const signature of INFRA_LOG_SIGNATURES) {
|
|
270
|
+
if (lower.includes(signature)) {
|
|
271
|
+
const check = checks.find((c) => normalise(c.state) === "failure");
|
|
272
|
+
return {
|
|
273
|
+
cls: "ci-infra",
|
|
274
|
+
recovery: "rerun-checks",
|
|
275
|
+
evidence: `infrastructure signature in ${check?.name === undefined ? "failed check" : check.name} log: "${signature}"`,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
218
280
|
const failing = unresolved.filter((c) => normalise(c.state) === "failure");
|
|
219
281
|
if (failing.length > 0) {
|
|
220
282
|
return {
|
package/src/orchestrator-tick.ts
CHANGED
|
@@ -400,6 +400,16 @@ export const TICK_SCOPE_CONSTRAINTS: { readonly [K in ReportScope]: string } = {
|
|
|
400
400
|
export const TICK_DELIVERY_RULE =
|
|
401
401
|
"This tick was injected locally, not sent from Telegram, so your end-of-turn text does NOT reach your operator. Hand anything reportable this turn to the durable outbox by running `omp-conductor report --text \"<the whole report>\"` (add `--kind digest` for the daily digest) and confirming it printed a report id; the daemon then owns delivery and retries until it lands. Never claim a report was sent otherwise, and never use telegram_send for a report -- that path leaves no record that it went out.";
|
|
402
402
|
|
|
403
|
+
/** The {@link TICK_DELIVERY_RULE} variant for a fleet whose bridge actually
|
|
404
|
+
* delivers the tick's ending text (#169). That is a narrower class than the
|
|
405
|
+
* approval surface being ready: `notifyMode: "always"` has to be set, and the
|
|
406
|
+
* profile has to be `interactive` — a `daemon` profile suppresses the
|
|
407
|
+
* `agent_end` notify post, so its end-of-turn text reaches nobody and the
|
|
408
|
+
* durable-outbox rule above is the honest one there. Everything here keeps the
|
|
409
|
+
* outbox as the message; only the opening claim about delivery differs. */
|
|
410
|
+
export const TICK_DELIVERY_RULE_BRIDGED =
|
|
411
|
+
"Your end-of-turn text IS delivered to the operator by the Telegram bridge (notifyMode: always). Still hand anything reportable to the durable outbox with omp-conductor report — and then close with at most one short line. The report is the message; never restate it in your closing text.";
|
|
412
|
+
|
|
403
413
|
/** Re-exported so the tick's own contract stays readable from one file: the
|
|
404
414
|
* constant itself lives beside the check that decides whether it is callable. */
|
|
405
415
|
export { TELEGRAM_APPROVAL_TOOL };
|
|
@@ -1195,6 +1205,29 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1195
1205
|
refreshComposedBriefBestEffort();
|
|
1196
1206
|
|
|
1197
1207
|
const scope = resolveTickScope();
|
|
1208
|
+
// The transport contract, read once from the same file at the same moment so
|
|
1209
|
+
// the approval line, the delivery rule and the narration line cannot disagree
|
|
1210
|
+
// (#169, #179). No access file means no fleet channel to judge; a session
|
|
1211
|
+
// with no bound bridge token means omp-telegram posts nothing, so both halves
|
|
1212
|
+
// fail toward "not delivered" — the durable outbox instruction — rather than
|
|
1213
|
+
// asserting a delivery the bridge cannot make.
|
|
1214
|
+
const approval =
|
|
1215
|
+
config.accessFile === undefined
|
|
1216
|
+
? undefined
|
|
1217
|
+
: session.bridgeTokenAtStart
|
|
1218
|
+
? readApprovalSurface(config.accessFile)
|
|
1219
|
+
: ({
|
|
1220
|
+
kind: "missing",
|
|
1221
|
+
reason:
|
|
1222
|
+
`${TELEGRAM_APPROVAL_TOOL} unavailable on local ticks: omp-telegram had no bot token when this ` +
|
|
1223
|
+
"session started, so it bound none and its tools stay dead however complete the access file looks " +
|
|
1224
|
+
"now — run `/telegram on` in this session, or restart it, to rebind the bridge",
|
|
1225
|
+
} as const);
|
|
1226
|
+
const profile =
|
|
1227
|
+
config.accessFile === undefined || !session.bridgeTokenAtStart
|
|
1228
|
+
? undefined
|
|
1229
|
+
: readDaemonProfile(config.accessFile);
|
|
1230
|
+
|
|
1198
1231
|
// A configured message owns the ordinary reporting and delivery clauses.
|
|
1199
1232
|
// Mechanical evidence is different: release-policy drift and repeated
|
|
1200
1233
|
// operational friction must not disappear because an operator customized the
|
|
@@ -1205,7 +1238,8 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1205
1238
|
session.scopeFallbackLogged = true;
|
|
1206
1239
|
pi.logger.info(`[omp-conductor] tick reporting scope: using ${DEFAULT_REPORT_SCOPE} — ${scope.fallback}`);
|
|
1207
1240
|
}
|
|
1208
|
-
|
|
1241
|
+
const bridged = approval?.kind === "ready" && approval.notifyMode === "always" && profile?.kind === "interactive";
|
|
1242
|
+
content = `${defaultTickMessage(new Date(), scope.briefPath, scope.policyPath)}\n${TICK_SCOPE_CONSTRAINTS[scope.scope]}\n${bridged ? TICK_DELIVERY_RULE_BRIDGED : TICK_DELIVERY_RULE}`;
|
|
1209
1243
|
}
|
|
1210
1244
|
let frictionStore: Store | undefined;
|
|
1211
1245
|
let frictionSignals: FrictionSignal[] = [];
|
|
@@ -1275,18 +1309,6 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1275
1309
|
// `telegram_ask` and `telegram_send` dead until `/telegram on`. Trusting the
|
|
1276
1310
|
// file alone there puts the tick straight back to mandating a call its surface
|
|
1277
1311
|
// cannot make, which is #114 exactly.
|
|
1278
|
-
const approval =
|
|
1279
|
-
config.accessFile === undefined
|
|
1280
|
-
? undefined
|
|
1281
|
-
: session.bridgeTokenAtStart
|
|
1282
|
-
? readApprovalSurface(config.accessFile)
|
|
1283
|
-
: ({
|
|
1284
|
-
kind: "missing",
|
|
1285
|
-
reason:
|
|
1286
|
-
`${TELEGRAM_APPROVAL_TOOL} unavailable on local ticks: omp-telegram had no bot token when this ` +
|
|
1287
|
-
"session started, so it bound none and its tools stay dead however complete the access file looks " +
|
|
1288
|
-
"now — run `/telegram on` in this session, or restart it, to rebind the bridge",
|
|
1289
|
-
} as const);
|
|
1290
1312
|
if (approval?.kind === "missing") {
|
|
1291
1313
|
content = `${content}\n${TICK_APPROVAL_UNAVAILABLE_RULE}`;
|
|
1292
1314
|
if (!session.approvalToolMissingLogged) {
|
|
@@ -1318,10 +1340,6 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1318
1340
|
// live bridge whose profile is not `daemon`.
|
|
1319
1341
|
//
|
|
1320
1342
|
// No `accessFile` means no fleet bridge to judge, exactly as above.
|
|
1321
|
-
const profile =
|
|
1322
|
-
config.accessFile === undefined || !session.bridgeTokenAtStart
|
|
1323
|
-
? undefined
|
|
1324
|
-
: readDaemonProfile(config.accessFile);
|
|
1325
1343
|
if (profile?.kind === "interactive") content = `${content}\n${TICK_NARRATION_RULE}`;
|
|
1326
1344
|
|
|
1327
1345
|
try {
|
package/src/store.ts
CHANGED
|
@@ -61,7 +61,7 @@ export const LIVE_STATES: readonly RunState[] = ["claimed", "running"];
|
|
|
61
61
|
* worktrees removed, so they must not consume slots, while their live PRs must
|
|
62
62
|
* still block duplicate attempts.
|
|
63
63
|
*/
|
|
64
|
-
const ACTIVE_STATES: readonly RunState[] = [...LIVE_STATES, "pushed-pending", "pushed-green"];
|
|
64
|
+
export const ACTIVE_STATES: readonly RunState[] = [...LIVE_STATES, "pushed-pending", "pushed-green"];
|
|
65
65
|
|
|
66
66
|
const LIVE_PLACEHOLDERS = LIVE_STATES.map(() => "?").join(", ");
|
|
67
67
|
const ACTIVE_PLACEHOLDERS = ACTIVE_STATES.map(() => "?").join(", ");
|
|
@@ -773,12 +773,19 @@ export function openStore(dbPath: string): Store {
|
|
|
773
773
|
const countFailures = db.query<{ n: number }, [string, number]>(
|
|
774
774
|
`SELECT COUNT(*) AS n FROM runs
|
|
775
775
|
WHERE project = ? AND issue = ? AND state = 'failed'
|
|
776
|
-
AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure'))`,
|
|
776
|
+
AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra'))`,
|
|
777
777
|
);
|
|
778
778
|
const countContinuations = db.query<{ n: number }, [string, number]>(
|
|
779
779
|
`SELECT COUNT(*) AS n FROM runs
|
|
780
780
|
WHERE project = ? AND issue = ? AND state IN ('killed', 'orphaned', 'blocked')
|
|
781
|
-
AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure'))`,
|
|
781
|
+
AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra'))`,
|
|
782
|
+
);
|
|
783
|
+
// How many times one issue reached a given class. Recovery uses it to bound a
|
|
784
|
+
// retry loop whose cause is persistent (e.g. a mirror that will not refresh):
|
|
785
|
+
// after `DISPATCH_INFRA_MAX_STRIKES` the daemon escalates instead of looping.
|
|
786
|
+
const countByClass = db.query<{ n: number }, [string, number, FailureClass]>(
|
|
787
|
+
`SELECT COUNT(*) AS n FROM runs
|
|
788
|
+
WHERE project = ? AND issue = ? AND failureClass = ?`,
|
|
782
789
|
);
|
|
783
790
|
// Newest first, and bounded: every row this returns costs `gh` calls to gather
|
|
784
791
|
// facts for, so a fleet with a long unclassified history classifies over
|
|
@@ -1142,6 +1149,10 @@ export function openStore(dbPath: string): Store {
|
|
|
1142
1149
|
return countContinuations.get(project, issue)?.n ?? 0;
|
|
1143
1150
|
},
|
|
1144
1151
|
|
|
1152
|
+
classCountFor(project: string, issue: number, cls: FailureClass): number {
|
|
1153
|
+
return countByClass.get(project, issue, cls)?.n ?? 0;
|
|
1154
|
+
},
|
|
1155
|
+
|
|
1145
1156
|
latestRun(project: string, issue: number): RunRecord | undefined {
|
|
1146
1157
|
const row = selectLatestRun.get(project, issue);
|
|
1147
1158
|
return row ? toRecord(row) : undefined;
|
package/src/tracker/github.ts
CHANGED
|
@@ -695,6 +695,22 @@ export function makeTracker(p: ProjectConfig, runGh: typeof gh = gh): Tracker {
|
|
|
695
695
|
}
|
|
696
696
|
},
|
|
697
697
|
|
|
698
|
+
async checkLog(detailsUrl: string): Promise<string | undefined> {
|
|
699
|
+
const args = runLogArgs(detailsUrl);
|
|
700
|
+
if (args === undefined) return undefined;
|
|
701
|
+
try {
|
|
702
|
+
const raw = await runGh(args);
|
|
703
|
+
const lines = raw.replaceAll(/\u001b\[[0-9;]*m/g, "").split("\n");
|
|
704
|
+
return lines.slice(-400).join("\n");
|
|
705
|
+
} catch {
|
|
706
|
+
// Best-effort, never throws: a revoked token, a deleted run or a flaky
|
|
707
|
+
// network must not error the classifier out of gathering facts. The
|
|
708
|
+
// caller treats `undefined` as "could not read the log" and classifies
|
|
709
|
+
// conservatively (#177).
|
|
710
|
+
return undefined;
|
|
711
|
+
}
|
|
712
|
+
},
|
|
713
|
+
|
|
698
714
|
async mergeable(prUrl: string): Promise<"conflicting" | "clean" | "unknown"> {
|
|
699
715
|
if (!PR_URL.test(prUrl)) return "unknown";
|
|
700
716
|
try {
|
package/src/types.ts
CHANGED
|
@@ -766,6 +766,12 @@ export interface Tracker {
|
|
|
766
766
|
* port: nothing downstream may read it as "no checks failed".
|
|
767
767
|
*/
|
|
768
768
|
checkConclusions(prUrl: string): Promise<{ name: string; state: string; link?: string }[]>;
|
|
769
|
+
/** The tail of a failed workflow run's log, ANSI stripped, or `undefined`
|
|
770
|
+
* when it cannot be fetched. Best-effort, never throws: the classifier uses
|
|
771
|
+
* it to tell an infrastructure outage from a deterministic test failure
|
|
772
|
+
* (#177), and "could not read the log" must fall through to the existing
|
|
773
|
+
* conservative classification rather than error out of fact-gathering. */
|
|
774
|
+
checkLog(detailsUrl: string): Promise<string | undefined>;
|
|
769
775
|
/** Whether the PR can merge into its base. `unknown` on any doubt, so a
|
|
770
776
|
* mergeability nobody could read never becomes a conflict recovery. */
|
|
771
777
|
mergeable(prUrl: string): Promise<"conflicting" | "clean" | "unknown">;
|
|
@@ -803,6 +809,7 @@ export const FAILURE_CLASSES = [
|
|
|
803
809
|
"admin-kill",
|
|
804
810
|
"ci-infra",
|
|
805
811
|
"ci-deterministic",
|
|
812
|
+
"dispatch-infra",
|
|
806
813
|
"merge-conflict",
|
|
807
814
|
"question",
|
|
808
815
|
"orphan-clean",
|
|
@@ -1155,6 +1162,10 @@ export interface Store {
|
|
|
1155
1162
|
failuresFor(project: string, issue: number): number;
|
|
1156
1163
|
/** Operational stops that require a bounded continuation resume. */
|
|
1157
1164
|
continuationsFor(project: string, issue: number): number;
|
|
1165
|
+
/** How many times one issue has reached a given failure class. A recovery
|
|
1166
|
+
* that keeps landing on the same class (e.g. a persistently broken mirror)
|
|
1167
|
+
* uses this to escalate instead of retrying forever. */
|
|
1168
|
+
classCountFor(project: string, issue: number, cls: FailureClass): number;
|
|
1158
1169
|
/** Newest attempt for one issue, whatever state it reached. `omp-conductor
|
|
1159
1170
|
* tail` resolves an issue number to a transcript through this; the number is
|
|
1160
1171
|
* what an operator has, the run id is not. */
|
|
@@ -1321,6 +1332,7 @@ export const VERB_NAMES = [
|
|
|
1321
1332
|
"conductor_push",
|
|
1322
1333
|
"conductor_pr_create",
|
|
1323
1334
|
"conductor_pr_update_branch",
|
|
1335
|
+
"conductor_pr_update",
|
|
1324
1336
|
"conductor_pr_merge",
|
|
1325
1337
|
"conductor_label",
|
|
1326
1338
|
"conductor_release",
|
package/src/unblock.ts
CHANGED
|
@@ -40,6 +40,10 @@ export interface UnblockOutcome {
|
|
|
40
40
|
continuationsUsed: number;
|
|
41
41
|
/** Newest attempt, when the store has one for this issue at all. */
|
|
42
42
|
latest?: RunRecord;
|
|
43
|
+
/** Whether any run for the issue is still in an active state — the answer
|
|
44
|
+
* to "will the dispatcher hold this issue as issue-active?" that #178
|
|
45
|
+
* found this verb guessing at. */
|
|
46
|
+
active: boolean;
|
|
43
47
|
/** Set when nothing was cleared because the newest attempt's work exists
|
|
44
48
|
* only in its worktree. Carries the salvage failure verbatim. */
|
|
45
49
|
refused?: string;
|
|
@@ -92,10 +96,12 @@ export async function unblockIssue(
|
|
|
92
96
|
// for clearing in-progress, so the row that carries it decides the set.
|
|
93
97
|
const latest = store.latestRun(project.name, issue);
|
|
94
98
|
const terminal = latest !== undefined && !LIVE_STATES.includes(latest.state);
|
|
99
|
+
const active = store.activeRuns(project.name).some((r) => r.issue === issue);
|
|
95
100
|
const counts = {
|
|
96
101
|
attemptsUsed: store.attemptsFor(project.name, issue),
|
|
97
102
|
failuresUsed: store.failuresFor(project.name, issue),
|
|
98
103
|
continuationsUsed: store.continuationsFor(project.name, issue),
|
|
104
|
+
active,
|
|
99
105
|
};
|
|
100
106
|
|
|
101
107
|
// The one case where this verb refuses. Clearing the labels here re-queues an
|
|
@@ -205,8 +211,31 @@ export function formatUnblock(
|
|
|
205
211
|
`run row to prove the worker is gone, "${project.stateLabels.inProgress}" was left in place, and ` +
|
|
206
212
|
"on its own it keeps the issue ineligible",
|
|
207
213
|
);
|
|
214
|
+
} else if (latest.state === "pushed-green" || latest.state === "pushed-pending") {
|
|
215
|
+
// #178: an unblocked issue whose newest run pushed a green PR is not "free
|
|
216
|
+
// for a fresh attempt" — the dispatcher continues the pushed run's branch,
|
|
217
|
+
// and only once the PR resolves does the issue settle (#175). Say that
|
|
218
|
+
// instead of promising a re-claim the next tick withholds.
|
|
219
|
+
lines.push(
|
|
220
|
+
latest.prUrl === undefined
|
|
221
|
+
? ` next tick eligible again, as long as the issue still carries "${project.queueLabel}" ` +
|
|
222
|
+
`(newest run is ${latest.state} with no recorded PR; the dispatcher still applies its open-PR check at claim time)`
|
|
223
|
+
: ` next tick eligible as a continuation of ${latest.prUrl} — the pushed run stays active until ` +
|
|
224
|
+
"that PR resolves; dispatch continues on its branch",
|
|
225
|
+
);
|
|
226
|
+
} else if (o.active) {
|
|
227
|
+
// A run other than the newest is still active (e.g. a live sibling under a
|
|
228
|
+
// terminal newest row), so the dispatcher will hold the issue as
|
|
229
|
+
// issue-active whatever the labels now say. #178's misleading case.
|
|
230
|
+
lines.push(
|
|
231
|
+
` in flight a run for this issue is still active, so the dispatcher holds the issue until it settles — ` +
|
|
232
|
+
`nothing is re-claimed before then ("${project.stateLabels.inProgress}" stays unless already released)`,
|
|
233
|
+
);
|
|
208
234
|
} else {
|
|
209
|
-
lines.push(
|
|
235
|
+
lines.push(
|
|
236
|
+
` next tick eligible again, as long as the issue still carries "${project.queueLabel}" ` +
|
|
237
|
+
"(the dispatcher still applies its open-PR check at claim time)",
|
|
238
|
+
);
|
|
210
239
|
}
|
|
211
240
|
|
|
212
241
|
return lines.join("\n");
|
package/src/verbs/actions.ts
CHANGED
|
@@ -97,6 +97,14 @@ export function githubVerbActions(project: ProjectConfig, run: CommandRunner = s
|
|
|
97
97
|
|
|
98
98
|
updatePrBranch: (prUrl) => gh(["pr", "update-branch", prUrl]),
|
|
99
99
|
|
|
100
|
+
updatePr: async (prUrl, fields) => {
|
|
101
|
+
const argv = ["pr", "edit", prUrl];
|
|
102
|
+
if (fields.title !== undefined) argv.push("--title", fields.title);
|
|
103
|
+
if (fields.body !== undefined) argv.push("--body", fields.body);
|
|
104
|
+
const outcome = await gh(argv);
|
|
105
|
+
return outcome.ok ? { ok: true } : { ok: false, stderr: outcome.stderr };
|
|
106
|
+
},
|
|
107
|
+
|
|
100
108
|
// `--match-head-commit` is the server-side half of the exact-head rule: the
|
|
101
109
|
// daemon already re-read the head and refused a stale one, and this makes
|
|
102
110
|
// GitHub refuse too if the branch moved in the milliseconds between. Belt
|
package/src/verbs/protocol.ts
CHANGED
|
@@ -145,6 +145,35 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
|
|
|
145
145
|
},
|
|
146
146
|
roleRefusalText: (role) => `conductor_pr_update_branch is not open to a ${role} session.`,
|
|
147
147
|
},
|
|
148
|
+
conductor_pr_update: {
|
|
149
|
+
name: "conductor_pr_update",
|
|
150
|
+
mutating: true,
|
|
151
|
+
allowedRoles: ["worker", "orchestrator"],
|
|
152
|
+
description:
|
|
153
|
+
"Update this run's pull request title and/or body. Never touches base, head or state — " +
|
|
154
|
+
"a field you may not change is a field this verb refuses rather than renames. A worker " +
|
|
155
|
+
"may only name its own run's PR.",
|
|
156
|
+
args: {
|
|
157
|
+
prUrl: {
|
|
158
|
+
type: "string",
|
|
159
|
+
required: false,
|
|
160
|
+
description:
|
|
161
|
+
"Full pull request URL. Required on an orchestrator channel; a run channel infers its own PR.",
|
|
162
|
+
},
|
|
163
|
+
title: {
|
|
164
|
+
type: "string",
|
|
165
|
+
required: false,
|
|
166
|
+
description: "New pull request title. At least one of title/body must be a non-empty string.",
|
|
167
|
+
},
|
|
168
|
+
body: {
|
|
169
|
+
type: "string",
|
|
170
|
+
required: false,
|
|
171
|
+
description: "New pull request body. At least one of title/body must be a non-empty string.",
|
|
172
|
+
},
|
|
173
|
+
rationale: RATIONALE_ARG,
|
|
174
|
+
},
|
|
175
|
+
roleRefusalText: (role) => `conductor_pr_update is not open to a ${role} session.`,
|
|
176
|
+
},
|
|
148
177
|
conductor_pr_merge: {
|
|
149
178
|
name: "conductor_pr_merge",
|
|
150
179
|
mutating: true,
|
package/src/verbs/server.ts
CHANGED
|
@@ -145,6 +145,10 @@ export interface VerbActions {
|
|
|
145
145
|
opts: { title: string; body: string; base: string },
|
|
146
146
|
): Promise<{ ok: true; url: string } | { ok: false; stderr: string }>;
|
|
147
147
|
updatePrBranch(prUrl: string): Promise<ActionOutcome>;
|
|
148
|
+
updatePr(
|
|
149
|
+
prUrl: string,
|
|
150
|
+
fields: { title?: string; body?: string },
|
|
151
|
+
): Promise<{ ok: true } | { ok: false; stderr: string }>;
|
|
148
152
|
mergePr(prUrl: string, headSha: string): Promise<ActionOutcome>;
|
|
149
153
|
setLabel(issue: number, label: string, action: "add" | "remove"): Promise<ActionOutcome>;
|
|
150
154
|
release(execution: ReleaseExecution): Promise<ActionOutcome>;
|
|
@@ -167,6 +171,14 @@ export interface VerbDeps {
|
|
|
167
171
|
* sentinel alongside disarming ticks, so one read covers both.
|
|
168
172
|
*/
|
|
169
173
|
fleetStop: () => string | undefined;
|
|
174
|
+
/**
|
|
175
|
+
* The epoch-ms timestamp the current pause began, or `undefined` when not
|
|
176
|
+
* paused (or when the sentinel does not parse). Lets the fleet-stop gate tell
|
|
177
|
+
* a mutating call from a run admitted *before* the pause — that run may
|
|
178
|
+
* finish — apart from one admitted after, which must be refused. Parallels
|
|
179
|
+
* {@link VerbDeps.fleetStop} by living behind the same per-call read.
|
|
180
|
+
*/
|
|
181
|
+
pausedAt: () => number | undefined;
|
|
170
182
|
log: (message: string) => void;
|
|
171
183
|
now: () => number;
|
|
172
184
|
}
|
|
@@ -387,17 +399,11 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
387
399
|
);
|
|
388
400
|
}
|
|
389
401
|
|
|
390
|
-
// 3.
|
|
391
|
-
// exempt: a run finishing its report during a `hold` still has to be able
|
|
392
|
-
// to say what it saw, and refusing that would only make it guess.
|
|
393
|
-
const stopped = spec.mutating ? deps.fleetStop() : undefined;
|
|
394
|
-
if (stopped !== undefined) {
|
|
395
|
-
return refuse("fleet-paused", `refused: ${stopped}. Mutating verbs are refused while the fleet is stopped.`);
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
// 4. The run's own row, re-read. A worker whose run was killed, orphaned or
|
|
402
|
+
// 3. The run's own row, re-read. A worker whose run was killed, orphaned or
|
|
399
403
|
// settled between its last turn and this call is no longer holding
|
|
400
|
-
// anything, and a push from it would land work nothing is tracking.
|
|
404
|
+
// anything, and a push from it would land work nothing is tracking. Read
|
|
405
|
+
// *before* the stop gate below, because the stop gate needs this row to
|
|
406
|
+
// tell an in-flight run (which may finish) from a post-pause call.
|
|
401
407
|
let run: RunRecord | undefined;
|
|
402
408
|
if (channel.kind === "run") {
|
|
403
409
|
run = deps.store.getRun(channel.runId);
|
|
@@ -409,6 +415,22 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
409
415
|
}
|
|
410
416
|
}
|
|
411
417
|
|
|
418
|
+
// 4. Stop always wins, and is re-read here rather than remembered. Reads are
|
|
419
|
+
// exempt: a run finishing its report during a `hold` still has to be able
|
|
420
|
+
// to say what it saw, and refusing that would only make it guess. A run
|
|
421
|
+
// admitted *before* the pause may finish its in-flight mutations — pause
|
|
422
|
+
// stops new claims, not the work already running (#174) — so only a call
|
|
423
|
+
// whose run started at or after the pause (or an orchestrator's call, with
|
|
424
|
+
// no run row at all) is refused.
|
|
425
|
+
const stopped = spec.mutating ? deps.fleetStop() : undefined;
|
|
426
|
+
if (stopped !== undefined) {
|
|
427
|
+
const at = deps.pausedAt();
|
|
428
|
+
const admittedBeforePause = run !== undefined && at !== undefined && run.startedAt < at;
|
|
429
|
+
if (!admittedBeforePause) {
|
|
430
|
+
return refuse("fleet-paused", `refused: ${stopped}. The pause stops new claims and new mutations; a run admitted before the pause may finish.`);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
412
434
|
switch (verb) {
|
|
413
435
|
case "conductor_push":
|
|
414
436
|
return pushVerb(deps, channel, args, refuse, allow);
|
|
@@ -416,6 +438,8 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
416
438
|
return prCreateVerb(deps, project, channel, args, refuse, allow);
|
|
417
439
|
case "conductor_pr_update_branch":
|
|
418
440
|
return prUpdateBranchVerb(deps, project, channel, args, refuse, allow);
|
|
441
|
+
case "conductor_pr_update":
|
|
442
|
+
return prUpdateVerb(deps, project, channel, args, refuse, allow);
|
|
419
443
|
case "conductor_pr_merge":
|
|
420
444
|
return prMergeVerb(deps, project, channel, args, refuse, allow);
|
|
421
445
|
case "conductor_label":
|
|
@@ -530,6 +554,18 @@ async function prCreateVerb(
|
|
|
530
554
|
);
|
|
531
555
|
}
|
|
532
556
|
if (closer !== undefined) {
|
|
557
|
+
// Adoption (#176): a continuation run is never bound to its issue's existing
|
|
558
|
+
// PR — the old row owns the prUrl, and this new row's prUrl stayed NULL — so
|
|
559
|
+
// when the open PR was opened from *this* run's branch, it is the same work.
|
|
560
|
+
// Bind it to this run instead of refusing: pr_status then answers from the
|
|
561
|
+
// live PR, and a merge can act on it. Branch mismatch (or an empty headRefName,
|
|
562
|
+
// which is never identity) keeps the recorded refusal.
|
|
563
|
+
if (closer.headRefName !== "" && closer.headRefName === run.branch) {
|
|
564
|
+
deps.store.updateRun(run.runId, { prUrl: closer.url });
|
|
565
|
+
return allow(
|
|
566
|
+
`adopted ${closer.url} — already open for ${run.branch} and closes #${run.issue}; title/body arguments were not applied`,
|
|
567
|
+
);
|
|
568
|
+
}
|
|
533
569
|
return refuse("open-pr-exists", `refused: ${closer.url} is already open and already closes #${run.issue}.`);
|
|
534
570
|
}
|
|
535
571
|
|
|
@@ -925,6 +961,73 @@ async function prStatusVerb(
|
|
|
925
961
|
return allow(`${prUrl} at ${headSha}: ${verification.status} — ${verification.reason}`, undefined, issue);
|
|
926
962
|
}
|
|
927
963
|
|
|
964
|
+
/**
|
|
965
|
+
* Update one pull request's title and/or body — nothing else.
|
|
966
|
+
*
|
|
967
|
+
* Same resolution as {@link prStatusVerb}: a worker edits its own run's PR (and
|
|
968
|
+
* only that one), an orchestrator names one this project's runs opened.
|
|
969
|
+
* `conductor_pr_update` exists so the floor's own edits go through the same
|
|
970
|
+
* checks and the same ledger as every other mutation, instead of a raw
|
|
971
|
+
* `gh pr edit` from a session that already disclosed it would do exactly that
|
|
972
|
+
* (#176).
|
|
973
|
+
*/
|
|
974
|
+
async function prUpdateVerb(
|
|
975
|
+
deps: VerbDeps,
|
|
976
|
+
project: ProjectConfig,
|
|
977
|
+
channel: VerbChannel,
|
|
978
|
+
args: Record<string, unknown>,
|
|
979
|
+
refuse: Refuse,
|
|
980
|
+
allow: Allow,
|
|
981
|
+
): Promise<Verdict> {
|
|
982
|
+
const asked = args["prUrl"];
|
|
983
|
+
let prUrl: string;
|
|
984
|
+
let issue: number | undefined;
|
|
985
|
+
|
|
986
|
+
if (channel.kind === "run") {
|
|
987
|
+
const own = deps.store.getRun(channel.runId);
|
|
988
|
+
if (own?.prUrl === undefined) {
|
|
989
|
+
return refuse("pr-missing", "refused: this run has no pull request yet. Open one with conductor_pr_create.");
|
|
990
|
+
}
|
|
991
|
+
if (typeof asked === "string" && asked !== own.prUrl) {
|
|
992
|
+
return refuse(
|
|
993
|
+
"pr-not-this-run",
|
|
994
|
+
`refused: this socket belongs to the run for #${channel.issue}, whose pull request is ${own.prUrl}. ` +
|
|
995
|
+
"A worker edits only its own run's PR.",
|
|
996
|
+
);
|
|
997
|
+
}
|
|
998
|
+
prUrl = own.prUrl;
|
|
999
|
+
issue = channel.issue;
|
|
1000
|
+
} else {
|
|
1001
|
+
if (typeof asked !== "string") {
|
|
1002
|
+
return refuse("malformed-argument", "refused: conductor_pr_update needs prUrl when it has no run to infer one from.");
|
|
1003
|
+
}
|
|
1004
|
+
const target = runForPr(deps, project.name, asked);
|
|
1005
|
+
if (target === undefined) {
|
|
1006
|
+
return refuse("pr-not-this-run", `refused: ${asked} is not a pull request any run in ${project.name} opened.`);
|
|
1007
|
+
}
|
|
1008
|
+
prUrl = asked;
|
|
1009
|
+
issue = target.issue;
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
const title = args["title"];
|
|
1013
|
+
const body = args["body"];
|
|
1014
|
+
if (!(typeof title === "string" && title !== "") && !(typeof body === "string" && body !== "")) {
|
|
1015
|
+
return refuse(
|
|
1016
|
+
"malformed-argument",
|
|
1017
|
+
"refused: conductor_pr_update changes nothing unless at least one of title or body is a non-empty string.",
|
|
1018
|
+
);
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
const outcome = await deps.actions.updatePr(prUrl, {
|
|
1022
|
+
...(typeof title === "string" && title !== "" ? { title } : {}),
|
|
1023
|
+
...(typeof body === "string" && body !== "" ? { body } : {}),
|
|
1024
|
+
});
|
|
1025
|
+
if (!outcome.ok) {
|
|
1026
|
+
return refuse("action-failed", `refused: gh could not update the pull request:\n${outcome.stderr}`, issue);
|
|
1027
|
+
}
|
|
1028
|
+
return allow(`updated ${prUrl}.`, undefined, issue);
|
|
1029
|
+
}
|
|
1030
|
+
|
|
928
1031
|
// ------------------------------------------------------------------ the listener
|
|
929
1032
|
|
|
930
1033
|
export interface VerbListener {
|
package/src/worktree.ts
CHANGED
|
@@ -108,6 +108,29 @@ async function gitSucceeds(args: string[], cwd?: string): Promise<boolean> {
|
|
|
108
108
|
return code === 0;
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
/** Fences the block below so it can be found, replaced, and never duplicated. */
|
|
112
|
+
const REF_LOCK_SIGNATURES = /cannot lock ref|unable to update local ref/i;
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Refreshes a bare mirror's refs from its remote. Two dispatch loops can race
|
|
116
|
+
* on git's *packed-refs* / ref locks for the same repo and one lands a transient
|
|
117
|
+
* `cannot lock ref` / `unable to update local ref` — that is not corruption, but
|
|
118
|
+
* an {@link ensureMirror} that throws on it aborts the whole dispatch at turn 0
|
|
119
|
+
* and chars a failed attempt (#168). Retry once after a short backoff; a second
|
|
120
|
+
* failure (or any non-ref-lock failure) propagates unchanged, so a genuinely
|
|
121
|
+
* broken mirror still surfaces loudly.
|
|
122
|
+
*/
|
|
123
|
+
async function refreshMirror(mirrorPath: string): Promise<string> {
|
|
124
|
+
try {
|
|
125
|
+
return await git(["remote", "update", "--prune"], mirrorPath);
|
|
126
|
+
} catch (err) {
|
|
127
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
128
|
+
if (!REF_LOCK_SIGNATURES.test(message)) throw err;
|
|
129
|
+
await new Promise<void>((r) => setTimeout(r, 500));
|
|
130
|
+
return git(["remote", "update", "--prune"], mirrorPath);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
111
134
|
/** Fences the block below so it can be found, replaced, and never duplicated. */
|
|
112
135
|
const EXCLUDE_BEGIN = "# >>> omp-conductor (managed; edit outside this block)";
|
|
113
136
|
const EXCLUDE_END = "# <<< omp-conductor";
|
|
@@ -262,7 +285,7 @@ export async function ensureMirror(
|
|
|
262
285
|
|
|
263
286
|
if (existsSync(mirrorPath)) {
|
|
264
287
|
await configureMirror(mirrorPath);
|
|
265
|
-
await
|
|
288
|
+
await refreshMirror(mirrorPath);
|
|
266
289
|
return mirrorPath;
|
|
267
290
|
}
|
|
268
291
|
|