omp-conductor 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/package.json +1 -1
- package/src/approval-surface.ts +8 -3
- package/src/board.ts +134 -7
- package/src/briefs/orchestrator.md +16 -0
- package/src/cli.ts +95 -2
- package/src/daemon.ts +117 -7
- package/src/failure-class.ts +79 -0
- package/src/orchestrator-tick.ts +35 -17
- package/src/store.ts +57 -4
- package/src/tracker/github.ts +16 -0
- package/src/types.ts +12 -0
- package/src/unblock.ts +30 -1
- package/src/verbs/actions.ts +8 -0
- package/src/verbs/protocol.ts +29 -0
- package/src/verbs/server.ts +113 -10
- package/src/worktree.ts +24 -1
package/src/daemon.ts
CHANGED
|
@@ -102,6 +102,11 @@ const GRAPH_HEALTH_INTERVAL_MS = 60_000;
|
|
|
102
102
|
* report an operator is waiting on must not sit in the outbox for the length of
|
|
103
103
|
* a poll interval, and delivery is owed even while claiming is paused (#123). */
|
|
104
104
|
const REPORT_DELIVERY_INTERVAL_MS = 30_000;
|
|
105
|
+
/** A dispatch-infra run (turn-0 git failure) is requeued so the next tick
|
|
106
|
+
* retries — but only a bounded number of times. Three strikes for one issue
|
|
107
|
+
* means the mirror itself is broken, not unlucky, and the sweep escalates
|
|
108
|
+
* instead of burning a turn-0 run per tick forever (#168, #177). */
|
|
109
|
+
const DISPATCH_INFRA_MAX_STRIKES = 3;
|
|
105
110
|
const DEFAULT_PORT = 8787;
|
|
106
111
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
107
112
|
|
|
@@ -121,8 +126,13 @@ export interface DaemonOpts {
|
|
|
121
126
|
project?: string;
|
|
122
127
|
}
|
|
123
128
|
|
|
124
|
-
/** Everything one tick touches
|
|
125
|
-
*
|
|
129
|
+
/** Everything one tick touches. `project` and `caps` are re-resolved at each
|
|
130
|
+
* tick boundary so an operator's config edit applies on the next tick rather
|
|
131
|
+
* than the next daemon restart (#170); a tick and the runs it admits see one
|
|
132
|
+
* consistent snapshot, and a mid-run edit never changes a live run's labels,
|
|
133
|
+
* model or caps — `handleIssue` destructures them at dispatch time. The rest
|
|
134
|
+
* are resolved once at startup so a tick never re-reads config mid-flight and
|
|
135
|
+
* changes its own limits underneath itself. */
|
|
126
136
|
interface Deps {
|
|
127
137
|
project: ProjectConfig;
|
|
128
138
|
caps: Caps;
|
|
@@ -176,6 +186,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
176
186
|
isPaused()
|
|
177
187
|
? "claiming is paused for this fleet (omp-conductor pause, hold or halt)"
|
|
178
188
|
: undefined,
|
|
189
|
+
pausedAt,
|
|
179
190
|
log,
|
|
180
191
|
now: () => Date.now(),
|
|
181
192
|
};
|
|
@@ -288,6 +299,30 @@ export function isPaused(): boolean {
|
|
|
288
299
|
return existsSync(join(stateDir(), "paused"));
|
|
289
300
|
}
|
|
290
301
|
|
|
302
|
+
/**
|
|
303
|
+
* The epoch-ms timestamp at which the current pause began, read from the same
|
|
304
|
+
* sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
|
|
305
|
+
* `undefined` when the fleet is not paused, or when the file's first line does
|
|
306
|
+
* not parse as a date — a legacy/blank sentinel keeps today's refuse-everything
|
|
307
|
+
* behavior, because a run admitted before an *unknown* pause cannot be proven
|
|
308
|
+
* innocent. {@link isPaused} is the authority on *whether*; this answers
|
|
309
|
+
* *since when*.
|
|
310
|
+
*/
|
|
311
|
+
export function pausedAt(): number | undefined {
|
|
312
|
+
const f = join(stateDir(), "paused");
|
|
313
|
+
if (!existsSync(f)) return undefined;
|
|
314
|
+
try {
|
|
315
|
+
const first = readFileSync(f, "utf8").split("\n")[0]?.trim();
|
|
316
|
+
if (first === undefined || first === "") return undefined;
|
|
317
|
+
const t = Date.parse(first);
|
|
318
|
+
return Number.isNaN(t) ? undefined : t;
|
|
319
|
+
} catch {
|
|
320
|
+
// Unreadable sentinel (permissions, corruption): fail closed like an
|
|
321
|
+
// unparseable line — refuse mutations while the pause is unprovable.
|
|
322
|
+
return undefined;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
291
326
|
export function setPaused(v: boolean): void {
|
|
292
327
|
const f = join(stateDir(), "paused");
|
|
293
328
|
if (v) {
|
|
@@ -1709,8 +1744,19 @@ export async function admitCandidates(
|
|
|
1709
1744
|
slots: number,
|
|
1710
1745
|
): Promise<AdmissionPass> {
|
|
1711
1746
|
const { project, caps, tracker, store } = d;
|
|
1712
|
-
const
|
|
1747
|
+
const activeRuns = store.activeRuns(project.name);
|
|
1748
|
+
const busyIssues = activeRuns.map((r) => r.issue);
|
|
1713
1749
|
const busy = new Set(busyIssues);
|
|
1750
|
+
// issue -> its active run rows, for the pushed-green admission bypass (#175):
|
|
1751
|
+
// only a worker-free pushed-green row may be bypassed, and only when *every*
|
|
1752
|
+
// active run for the issue is worker-free. A live (claimed/running) row still
|
|
1753
|
+
// holds unconditionally.
|
|
1754
|
+
const activeByIssue = new Map<number, RunRecord[]>();
|
|
1755
|
+
for (const run of activeRuns) {
|
|
1756
|
+
const list = activeByIssue.get(run.issue);
|
|
1757
|
+
if (list === undefined) activeByIssue.set(run.issue, [run]);
|
|
1758
|
+
else list.push(run);
|
|
1759
|
+
}
|
|
1714
1760
|
const holds: AdmissionHold[] = [];
|
|
1715
1761
|
const hold = (issue: number, reason: AdmissionHoldReason): void => {
|
|
1716
1762
|
holds.push({ issue, reason });
|
|
@@ -1769,8 +1815,17 @@ export async function admitCandidates(
|
|
|
1769
1815
|
continue;
|
|
1770
1816
|
}
|
|
1771
1817
|
if (busy.has(issue)) {
|
|
1772
|
-
|
|
1773
|
-
|
|
1818
|
+
// A pushed-green row is worker-free by definition (it is not in
|
|
1819
|
+
// LIVE_STATES): its PR is live but no process is writing to its branch.
|
|
1820
|
+
// So an issue whose active runs are ALL pushed-green is not actually
|
|
1821
|
+
// occupied — the corrective attempt the operator unblocked may be
|
|
1822
|
+
// admitted as a continuation of that PR, and the open-PR gate below
|
|
1823
|
+
// decides the identity. Any live row still holds (#175).
|
|
1824
|
+
const allWorkerFree = (activeByIssue.get(issue) ?? []).every((r) => r.state === "pushed-green");
|
|
1825
|
+
if (!allWorkerFree) {
|
|
1826
|
+
hold(issue, "issue-active");
|
|
1827
|
+
continue;
|
|
1828
|
+
}
|
|
1774
1829
|
}
|
|
1775
1830
|
|
|
1776
1831
|
const priorRuns = store.attemptsFor(project.name, issue);
|
|
@@ -1888,7 +1943,8 @@ export async function admitCandidates(
|
|
|
1888
1943
|
latest?.state === "blocked" ||
|
|
1889
1944
|
latest?.state === "failed" ||
|
|
1890
1945
|
latest?.state === "killed" ||
|
|
1891
|
-
latest?.state === "orphaned"
|
|
1946
|
+
latest?.state === "orphaned" ||
|
|
1947
|
+
latest?.state === "pushed-green"
|
|
1892
1948
|
? latest
|
|
1893
1949
|
: undefined;
|
|
1894
1950
|
// The second half asks "is this open PR our retained work", and accepts
|
|
@@ -1966,6 +2022,24 @@ export async function dispatchAdmissions(
|
|
|
1966
2022
|
// ----------------------------------------------------------------------- a tick
|
|
1967
2023
|
|
|
1968
2024
|
export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
2025
|
+
// A config edit takes effect on the next tick, not the next daemon restart
|
|
2026
|
+
// (#170). Re-resolve the project and its caps at the tick boundary so a tick
|
|
2027
|
+
// and every run it admits see one consistent snapshot; a failed read keeps
|
|
2028
|
+
// the boot values rather than wedging the tick, and the next tick tries
|
|
2029
|
+
// again.
|
|
2030
|
+
try {
|
|
2031
|
+
const cfg = loadConfig();
|
|
2032
|
+
const fresh = findProject(cfg, d.project.name);
|
|
2033
|
+
const freshCaps = resolveCaps(fresh, cfg.defaults);
|
|
2034
|
+
if (JSON.stringify(fresh) !== JSON.stringify(d.project) || JSON.stringify(freshCaps) !== JSON.stringify(d.caps)) {
|
|
2035
|
+
log(`config reloaded: project ${d.project.name} changed on disk — applying from this tick`);
|
|
2036
|
+
}
|
|
2037
|
+
d.project = fresh;
|
|
2038
|
+
d.caps = freshCaps;
|
|
2039
|
+
} catch (err) {
|
|
2040
|
+
log(`config reload failed (${errText(err)}) — continuing with the values loaded at boot`);
|
|
2041
|
+
}
|
|
2042
|
+
|
|
1969
2043
|
// Before the pause check, deliberately. This one is not about dispatch: the
|
|
1970
2044
|
// orchestrator is a different process, and it can be wedged while this fleet
|
|
1971
2045
|
// is paused — which is exactly the state the reference fleet was in when the
|
|
@@ -2643,6 +2717,14 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
2643
2717
|
}
|
|
2644
2718
|
if (run.state === "failed" && facts.pr === "open") {
|
|
2645
2719
|
facts.checks = await tracker.checkConclusions(run.prUrl);
|
|
2720
|
+
// When a check failed with a reachable log, pull its tail so the
|
|
2721
|
+
// table can tell an infra outage (#177) from a real test failure by
|
|
2722
|
+
// the log's own words. First failure wins; a log that cannot be
|
|
2723
|
+
// fetched is left undefined and classification stays conservative.
|
|
2724
|
+
const firstFailure = facts.checks.find((c) => c.state === "failure" && c.link !== undefined);
|
|
2725
|
+
if (firstFailure?.link !== undefined) {
|
|
2726
|
+
facts.failingLog = await tracker.checkLog(firstFailure.link);
|
|
2727
|
+
}
|
|
2646
2728
|
}
|
|
2647
2729
|
}
|
|
2648
2730
|
} catch (err) {
|
|
@@ -2730,6 +2812,26 @@ async function recoverRun(
|
|
|
2730
2812
|
}
|
|
2731
2813
|
|
|
2732
2814
|
if (recovery === "requeue") {
|
|
2815
|
+
// A dispatch-infra requeue that keeps landing on the same issue means the
|
|
2816
|
+
// mirror for its repo is persistently broken — a ref-lock that retry already
|
|
2817
|
+
// exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
|
|
2818
|
+
// with no chance of success, so after a bounded number of strikes this
|
|
2819
|
+
// escalates to a human instead (#168, #177).
|
|
2820
|
+
if (cls === "dispatch-infra" && store.classCountFor(project.name, run.issue, "dispatch-infra") >= DISPATCH_INFRA_MAX_STRIKES) {
|
|
2821
|
+
await safeEscalate(d, {
|
|
2822
|
+
tier: 1,
|
|
2823
|
+
project: project.name,
|
|
2824
|
+
issue: run.issue,
|
|
2825
|
+
summary: `[dispatch-infra] #${run.issue}: the mirror for ${run.repo} is failing persistently — ${evidence}`,
|
|
2826
|
+
detail: [
|
|
2827
|
+
`The dispatcher could not provision a worktree for #${run.issue} ${DISPATCH_INFRA_MAX_STRIKES} times in a row, all before the worker's first turn.`,
|
|
2828
|
+
"The mirror on this host needs attention (check disk, SSH/HTTPS credentials, and the mirror root).",
|
|
2829
|
+
].join("\n"),
|
|
2830
|
+
});
|
|
2831
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
2832
|
+
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
2833
|
+
return;
|
|
2834
|
+
}
|
|
2733
2835
|
// Only when the tracker still shows this issue as ours to hand back. An
|
|
2734
2836
|
// issue that is closed, or has no state label, was resolved by another route
|
|
2735
2837
|
// and requeueing it would dispatch work nobody asked for.
|
|
@@ -2772,7 +2874,15 @@ async function recoverRun(
|
|
|
2772
2874
|
);
|
|
2773
2875
|
}
|
|
2774
2876
|
if (run.lastError !== undefined && cls !== "question") detail.push(run.lastError);
|
|
2775
|
-
|
|
2877
|
+
// #172: an unwritten transcript is "the run died before it flushed", not a
|
|
2878
|
+
// link to a file the operator will open and find missing.
|
|
2879
|
+
detail.push(
|
|
2880
|
+
run.sessionFile === undefined
|
|
2881
|
+
? "Session: (no transcript)"
|
|
2882
|
+
: existsSync(run.sessionFile)
|
|
2883
|
+
? `Session: ${run.sessionFile}`
|
|
2884
|
+
: `Session: ${run.sessionFile} (file was never written — the run died before its transcript was flushed)`,
|
|
2885
|
+
);
|
|
2776
2886
|
// The class and the run are in the summary, which is what the notifications
|
|
2777
2887
|
// ledger dedupes on — so one class escalates once per run rather than every
|
|
2778
2888
|
// five minutes.
|
package/src/failure-class.ts
CHANGED
|
@@ -22,6 +22,10 @@ export interface ClassifyFacts {
|
|
|
22
22
|
pr?: "open" | "merged" | "closed";
|
|
23
23
|
mergeable?: "conflicting" | "clean" | "unknown";
|
|
24
24
|
checks?: { name: string; state: string; link?: string }[];
|
|
25
|
+
/** Tail (ANSI-stripped) of the first failed check's log, when one was
|
|
26
|
+
* reachable. Lets the table tell an infrastructure outage (#177) from a
|
|
27
|
+
* deterministic test failure by the log's own words. */
|
|
28
|
+
failingLog?: string;
|
|
25
29
|
}
|
|
26
30
|
|
|
27
31
|
export interface Classification {
|
|
@@ -50,6 +54,24 @@ const INFRA_CHECK_STATES: Record<string, true> = {
|
|
|
50
54
|
/** States that mean "this check has a verdict and it is good". */
|
|
51
55
|
const SUCCESS_CHECK_STATES: Record<string, true> = { success: true, neutral: true };
|
|
52
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Substrings in a failed check's log that prove the failure was infrastructure,
|
|
59
|
+
* not the diff (#177). Each is a registry/docker/runner fault a worker cannot
|
|
60
|
+
* have introduced: a rate limit, an image-manifest resolution failure, a runner
|
|
61
|
+
* being torn down under the job, or a DNS failure. Matched lowercased against
|
|
62
|
+
* the log tail.
|
|
63
|
+
*
|
|
64
|
+
* Deliberately *not* matching bare `failed to solve:` — a docker build failure
|
|
65
|
+
* often prints it with a real resolution error, so the closing words carry the
|
|
66
|
+
* signal, not the phrase.
|
|
67
|
+
*/
|
|
68
|
+
const INFRA_LOG_SIGNATURES = [
|
|
69
|
+
"429 too many requests",
|
|
70
|
+
"failed to resolve source metadata for",
|
|
71
|
+
"the runner has received a shutdown signal",
|
|
72
|
+
"could not resolve host",
|
|
73
|
+
];
|
|
74
|
+
|
|
53
75
|
function normalise(state: string): string {
|
|
54
76
|
return state.trim().toLowerCase();
|
|
55
77
|
}
|
|
@@ -127,6 +149,35 @@ export function neverStarted(
|
|
|
127
149
|
return undefined;
|
|
128
150
|
}
|
|
129
151
|
|
|
152
|
+
/**
|
|
153
|
+
* Evidence that the conductor's own git path failed before the worker's first
|
|
154
|
+
* turn, or `undefined` when this row is something else.
|
|
155
|
+
*
|
|
156
|
+
* A mirror ref-lock (#168), a dead mirror, a transiently unreachable host: the
|
|
157
|
+
* worker never touched the issue, so none of this should charge an
|
|
158
|
+
* implementation attempt. `lastError` of this shape is written only by the
|
|
159
|
+
* dispatch catch (`errText(err)` = the stack, which prefixes the thrown
|
|
160
|
+
* "git … exited N: …" message with "Error: ") and by state messages — never by
|
|
161
|
+
* a worker — so it is proof of the daemon's own git rather than a worker's
|
|
162
|
+
* failed push.
|
|
163
|
+
*
|
|
164
|
+
* Exported for the same reason {@link neverStarted} is: the store's one-time
|
|
165
|
+
* repair has to recognise exactly what the classifier does, or history and
|
|
166
|
+
* future disagree about which rows were the daemon's fault.
|
|
167
|
+
*/
|
|
168
|
+
export function dispatchInfra(
|
|
169
|
+
run: Pick<RunRecord, "turns" | "lastError" | "prUrl" | "headSha" | "salvageSha">,
|
|
170
|
+
): string | undefined {
|
|
171
|
+
if (run.turns !== 0) return undefined;
|
|
172
|
+
if (run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined) return undefined;
|
|
173
|
+
if (run.lastError === undefined || !DISPATCH_GIT_ERROR.test(run.lastError)) return undefined;
|
|
174
|
+
return `dispatch failed in the conductor's own git path before turn 1: ${run.lastError.split("\n")[0]}`;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// See {@link dispatchInfra}: the "Error: " prefix is what `errText` puts in
|
|
178
|
+
// front of the thrown message, so a bare /^git / would never match a real row.
|
|
179
|
+
const DISPATCH_GIT_ERROR = /^(?:Error: )?git .+ exited \d+/s;
|
|
180
|
+
|
|
130
181
|
export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classification {
|
|
131
182
|
const hasArtifacts = run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined;
|
|
132
183
|
|
|
@@ -161,6 +212,16 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
|
|
|
161
212
|
}
|
|
162
213
|
}
|
|
163
214
|
|
|
215
|
+
// Dispatch infra: the conductor's own git path failed before the worker's
|
|
216
|
+
// first turn. See {@link dispatchInfra} — the worker never touched the issue,
|
|
217
|
+
// so this must not charge an implementation attempt.
|
|
218
|
+
if (run.state === "failed") {
|
|
219
|
+
const evidence = dispatchInfra(run);
|
|
220
|
+
if (evidence !== undefined) {
|
|
221
|
+
return { cls: "dispatch-infra", recovery: "requeue", evidence };
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
164
225
|
if (run.state === "blocked") {
|
|
165
226
|
return {
|
|
166
227
|
cls: "question",
|
|
@@ -215,6 +276,24 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
|
|
|
215
276
|
const checks = facts.checks ?? [];
|
|
216
277
|
const unresolved = checks.filter((c) => !SUCCESS_CHECK_STATES[normalise(c.state)] === true);
|
|
217
278
|
if (checks.length > 0 && unresolved.length > 0) {
|
|
279
|
+
// A failed check whose *log* smells like infrastructure — a registry 429,
|
|
280
|
+
// a runner shutdown, a DNS failure (#177). The check has a verdict, so
|
|
281
|
+
// the check-state table above cannot call it infra; only the log can. It
|
|
282
|
+
// beats ci-deterministic because charging an implementation attempt for a
|
|
283
|
+
// rate limit is exactly the waste that class exists to prevent.
|
|
284
|
+
if (facts.failingLog !== undefined) {
|
|
285
|
+
const lower = facts.failingLog.toLowerCase();
|
|
286
|
+
for (const signature of INFRA_LOG_SIGNATURES) {
|
|
287
|
+
if (lower.includes(signature)) {
|
|
288
|
+
const check = checks.find((c) => normalise(c.state) === "failure");
|
|
289
|
+
return {
|
|
290
|
+
cls: "ci-infra",
|
|
291
|
+
recovery: "rerun-checks",
|
|
292
|
+
evidence: `infrastructure signature in ${check?.name === undefined ? "failed check" : check.name} log: "${signature}"`,
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
218
297
|
const failing = unresolved.filter((c) => normalise(c.state) === "failure");
|
|
219
298
|
if (failing.length > 0) {
|
|
220
299
|
return {
|
package/src/orchestrator-tick.ts
CHANGED
|
@@ -400,6 +400,16 @@ export const TICK_SCOPE_CONSTRAINTS: { readonly [K in ReportScope]: string } = {
|
|
|
400
400
|
export const TICK_DELIVERY_RULE =
|
|
401
401
|
"This tick was injected locally, not sent from Telegram, so your end-of-turn text does NOT reach your operator. Hand anything reportable this turn to the durable outbox by running `omp-conductor report --text \"<the whole report>\"` (add `--kind digest` for the daily digest) and confirming it printed a report id; the daemon then owns delivery and retries until it lands. Never claim a report was sent otherwise, and never use telegram_send for a report -- that path leaves no record that it went out.";
|
|
402
402
|
|
|
403
|
+
/** The {@link TICK_DELIVERY_RULE} variant for a fleet whose bridge actually
|
|
404
|
+
* delivers the tick's ending text (#169). That is a narrower class than the
|
|
405
|
+
* approval surface being ready: `notifyMode: "always"` has to be set, and the
|
|
406
|
+
* profile has to be `interactive` — a `daemon` profile suppresses the
|
|
407
|
+
* `agent_end` notify post, so its end-of-turn text reaches nobody and the
|
|
408
|
+
* durable-outbox rule above is the honest one there. Everything here keeps the
|
|
409
|
+
* outbox as the message; only the opening claim about delivery differs. */
|
|
410
|
+
export const TICK_DELIVERY_RULE_BRIDGED =
|
|
411
|
+
"Your end-of-turn text IS delivered to the operator by the Telegram bridge (notifyMode: always). Still hand anything reportable to the durable outbox with omp-conductor report — and then close with at most one short line. The report is the message; never restate it in your closing text.";
|
|
412
|
+
|
|
403
413
|
/** Re-exported so the tick's own contract stays readable from one file: the
|
|
404
414
|
* constant itself lives beside the check that decides whether it is callable. */
|
|
405
415
|
export { TELEGRAM_APPROVAL_TOOL };
|
|
@@ -1195,6 +1205,29 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1195
1205
|
refreshComposedBriefBestEffort();
|
|
1196
1206
|
|
|
1197
1207
|
const scope = resolveTickScope();
|
|
1208
|
+
// The transport contract, read once from the same file at the same moment so
|
|
1209
|
+
// the approval line, the delivery rule and the narration line cannot disagree
|
|
1210
|
+
// (#169, #179). No access file means no fleet channel to judge; a session
|
|
1211
|
+
// with no bound bridge token means omp-telegram posts nothing, so both halves
|
|
1212
|
+
// fail toward "not delivered" — the durable outbox instruction — rather than
|
|
1213
|
+
// asserting a delivery the bridge cannot make.
|
|
1214
|
+
const approval =
|
|
1215
|
+
config.accessFile === undefined
|
|
1216
|
+
? undefined
|
|
1217
|
+
: session.bridgeTokenAtStart
|
|
1218
|
+
? readApprovalSurface(config.accessFile)
|
|
1219
|
+
: ({
|
|
1220
|
+
kind: "missing",
|
|
1221
|
+
reason:
|
|
1222
|
+
`${TELEGRAM_APPROVAL_TOOL} unavailable on local ticks: omp-telegram had no bot token when this ` +
|
|
1223
|
+
"session started, so it bound none and its tools stay dead however complete the access file looks " +
|
|
1224
|
+
"now — run `/telegram on` in this session, or restart it, to rebind the bridge",
|
|
1225
|
+
} as const);
|
|
1226
|
+
const profile =
|
|
1227
|
+
config.accessFile === undefined || !session.bridgeTokenAtStart
|
|
1228
|
+
? undefined
|
|
1229
|
+
: readDaemonProfile(config.accessFile);
|
|
1230
|
+
|
|
1198
1231
|
// A configured message owns the ordinary reporting and delivery clauses.
|
|
1199
1232
|
// Mechanical evidence is different: release-policy drift and repeated
|
|
1200
1233
|
// operational friction must not disappear because an operator customized the
|
|
@@ -1205,7 +1238,8 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1205
1238
|
session.scopeFallbackLogged = true;
|
|
1206
1239
|
pi.logger.info(`[omp-conductor] tick reporting scope: using ${DEFAULT_REPORT_SCOPE} — ${scope.fallback}`);
|
|
1207
1240
|
}
|
|
1208
|
-
|
|
1241
|
+
const bridged = approval?.kind === "ready" && approval.notifyMode === "always" && profile?.kind === "interactive";
|
|
1242
|
+
content = `${defaultTickMessage(new Date(), scope.briefPath, scope.policyPath)}\n${TICK_SCOPE_CONSTRAINTS[scope.scope]}\n${bridged ? TICK_DELIVERY_RULE_BRIDGED : TICK_DELIVERY_RULE}`;
|
|
1209
1243
|
}
|
|
1210
1244
|
let frictionStore: Store | undefined;
|
|
1211
1245
|
let frictionSignals: FrictionSignal[] = [];
|
|
@@ -1275,18 +1309,6 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1275
1309
|
// `telegram_ask` and `telegram_send` dead until `/telegram on`. Trusting the
|
|
1276
1310
|
// file alone there puts the tick straight back to mandating a call its surface
|
|
1277
1311
|
// cannot make, which is #114 exactly.
|
|
1278
|
-
const approval =
|
|
1279
|
-
config.accessFile === undefined
|
|
1280
|
-
? undefined
|
|
1281
|
-
: session.bridgeTokenAtStart
|
|
1282
|
-
? readApprovalSurface(config.accessFile)
|
|
1283
|
-
: ({
|
|
1284
|
-
kind: "missing",
|
|
1285
|
-
reason:
|
|
1286
|
-
`${TELEGRAM_APPROVAL_TOOL} unavailable on local ticks: omp-telegram had no bot token when this ` +
|
|
1287
|
-
"session started, so it bound none and its tools stay dead however complete the access file looks " +
|
|
1288
|
-
"now — run `/telegram on` in this session, or restart it, to rebind the bridge",
|
|
1289
|
-
} as const);
|
|
1290
1312
|
if (approval?.kind === "missing") {
|
|
1291
1313
|
content = `${content}\n${TICK_APPROVAL_UNAVAILABLE_RULE}`;
|
|
1292
1314
|
if (!session.approvalToolMissingLogged) {
|
|
@@ -1318,10 +1340,6 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1318
1340
|
// live bridge whose profile is not `daemon`.
|
|
1319
1341
|
//
|
|
1320
1342
|
// No `accessFile` means no fleet bridge to judge, exactly as above.
|
|
1321
|
-
const profile =
|
|
1322
|
-
config.accessFile === undefined || !session.bridgeTokenAtStart
|
|
1323
|
-
? undefined
|
|
1324
|
-
: readDaemonProfile(config.accessFile);
|
|
1325
1343
|
if (profile?.kind === "interactive") content = `${content}\n${TICK_NARRATION_RULE}`;
|
|
1326
1344
|
|
|
1327
1345
|
try {
|
package/src/store.ts
CHANGED
|
@@ -15,7 +15,7 @@ import { dirname, join } from "node:path";
|
|
|
15
15
|
|
|
16
16
|
import { stateDir } from "./config.ts";
|
|
17
17
|
import { DECISION_TTL_MS, DEFAULT_CAPS } from "./types.ts";
|
|
18
|
-
import { neverStarted } from "./failure-class.ts";
|
|
18
|
+
import { dispatchInfra, neverStarted } from "./failure-class.ts";
|
|
19
19
|
import type {
|
|
20
20
|
DecisionDraft,
|
|
21
21
|
FailureClass,
|
|
@@ -61,7 +61,7 @@ export const LIVE_STATES: readonly RunState[] = ["claimed", "running"];
|
|
|
61
61
|
* worktrees removed, so they must not consume slots, while their live PRs must
|
|
62
62
|
* still block duplicate attempts.
|
|
63
63
|
*/
|
|
64
|
-
const ACTIVE_STATES: readonly RunState[] = [...LIVE_STATES, "pushed-pending", "pushed-green"];
|
|
64
|
+
export const ACTIVE_STATES: readonly RunState[] = [...LIVE_STATES, "pushed-pending", "pushed-green"];
|
|
65
65
|
|
|
66
66
|
const LIVE_PLACEHOLDERS = LIVE_STATES.map(() => "?").join(", ");
|
|
67
67
|
const ACTIVE_PLACEHOLDERS = ACTIVE_STATES.map(() => "?").join(", ");
|
|
@@ -726,6 +726,48 @@ export function openStore(dbPath: string): Store {
|
|
|
726
726
|
for (const row of misread) reclassify.run(row.id);
|
|
727
727
|
}
|
|
728
728
|
|
|
729
|
+
// The same repair, for the case the one above deliberately leaves alone: its
|
|
730
|
+
// comment names it exactly ("a turn-0 row carrying an unrecognised error (a
|
|
731
|
+
// failed mirror fetch, say) is untouched"). #177 gave that fault a name, but
|
|
732
|
+
// only for rows written after it existed. Every earlier one was logged
|
|
733
|
+
// `unknown`, whose recovery is `escalate` — so `runsNeedingClassification`
|
|
734
|
+
// never revisits it and `countFailures` counts it forever. With
|
|
735
|
+
// `maxAttemptsPerIssue` defaulting to 2, a single mirror ref-lock is half an
|
|
736
|
+
// issue's entire budget spent on a fault that was never in its code.
|
|
737
|
+
//
|
|
738
|
+
// Narrow the same way: exactly `unknown`, state `failed`, and the
|
|
739
|
+
// CLASSIFIER's own predicate rather than a second copy of the rule.
|
|
740
|
+
// Deliberately only `failureClass` — not `recoveryAction`/`recoveredAt`.
|
|
741
|
+
// Relieving the budget is the point; re-animating a months-old run into the
|
|
742
|
+
// requeue sweep is not, and clearing the recovery columns would do exactly
|
|
743
|
+
// that. Idempotent for free: these rows then read `dispatch-infra`, which the
|
|
744
|
+
// WHERE no longer matches.
|
|
745
|
+
const mirrorBroke = db
|
|
746
|
+
.query<
|
|
747
|
+
{ id: string; lastError: string | null; prUrl: string | null; headSha: string | null; salvageSha: string | null },
|
|
748
|
+
[]
|
|
749
|
+
>(
|
|
750
|
+
`SELECT id, lastError, prUrl, headSha, salvageSha FROM runs
|
|
751
|
+
WHERE failureClass = 'unknown' AND turns = 0 AND state = 'failed'`,
|
|
752
|
+
)
|
|
753
|
+
.all()
|
|
754
|
+
.filter(
|
|
755
|
+
(row) =>
|
|
756
|
+
dispatchInfra({
|
|
757
|
+
turns: 0,
|
|
758
|
+
...(row.lastError === null ? {} : { lastError: row.lastError }),
|
|
759
|
+
...(row.prUrl === null ? {} : { prUrl: row.prUrl }),
|
|
760
|
+
...(row.headSha === null ? {} : { headSha: row.headSha }),
|
|
761
|
+
...(row.salvageSha === null ? {} : { salvageSha: row.salvageSha }),
|
|
762
|
+
}) !== undefined,
|
|
763
|
+
);
|
|
764
|
+
if (mirrorBroke.length > 0) {
|
|
765
|
+
const reclassify = db.query<unknown, [string]>(
|
|
766
|
+
`UPDATE runs SET failureClass = 'dispatch-infra' WHERE id = ?`,
|
|
767
|
+
);
|
|
768
|
+
for (const row of mirrorBroke) reclassify.run(row.id);
|
|
769
|
+
}
|
|
770
|
+
|
|
729
771
|
const insertRun = db.query<unknown, SqlValue[]>(
|
|
730
772
|
`INSERT INTO runs (
|
|
731
773
|
id, project, issue, repo, branch, worktree, state, attempt, turns,
|
|
@@ -773,12 +815,19 @@ export function openStore(dbPath: string): Store {
|
|
|
773
815
|
const countFailures = db.query<{ n: number }, [string, number]>(
|
|
774
816
|
`SELECT COUNT(*) AS n FROM runs
|
|
775
817
|
WHERE project = ? AND issue = ? AND state = 'failed'
|
|
776
|
-
AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure'))`,
|
|
818
|
+
AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra'))`,
|
|
777
819
|
);
|
|
778
820
|
const countContinuations = db.query<{ n: number }, [string, number]>(
|
|
779
821
|
`SELECT COUNT(*) AS n FROM runs
|
|
780
822
|
WHERE project = ? AND issue = ? AND state IN ('killed', 'orphaned', 'blocked')
|
|
781
|
-
AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure'))`,
|
|
823
|
+
AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra'))`,
|
|
824
|
+
);
|
|
825
|
+
// How many times one issue reached a given class. Recovery uses it to bound a
|
|
826
|
+
// retry loop whose cause is persistent (e.g. a mirror that will not refresh):
|
|
827
|
+
// after `DISPATCH_INFRA_MAX_STRIKES` the daemon escalates instead of looping.
|
|
828
|
+
const countByClass = db.query<{ n: number }, [string, number, FailureClass]>(
|
|
829
|
+
`SELECT COUNT(*) AS n FROM runs
|
|
830
|
+
WHERE project = ? AND issue = ? AND failureClass = ?`,
|
|
782
831
|
);
|
|
783
832
|
// Newest first, and bounded: every row this returns costs `gh` calls to gather
|
|
784
833
|
// facts for, so a fleet with a long unclassified history classifies over
|
|
@@ -1142,6 +1191,10 @@ export function openStore(dbPath: string): Store {
|
|
|
1142
1191
|
return countContinuations.get(project, issue)?.n ?? 0;
|
|
1143
1192
|
},
|
|
1144
1193
|
|
|
1194
|
+
classCountFor(project: string, issue: number, cls: FailureClass): number {
|
|
1195
|
+
return countByClass.get(project, issue, cls)?.n ?? 0;
|
|
1196
|
+
},
|
|
1197
|
+
|
|
1145
1198
|
latestRun(project: string, issue: number): RunRecord | undefined {
|
|
1146
1199
|
const row = selectLatestRun.get(project, issue);
|
|
1147
1200
|
return row ? toRecord(row) : undefined;
|
package/src/tracker/github.ts
CHANGED
|
@@ -695,6 +695,22 @@ export function makeTracker(p: ProjectConfig, runGh: typeof gh = gh): Tracker {
|
|
|
695
695
|
}
|
|
696
696
|
},
|
|
697
697
|
|
|
698
|
+
async checkLog(detailsUrl: string): Promise<string | undefined> {
|
|
699
|
+
const args = runLogArgs(detailsUrl);
|
|
700
|
+
if (args === undefined) return undefined;
|
|
701
|
+
try {
|
|
702
|
+
const raw = await runGh(args);
|
|
703
|
+
const lines = raw.replaceAll(/\u001b\[[0-9;]*m/g, "").split("\n");
|
|
704
|
+
return lines.slice(-400).join("\n");
|
|
705
|
+
} catch {
|
|
706
|
+
// Best-effort, never throws: a revoked token, a deleted run or a flaky
|
|
707
|
+
// network must not error the classifier out of gathering facts. The
|
|
708
|
+
// caller treats `undefined` as "could not read the log" and classifies
|
|
709
|
+
// conservatively (#177).
|
|
710
|
+
return undefined;
|
|
711
|
+
}
|
|
712
|
+
},
|
|
713
|
+
|
|
698
714
|
async mergeable(prUrl: string): Promise<"conflicting" | "clean" | "unknown"> {
|
|
699
715
|
if (!PR_URL.test(prUrl)) return "unknown";
|
|
700
716
|
try {
|
package/src/types.ts
CHANGED
|
@@ -766,6 +766,12 @@ export interface Tracker {
|
|
|
766
766
|
* port: nothing downstream may read it as "no checks failed".
|
|
767
767
|
*/
|
|
768
768
|
checkConclusions(prUrl: string): Promise<{ name: string; state: string; link?: string }[]>;
|
|
769
|
+
/** The tail of a failed workflow run's log, ANSI stripped, or `undefined`
|
|
770
|
+
* when it cannot be fetched. Best-effort, never throws: the classifier uses
|
|
771
|
+
* it to tell an infrastructure outage from a deterministic test failure
|
|
772
|
+
* (#177), and "could not read the log" must fall through to the existing
|
|
773
|
+
* conservative classification rather than error out of fact-gathering. */
|
|
774
|
+
checkLog(detailsUrl: string): Promise<string | undefined>;
|
|
769
775
|
/** Whether the PR can merge into its base. `unknown` on any doubt, so a
|
|
770
776
|
* mergeability nobody could read never becomes a conflict recovery. */
|
|
771
777
|
mergeable(prUrl: string): Promise<"conflicting" | "clean" | "unknown">;
|
|
@@ -803,6 +809,7 @@ export const FAILURE_CLASSES = [
|
|
|
803
809
|
"admin-kill",
|
|
804
810
|
"ci-infra",
|
|
805
811
|
"ci-deterministic",
|
|
812
|
+
"dispatch-infra",
|
|
806
813
|
"merge-conflict",
|
|
807
814
|
"question",
|
|
808
815
|
"orphan-clean",
|
|
@@ -1155,6 +1162,10 @@ export interface Store {
|
|
|
1155
1162
|
failuresFor(project: string, issue: number): number;
|
|
1156
1163
|
/** Operational stops that require a bounded continuation resume. */
|
|
1157
1164
|
continuationsFor(project: string, issue: number): number;
|
|
1165
|
+
/** How many times one issue has reached a given failure class. A recovery
|
|
1166
|
+
* that keeps landing on the same class (e.g. a persistently broken mirror)
|
|
1167
|
+
* uses this to escalate instead of retrying forever. */
|
|
1168
|
+
classCountFor(project: string, issue: number, cls: FailureClass): number;
|
|
1158
1169
|
/** Newest attempt for one issue, whatever state it reached. `omp-conductor
|
|
1159
1170
|
* tail` resolves an issue number to a transcript through this; the number is
|
|
1160
1171
|
* what an operator has, the run id is not. */
|
|
@@ -1321,6 +1332,7 @@ export const VERB_NAMES = [
|
|
|
1321
1332
|
"conductor_push",
|
|
1322
1333
|
"conductor_pr_create",
|
|
1323
1334
|
"conductor_pr_update_branch",
|
|
1335
|
+
"conductor_pr_update",
|
|
1324
1336
|
"conductor_pr_merge",
|
|
1325
1337
|
"conductor_label",
|
|
1326
1338
|
"conductor_release",
|
package/src/unblock.ts
CHANGED
|
@@ -40,6 +40,10 @@ export interface UnblockOutcome {
|
|
|
40
40
|
continuationsUsed: number;
|
|
41
41
|
/** Newest attempt, when the store has one for this issue at all. */
|
|
42
42
|
latest?: RunRecord;
|
|
43
|
+
/** Whether any run for the issue is still in an active state — the answer
|
|
44
|
+
* to "will the dispatcher hold this issue as issue-active?" that #178
|
|
45
|
+
* found this verb guessing at. */
|
|
46
|
+
active: boolean;
|
|
43
47
|
/** Set when nothing was cleared because the newest attempt's work exists
|
|
44
48
|
* only in its worktree. Carries the salvage failure verbatim. */
|
|
45
49
|
refused?: string;
|
|
@@ -92,10 +96,12 @@ export async function unblockIssue(
|
|
|
92
96
|
// for clearing in-progress, so the row that carries it decides the set.
|
|
93
97
|
const latest = store.latestRun(project.name, issue);
|
|
94
98
|
const terminal = latest !== undefined && !LIVE_STATES.includes(latest.state);
|
|
99
|
+
const active = store.activeRuns(project.name).some((r) => r.issue === issue);
|
|
95
100
|
const counts = {
|
|
96
101
|
attemptsUsed: store.attemptsFor(project.name, issue),
|
|
97
102
|
failuresUsed: store.failuresFor(project.name, issue),
|
|
98
103
|
continuationsUsed: store.continuationsFor(project.name, issue),
|
|
104
|
+
active,
|
|
99
105
|
};
|
|
100
106
|
|
|
101
107
|
// The one case where this verb refuses. Clearing the labels here re-queues an
|
|
@@ -205,8 +211,31 @@ export function formatUnblock(
|
|
|
205
211
|
`run row to prove the worker is gone, "${project.stateLabels.inProgress}" was left in place, and ` +
|
|
206
212
|
"on its own it keeps the issue ineligible",
|
|
207
213
|
);
|
|
214
|
+
} else if (latest.state === "pushed-green" || latest.state === "pushed-pending") {
|
|
215
|
+
// #178: an unblocked issue whose newest run pushed a green PR is not "free
|
|
216
|
+
// for a fresh attempt" — the dispatcher continues the pushed run's branch,
|
|
217
|
+
// and only once the PR resolves does the issue settle (#175). Say that
|
|
218
|
+
// instead of promising a re-claim the next tick withholds.
|
|
219
|
+
lines.push(
|
|
220
|
+
latest.prUrl === undefined
|
|
221
|
+
? ` next tick eligible again, as long as the issue still carries "${project.queueLabel}" ` +
|
|
222
|
+
`(newest run is ${latest.state} with no recorded PR; the dispatcher still applies its open-PR check at claim time)`
|
|
223
|
+
: ` next tick eligible as a continuation of ${latest.prUrl} — the pushed run stays active until ` +
|
|
224
|
+
"that PR resolves; dispatch continues on its branch",
|
|
225
|
+
);
|
|
226
|
+
} else if (o.active) {
|
|
227
|
+
// A run other than the newest is still active (e.g. a live sibling under a
|
|
228
|
+
// terminal newest row), so the dispatcher will hold the issue as
|
|
229
|
+
// issue-active whatever the labels now say. #178's misleading case.
|
|
230
|
+
lines.push(
|
|
231
|
+
` in flight a run for this issue is still active, so the dispatcher holds the issue until it settles — ` +
|
|
232
|
+
`nothing is re-claimed before then ("${project.stateLabels.inProgress}" stays unless already released)`,
|
|
233
|
+
);
|
|
208
234
|
} else {
|
|
209
|
-
lines.push(
|
|
235
|
+
lines.push(
|
|
236
|
+
` next tick eligible again, as long as the issue still carries "${project.queueLabel}" ` +
|
|
237
|
+
"(the dispatcher still applies its open-PR check at claim time)",
|
|
238
|
+
);
|
|
210
239
|
}
|
|
211
240
|
|
|
212
241
|
return lines.join("\n");
|
package/src/verbs/actions.ts
CHANGED
|
@@ -97,6 +97,14 @@ export function githubVerbActions(project: ProjectConfig, run: CommandRunner = s
|
|
|
97
97
|
|
|
98
98
|
updatePrBranch: (prUrl) => gh(["pr", "update-branch", prUrl]),
|
|
99
99
|
|
|
100
|
+
updatePr: async (prUrl, fields) => {
|
|
101
|
+
const argv = ["pr", "edit", prUrl];
|
|
102
|
+
if (fields.title !== undefined) argv.push("--title", fields.title);
|
|
103
|
+
if (fields.body !== undefined) argv.push("--body", fields.body);
|
|
104
|
+
const outcome = await gh(argv);
|
|
105
|
+
return outcome.ok ? { ok: true } : { ok: false, stderr: outcome.stderr };
|
|
106
|
+
},
|
|
107
|
+
|
|
100
108
|
// `--match-head-commit` is the server-side half of the exact-head rule: the
|
|
101
109
|
// daemon already re-read the head and refused a stale one, and this makes
|
|
102
110
|
// GitHub refuse too if the branch moved in the milliseconds between. Belt
|