omp-conductor 0.2.2 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +212 -35
- package/package.json +1 -1
- package/skills/conductor-onboarding/SKILL.md +33 -22
- package/src/briefs/orchestrator.md +46 -14
- package/src/briefs/worker.md +9 -2
- package/src/cli.ts +291 -13
- package/src/config.ts +127 -16
- package/src/daemon.ts +474 -50
- package/src/lifecycle.ts +6 -4
- package/src/orchestrator-tick.ts +209 -45
- package/src/plugin.ts +60 -0
- package/src/setup.ts +98 -4
- package/src/store.ts +20 -1
- package/src/tracker/github.ts +94 -2
- package/src/types.ts +56 -3
- package/src/unblock.ts +113 -0
- package/src/worker.ts +14 -0
- package/src/worktree.ts +119 -0
package/src/daemon.ts
CHANGED
|
@@ -7,11 +7,13 @@
|
|
|
7
7
|
* of it, so concurrency, daily volume, spend and per-issue attempts are counted
|
|
8
8
|
* here and enforced before anything is claimed.
|
|
9
9
|
*/
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
12
|
+
import { dirname, join, relative } from "node:path";
|
|
12
13
|
import { configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
|
|
13
14
|
import { createEscalator } from "./escalate.ts";
|
|
14
15
|
import { livingDaemon } from "./lifecycle.ts";
|
|
16
|
+
import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
15
17
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
16
18
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
17
19
|
import { branchName, route } from "./routing.ts";
|
|
@@ -28,8 +30,15 @@ import type {
|
|
|
28
30
|
Store,
|
|
29
31
|
Tracker,
|
|
30
32
|
} from "./types.ts";
|
|
31
|
-
import { renderBrief, runWorker } from "./worker.ts";
|
|
32
|
-
import {
|
|
33
|
+
import { type KilledBy, renderBrief, runWorker } from "./worker.ts";
|
|
34
|
+
import {
|
|
35
|
+
addWorktree,
|
|
36
|
+
mirrorPathFor,
|
|
37
|
+
removeWorktree,
|
|
38
|
+
salvageWip,
|
|
39
|
+
type SalvageOutcome,
|
|
40
|
+
worktreePathFor,
|
|
41
|
+
} from "./worktree.ts";
|
|
33
42
|
|
|
34
43
|
/** Long enough that the tracker is not polled raw, short enough that a human
|
|
35
44
|
* who labels an issue sees it picked up within a coffee break. */
|
|
@@ -61,6 +70,8 @@ interface Deps {
|
|
|
61
70
|
tracker: Tracker;
|
|
62
71
|
store: Store;
|
|
63
72
|
escalate(e: Escalation): Promise<void>;
|
|
73
|
+
integrity: IntegrityGate;
|
|
74
|
+
stall: StallGate;
|
|
64
75
|
}
|
|
65
76
|
|
|
66
77
|
// ---------------------------------------------------------------- paths & pause
|
|
@@ -70,6 +81,100 @@ export function dbPath(): string {
|
|
|
70
81
|
return join(stateDir(), "conductor.db");
|
|
71
82
|
}
|
|
72
83
|
|
|
84
|
+
// ------------------------------------------------------- orchestrator liveness
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Whether the wedged-orchestrator page has already gone out for the stall
|
|
88
|
+
* currently on disk. One page per episode: the marker persists until a tick is
|
|
89
|
+
* consumed, so paging per five minutes would be paging forever.
|
|
90
|
+
*/
|
|
91
|
+
export interface StallGate {
|
|
92
|
+
paged: boolean;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export interface StallVerdict {
|
|
96
|
+
/** The marker's own line, when one is there. */
|
|
97
|
+
since?: string;
|
|
98
|
+
page: boolean;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Reads the orchestrator's stall marker and decides whether this tick pages.
|
|
103
|
+
*
|
|
104
|
+
* The marker is written by the tick extension inside the orchestrator session
|
|
105
|
+
* ({@link STALL_MARKER_FILE}) when two of its own prompts go unconsumed — the
|
|
106
|
+
* one signal that separates "the process is alive" from "the loop is reading
|
|
107
|
+
* its queue". Every other guard in this system reads healthy through a wedge:
|
|
108
|
+
* the herdr recovery plugin tests for a live process and an agent label, both
|
|
109
|
+
* of which survive it, and `/healthz` describes this daemon, which is a
|
|
110
|
+
* different process entirely.
|
|
111
|
+
*
|
|
112
|
+
* The daemon is the natural watcher precisely because it is that different
|
|
113
|
+
* process: it already wakes every five minutes, it owns a working escalation
|
|
114
|
+
* path, and nothing about its health depends on the session that is stuck. A
|
|
115
|
+
* wedged loop cannot page for itself, and the herdr plugin only runs on session
|
|
116
|
+
* lifecycle events — a session that stays alive and stops working emits none.
|
|
117
|
+
*
|
|
118
|
+
* Resets when the marker disappears, so a second stall days later pages again.
|
|
119
|
+
*/
|
|
120
|
+
export function checkStall(gate: StallGate, marker: string): StallVerdict {
|
|
121
|
+
if (!existsSync(marker)) {
|
|
122
|
+
gate.paged = false;
|
|
123
|
+
return { page: false };
|
|
124
|
+
}
|
|
125
|
+
const page = !gate.paged;
|
|
126
|
+
let since: string | undefined;
|
|
127
|
+
try {
|
|
128
|
+
const body = readFileSync(marker, "utf8").split("\n")[0]?.trim();
|
|
129
|
+
if (body !== undefined && body !== "") since = body;
|
|
130
|
+
} catch {
|
|
131
|
+
// An unreadable marker still means stalled; the timestamp is a nicety.
|
|
132
|
+
}
|
|
133
|
+
return { ...(since === undefined ? {} : { since }), page };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Pages tier 2 once when the orchestrator session stops draining its queue.
|
|
138
|
+
*
|
|
139
|
+
* Deliberately does not restart anything. A wedge lands mid-turn, this process
|
|
140
|
+
* cannot tell a half-applied edit from an idle loop, and killing the session
|
|
141
|
+
* could destroy work an operator would rather read first — the same refusal to
|
|
142
|
+
* guess that the recovery plugin is built on.
|
|
143
|
+
*/
|
|
144
|
+
async function watchOrchestrator(d: Deps): Promise<void> {
|
|
145
|
+
const marker = join(stateDir(), STALL_MARKER_FILE);
|
|
146
|
+
const verdict = checkStall(d.stall, marker);
|
|
147
|
+
if (!verdict.page) return;
|
|
148
|
+
|
|
149
|
+
log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
|
|
150
|
+
const delivered = await safeEscalate(d, {
|
|
151
|
+
tier: 2,
|
|
152
|
+
project: d.project.name,
|
|
153
|
+
issue: NO_ISSUE,
|
|
154
|
+
// Keyed on the marker's own timestamp, not the date. The dedup ledger keys
|
|
155
|
+
// on this summary, and two wedges in one day is not a hypothetical — the
|
|
156
|
+
// failure mode is a session that gets stuck, gets restarted, and gets stuck
|
|
157
|
+
// again on the same cause an hour later. A day-keyed summary would report
|
|
158
|
+
// the first and silently swallow every one after it.
|
|
159
|
+
summary:
|
|
160
|
+
`Orchestrator session wedged (${verdict.since ?? `marker at ${marker}`}) — ` +
|
|
161
|
+
`it has stopped reading its queue (${d.project.name})`,
|
|
162
|
+
detail: [
|
|
163
|
+
verdict.since ?? "Marker present with no readable timestamp.",
|
|
164
|
+
`Marker: ${marker}`,
|
|
165
|
+
"",
|
|
166
|
+
"Its process and its herdr agent label are both healthy, which is why nothing else noticed:",
|
|
167
|
+
"the loop is alive and consuming nothing, so ticks and your messages queue behind it unread.",
|
|
168
|
+
"",
|
|
169
|
+
"Attach and look before you act — a wedge lands mid-turn. Then SIGTERM the omp process:",
|
|
170
|
+
"herdr-conductor resumes it by exact identity, and the first consumed tick clears this marker.",
|
|
171
|
+
"",
|
|
172
|
+
"Dispatch is unaffected: workers keep running. What stops is drain, groom, report and merge.",
|
|
173
|
+
].join("\n"),
|
|
174
|
+
});
|
|
175
|
+
markPaged(d.stall, delivered);
|
|
176
|
+
}
|
|
177
|
+
|
|
73
178
|
/**
|
|
74
179
|
* Pause is a file rather than process state on purpose: `omp-conductor pause`
|
|
75
180
|
* and `/conductor pause` run in a different process from the daemon, and a flag
|
|
@@ -90,6 +195,99 @@ export function setPaused(v: boolean): void {
|
|
|
90
195
|
}
|
|
91
196
|
}
|
|
92
197
|
|
|
198
|
+
// ----------------------------------------------------------- package integrity
|
|
199
|
+
|
|
200
|
+
/** Enough differing paths to tell a deploy from a tamper at a glance; the full
|
|
201
|
+
* list is on the host, and the answer is always "go look at the host". */
|
|
202
|
+
const INTEGRITY_SAMPLE = 5;
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* What the daemon booted with, and whether it has already paged about losing
|
|
206
|
+
* it. Lives exactly as long as one `runDaemon()` call — which is the whole
|
|
207
|
+
* trick: a restart re-records both.
|
|
208
|
+
*/
|
|
209
|
+
export interface IntegrityGate {
|
|
210
|
+
baseline: Map<string, string>;
|
|
211
|
+
paged: boolean;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
export interface IntegrityVerdict {
|
|
215
|
+
/** Labelled, sorted differences; empty when the package is untouched. */
|
|
216
|
+
diff: string[];
|
|
217
|
+
/** Any difference at all stops the fleet. */
|
|
218
|
+
pause: boolean;
|
|
219
|
+
/** First divergent tick only — a page every five minutes is a page nobody reads. */
|
|
220
|
+
page: boolean;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* sha256 of every source file the running package is made of, keyed by path
|
|
225
|
+
* relative to `root`.
|
|
226
|
+
*
|
|
227
|
+
* `import.meta.dir` is the installed `src/` of the code executing right now, so
|
|
228
|
+
* this is a self-portrait: what was actually deployed, not what some checkout
|
|
229
|
+
* on disk happens to contain. `.ts` and `.md` because both are executable in
|
|
230
|
+
* this package — the briefs under `src/briefs/` are the sessions' instructions,
|
|
231
|
+
* and rewriting one of those buys more than rewriting the dispatcher does.
|
|
232
|
+
* (A checkout also carries `*.test.ts`, which the published package excludes, so
|
|
233
|
+
* a daemon started from one is watching its tests too. That is the honest
|
|
234
|
+
* answer — its code did change — and it costs nothing on a real install.)
|
|
235
|
+
*
|
|
236
|
+
* Walking and hashing the ~30 files of this package measures 0.6 ms warm, once
|
|
237
|
+
* per five-minute tick, so a tick does it inline. No cache and no mtime
|
|
238
|
+
* shortcut on purpose: a cache is a second thing that can be wrong, and mtime
|
|
239
|
+
* is the first field anyone covering their tracks restores.
|
|
240
|
+
*/
|
|
241
|
+
export function packageManifest(root: string = import.meta.dir): Map<string, string> {
|
|
242
|
+
const out = new Map<string, string>();
|
|
243
|
+
const walk = (dir: string): void => {
|
|
244
|
+
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
|
245
|
+
const full = join(dir, e.name);
|
|
246
|
+
if (e.isDirectory()) walk(full);
|
|
247
|
+
else if (e.isFile() && (e.name.endsWith(".ts") || e.name.endsWith(".md")))
|
|
248
|
+
out.set(relative(root, full), createHash("sha256").update(readFileSync(full)).digest("hex"));
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
walk(root);
|
|
252
|
+
return out;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Labelled rather than three arrays because every consumer — the log line, the
|
|
257
|
+
* page, the test — wants one readable list of what moved.
|
|
258
|
+
*/
|
|
259
|
+
export function manifestDiff(before: Map<string, string>, after: Map<string, string>): string[] {
|
|
260
|
+
const out: string[] = [];
|
|
261
|
+
for (const [path, hash] of before) {
|
|
262
|
+
const now = after.get(path);
|
|
263
|
+
if (now === undefined) out.push(`removed ${path}`);
|
|
264
|
+
else if (now !== hash) out.push(`changed ${path}`);
|
|
265
|
+
}
|
|
266
|
+
for (const path of after.keys()) if (!before.has(path)) out.push(`added ${path}`);
|
|
267
|
+
return out.sort();
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* The tick's decision, split from its effects so the once-only page is a thing
|
|
272
|
+
* a test can hold.
|
|
273
|
+
*
|
|
274
|
+
* `pause` stays true on every divergent tick, deliberately: an operator who
|
|
275
|
+
* resumes without restarting gets re-paused, because the boundary is still
|
|
276
|
+
* broken. `page` asks whether this tick should *try* — the caller latches the
|
|
277
|
+
* gate with {@link markPaged} only once a page actually went out, so a Telegram
|
|
278
|
+
* outage during the one tick that noticed does not buy permanent silence.
|
|
279
|
+
*/
|
|
280
|
+
export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>): IntegrityVerdict {
|
|
281
|
+
const diff = manifestDiff(gate.baseline, current);
|
|
282
|
+
if (diff.length === 0) return { diff, pause: false, page: false };
|
|
283
|
+
return { diff, pause: true, page: !gate.paged };
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** Latch a once-only page, after delivery is confirmed and never before. */
|
|
287
|
+
export function markPaged(gate: { paged: boolean }, delivered: boolean): void {
|
|
288
|
+
if (delivered) gate.paged = true;
|
|
289
|
+
}
|
|
290
|
+
|
|
93
291
|
// ---------------------------------------------------------------------- helpers
|
|
94
292
|
|
|
95
293
|
function log(msg: string): void {
|
|
@@ -158,13 +356,76 @@ async function swapLabel(tracker: Tracker, issue: number, from: string, to: stri
|
|
|
158
356
|
* The escalator throws when no transport is configured or Telegram rejects, and
|
|
159
357
|
* only records the dedup marker on success. A page that cannot be delivered
|
|
160
358
|
* must not take the tick down with it — log it and let the next tick retry.
|
|
359
|
+
*
|
|
360
|
+
* Returns whether it actually went out, because "page once" and "page once
|
|
361
|
+
* *successfully*" are different promises: a caller that latches a once-only
|
|
362
|
+
* gate on the attempt turns one failed delivery into permanent silence about a
|
|
363
|
+
* condition that is still true.
|
|
161
364
|
*/
|
|
162
|
-
async function safeEscalate(d: Deps, e: Escalation): Promise<
|
|
365
|
+
async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<boolean> {
|
|
163
366
|
try {
|
|
164
367
|
await d.escalate(e);
|
|
368
|
+
return true;
|
|
165
369
|
} catch (err) {
|
|
166
370
|
log(`escalation for #${e.issue} could not be delivered: ${errText(err)}`);
|
|
371
|
+
return false;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* What a salvage attempt contributes to the escalation: where the work went, or
|
|
377
|
+
* that it went nowhere. Split from the effects below for the same reason
|
|
378
|
+
* `checkIntegrity` is — this wording is the whole thing a human acts on, so it
|
|
379
|
+
* is worth a test holding it, and the sha in it is the only pointer to work
|
|
380
|
+
* that no longer has any other copy.
|
|
381
|
+
*/
|
|
382
|
+
export function salvageLines(outcome: SalvageOutcome, worktree: string): string[] {
|
|
383
|
+
const kept = `Worktree kept for inspection: ${worktree}`;
|
|
384
|
+
|
|
385
|
+
if (outcome.kind === "nothing") return [`${kept} — nothing uncommitted to salvage`];
|
|
386
|
+
|
|
387
|
+
if (outcome.kind === "failed") {
|
|
388
|
+
return [
|
|
389
|
+
`WIP SALVAGE FAILED: ${outcome.error}`,
|
|
390
|
+
`Uncommitted work in ${worktree} is the only copy of it, and the next attempt removes that tree.`,
|
|
391
|
+
];
|
|
167
392
|
}
|
|
393
|
+
|
|
394
|
+
return [
|
|
395
|
+
`WIP committed to ${outcome.branch} @ ${outcome.sha}` +
|
|
396
|
+
(outcome.pushed
|
|
397
|
+
? " and pushed — the work outlives this worktree"
|
|
398
|
+
: ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`),
|
|
399
|
+
kept,
|
|
400
|
+
];
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* Commits and pushes whatever a dead run left uncommitted, logs the outcome and
|
|
405
|
+
* returns the escalation lines that say where that work now lives.
|
|
406
|
+
*
|
|
407
|
+
* Only ever called on a non-graceful end — a cap kill, a crashed session, a
|
|
408
|
+
* dispatch error. A `blocked` run stopped on purpose, with turns still in hand
|
|
409
|
+
* and a brief that tells it to report rather than push, so nothing is committed
|
|
410
|
+
* behind its back. The rest never got the chance: the kill is external and
|
|
411
|
+
* lands mid-edit, in the tree the next attempt removes `--force`.
|
|
412
|
+
*/
|
|
413
|
+
async function salvage(
|
|
414
|
+
issue: number,
|
|
415
|
+
attempt: number,
|
|
416
|
+
reason: string,
|
|
417
|
+
worktree: string,
|
|
418
|
+
): Promise<string[]> {
|
|
419
|
+
const lines = salvageLines(await salvageWip(worktree, issue, attempt, reason), worktree);
|
|
420
|
+
log(`#${issue} salvage: ${lines.join(" ")}`);
|
|
421
|
+
return lines;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
/** How a run's end is named — in the salvage commit, and to whoever reads it. */
|
|
425
|
+
function endedBy(killedBy: KilledBy | undefined): string {
|
|
426
|
+
if (killedBy === "turns") return "the turns cap";
|
|
427
|
+
if (killedBy === "wallclock") return "the wall-clock cap";
|
|
428
|
+
return "a failed run";
|
|
168
429
|
}
|
|
169
430
|
|
|
170
431
|
async function buildBrief(
|
|
@@ -202,6 +463,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
202
463
|
|
|
203
464
|
let claimed = false;
|
|
204
465
|
let run: RunRecord | undefined;
|
|
466
|
+
// Hoisted out of the try so the catch path can still name the tree: a crash
|
|
467
|
+
// mid-dispatch is one of the non-graceful ends whose uncommitted work has to
|
|
468
|
+
// be salvaged too, and it is the path least likely to have committed first.
|
|
469
|
+
let worktreePath: string | undefined;
|
|
205
470
|
|
|
206
471
|
try {
|
|
207
472
|
// Claim on the tracker FIRST, before any local work. The label — not the
|
|
@@ -234,7 +499,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
234
499
|
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
235
500
|
await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
|
|
236
501
|
|
|
237
|
-
|
|
502
|
+
worktreePath = await addWorktree(
|
|
238
503
|
r.repo,
|
|
239
504
|
project.mirrorRoot,
|
|
240
505
|
project.workspaceRoot,
|
|
@@ -258,6 +523,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
258
523
|
sessionDir,
|
|
259
524
|
...(project.workerModel === undefined ? {} : { model: project.workerModel }),
|
|
260
525
|
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
526
|
+
// Recorded the moment the session opens its transcript, not when the run
|
|
527
|
+
// ends: `omp-conductor tail` resolves an issue to a file through this row,
|
|
528
|
+
// and a path written at completion is a path nobody can follow live. The
|
|
529
|
+
// completion-time update below writes the same value again, harmlessly.
|
|
530
|
+
onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
|
|
261
531
|
});
|
|
262
532
|
|
|
263
533
|
// A configured model the harness could not honour means this run was done by
|
|
@@ -288,6 +558,9 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
288
558
|
});
|
|
289
559
|
} else if (result.state === "failed" || result.state === "killed") {
|
|
290
560
|
await swapLabel(tracker, issue, inProgress, project.stateLabels.failed);
|
|
561
|
+
// Before the escalation is composed, so it can say where the work went —
|
|
562
|
+
// and long before the next attempt provisions over this tree.
|
|
563
|
+
const salvaged = await salvage(issue, attempt, endedBy(result.killedBy), worktreePath);
|
|
291
564
|
await safeEscalate(d, {
|
|
292
565
|
tier: 1,
|
|
293
566
|
project: project.name,
|
|
@@ -302,7 +575,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
302
575
|
detail: [
|
|
303
576
|
`${r.issue.title}`,
|
|
304
577
|
r.issue.url,
|
|
305
|
-
|
|
578
|
+
...salvaged,
|
|
306
579
|
`Session: ${result.sessionFile ?? "(no transcript)"}`,
|
|
307
580
|
"",
|
|
308
581
|
result.report,
|
|
@@ -336,28 +609,175 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
336
609
|
log(`#${issue} could not be relabelled: ${errText(relabelErr)}`);
|
|
337
610
|
}
|
|
338
611
|
}
|
|
612
|
+
// A crash lands anywhere, including mid-edit in a tree holding the only
|
|
613
|
+
// copy of real work. Nothing else on this path so much as looks at it.
|
|
614
|
+
const salvaged =
|
|
615
|
+
worktreePath === undefined
|
|
616
|
+
? []
|
|
617
|
+
: await salvage(issue, attempt, "a dispatch error", worktreePath);
|
|
618
|
+
|
|
339
619
|
await safeEscalate(d, {
|
|
340
620
|
tier: 1,
|
|
341
621
|
project: project.name,
|
|
342
622
|
issue,
|
|
343
623
|
runId: run?.id,
|
|
344
624
|
summary: `#${issue} could not be dispatched on attempt ${attempt}`,
|
|
345
|
-
detail,
|
|
625
|
+
detail: salvaged.length === 0 ? detail : [detail, "", ...salvaged].join("\n"),
|
|
346
626
|
});
|
|
347
627
|
// The worktree, if one was created, is deliberately left in place: this is
|
|
348
|
-
// a failure path.
|
|
628
|
+
// a failure path, and whatever it still held is now a commit on the branch.
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
// -------------------------------------------------------------------- admission
|
|
633
|
+
|
|
634
|
+
/** A candidate cleared for dispatch, with the attempt number it will run as. */
|
|
635
|
+
export interface Admission {
|
|
636
|
+
r: Routed;
|
|
637
|
+
attempt: number;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* Which routed candidates get a worker this tick — in queue order, never more
|
|
642
|
+
* than `slots` of them.
|
|
643
|
+
*
|
|
644
|
+
* Exported so the admission rules can be pinned without spawning a worker.
|
|
645
|
+
* Every one of them exists because of a live incident, and each guards a
|
|
646
|
+
* different way the same issue gets worked twice.
|
|
647
|
+
*
|
|
648
|
+
* Takes the slice of `Deps` it actually reads rather than the whole thing: what
|
|
649
|
+
* admission is allowed to consult is the point of the function, and a `Deps`
|
|
650
|
+
* that grows a field has no business breaking these tests.
|
|
651
|
+
*/
|
|
652
|
+
export async function admitCandidates(
|
|
653
|
+
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate">,
|
|
654
|
+
routed: Routed[],
|
|
655
|
+
slots: number,
|
|
656
|
+
): Promise<Admission[]> {
|
|
657
|
+
const { project, caps, tracker, store } = d;
|
|
658
|
+
const busy = new Set(store.activeRuns(project.name).map((r) => r.issue));
|
|
659
|
+
|
|
660
|
+
const admitted: Admission[] = [];
|
|
661
|
+
for (const r of routed) {
|
|
662
|
+
if (admitted.length >= slots) break;
|
|
663
|
+
if (busy.has(r.issue.number)) continue;
|
|
664
|
+
|
|
665
|
+
const prior = store.attemptsFor(project.name, r.issue.number);
|
|
666
|
+
if (prior >= caps.maxAttemptsPerIssue) {
|
|
667
|
+
await safeEscalate(d, {
|
|
668
|
+
tier: 1,
|
|
669
|
+
project: project.name,
|
|
670
|
+
issue: r.issue.number,
|
|
671
|
+
summary: `#${r.issue.number} has used all ${caps.maxAttemptsPerIssue} attempts`,
|
|
672
|
+
detail: [
|
|
673
|
+
r.issue.title,
|
|
674
|
+
r.issue.url,
|
|
675
|
+
"Another attempt almost always means the issue itself is underspecified.",
|
|
676
|
+
"Rewrite the acceptance criteria, or take it off the queue.",
|
|
677
|
+
].join("\n"),
|
|
678
|
+
});
|
|
679
|
+
continue;
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// The busy set is built from run rows, so it can only speak for work this
|
|
683
|
+
// database recorded. Work pushed before this store existed — a migration, a
|
|
684
|
+
// wiped or relocated state dir, a restore onto a new host — looks exactly
|
|
685
|
+
// like fresh work, and a worker sent at it re-implements a finished PR. The
|
|
686
|
+
// tracker is the only party that remembers, so it is asked. The cost is
|
|
687
|
+
// bounded by free slots, not by queue depth: the call sits behind the two
|
|
688
|
+
// cheap local filters and the loop stops once the slots are full.
|
|
689
|
+
let closer: string | undefined;
|
|
690
|
+
try {
|
|
691
|
+
closer = await tracker.openCloserFor(r.issue.number);
|
|
692
|
+
} catch (err) {
|
|
693
|
+
// Fail closed, per candidate. An API error means "unknown whether
|
|
694
|
+
// finished work exists", and admitting on unknown recreates precisely the
|
|
695
|
+
// duplicate-work failure this guard exists to kill: the worst case of
|
|
696
|
+
// holding is a five-minute delay, the worst case of admitting is a burned
|
|
697
|
+
// attempt and a second PR on the same issue. Holding one candidate rather
|
|
698
|
+
// than aborting the loop is what keeps a transient GitHub failure from
|
|
699
|
+
// deadlocking the whole dispatcher; the next tick retries by itself.
|
|
700
|
+
log(`#${r.issue.number} held: open-PR check failed (${errText(err)}) — retrying next tick`);
|
|
701
|
+
continue;
|
|
702
|
+
}
|
|
703
|
+
if (closer !== undefined) {
|
|
704
|
+
log(`#${r.issue.number} skipped: open PR ${closer} already closes it`);
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
admitted.push({ r, attempt: prior + 1 });
|
|
349
709
|
}
|
|
710
|
+
|
|
711
|
+
return admitted;
|
|
350
712
|
}
|
|
351
713
|
|
|
352
714
|
// ----------------------------------------------------------------------- a tick
|
|
353
715
|
|
|
354
716
|
async function tick(d: Deps): Promise<void> {
|
|
717
|
+
// Before the pause check, deliberately. This one is not about dispatch: the
|
|
718
|
+
// orchestrator is a different process, and it can be wedged while this fleet
|
|
719
|
+
// is paused — which is exactly the state the reference fleet was in when the
|
|
720
|
+
// failure happened. A pause silences claiming, not the operator's right to
|
|
721
|
+
// know their supervising session stopped reading its queue.
|
|
722
|
+
await watchOrchestrator(d);
|
|
723
|
+
|
|
355
724
|
// A paused fleet claims nothing. Checked first so pausing takes effect on the
|
|
356
725
|
// next tick without signalling the process.
|
|
357
726
|
if (isPaused()) return;
|
|
358
727
|
|
|
359
728
|
const { project, caps, store } = d;
|
|
360
729
|
|
|
730
|
+
// "Nobody patches the running conductor" is a hard boundary in both briefs —
|
|
731
|
+
// which makes it prompt text, and prompt text is a request. This is the half
|
|
732
|
+
// that does not negotiate: the package that dispatched the last worker must
|
|
733
|
+
// still be the package on disk, or nothing else this tick does is
|
|
734
|
+
// attributable. A legitimate deploy never trips it, because installing a new
|
|
735
|
+
// build and restarting the unit re-records the baseline from the new files;
|
|
736
|
+
// only an edit *underneath* a live daemon diverges from it.
|
|
737
|
+
//
|
|
738
|
+
// Below the pause gate on purpose, unlike the stall watch above. The property
|
|
739
|
+
// being defended is that no work is dispatched under a package the operator
|
|
740
|
+
// did not install — and a paused fleet dispatches nothing, so nothing needs
|
|
741
|
+
// attributing yet. Tampering during a pause is not missed, only deferred: the
|
|
742
|
+
// baseline is boot's, so the first tick after `resume` compares against it and
|
|
743
|
+
// pauses again before claiming anything. Checking above the gate instead would
|
|
744
|
+
// page on every legitimate build an operator deploys into a parked fleet,
|
|
745
|
+
// which is exactly when they deploy them.
|
|
746
|
+
const integrity = checkIntegrity(d.integrity, packageManifest());
|
|
747
|
+
if (integrity.pause) {
|
|
748
|
+
const shown = integrity.diff.slice(0, INTEGRITY_SAMPLE);
|
|
749
|
+
log(
|
|
750
|
+
`ERROR: the installed conductor changed under this daemon — ${integrity.diff.length} file(s) differ ` +
|
|
751
|
+
`(${shown.join(", ")}${integrity.diff.length > shown.length ? ", …" : ""}) — pausing`,
|
|
752
|
+
);
|
|
753
|
+
setPaused(true);
|
|
754
|
+
if (integrity.page) {
|
|
755
|
+
const delivered = await safeEscalate(d, {
|
|
756
|
+
tier: 2,
|
|
757
|
+
project: project.name,
|
|
758
|
+
issue: NO_ISSUE,
|
|
759
|
+
// Dated for the same reason the spend cap is: the dedup key is the
|
|
760
|
+
// summary, and a second tamper months later must not be swallowed as a
|
|
761
|
+
// repeat of the first.
|
|
762
|
+
summary:
|
|
763
|
+
`Installed conductor changed under a running daemon on ${new Date().toISOString().slice(0, 10)}: ` +
|
|
764
|
+
`${integrity.diff.length} file(s) differ (first: ${integrity.diff[0]}) — ${project.name} is paused`,
|
|
765
|
+
detail: [
|
|
766
|
+
`Package root: ${import.meta.dir}`,
|
|
767
|
+
...shown,
|
|
768
|
+
...(integrity.diff.length > shown.length ? [`… and ${integrity.diff.length - shown.length} more`] : []),
|
|
769
|
+
"",
|
|
770
|
+
"If you deployed a new build, restart the daemon — the restart re-records the baseline.",
|
|
771
|
+
"If you did not, the host edited itself while it was dispatching work: treat every run since",
|
|
772
|
+
"the last known-good restart as unattributable before resuming.",
|
|
773
|
+
"`omp-conductor resume` alone will not hold — the next tick re-pauses while the files differ.",
|
|
774
|
+
].join("\n"),
|
|
775
|
+
});
|
|
776
|
+
markPaged(d.integrity, delivered);
|
|
777
|
+
}
|
|
778
|
+
return;
|
|
779
|
+
}
|
|
780
|
+
|
|
361
781
|
// route() filters the queue through isEligible() itself, so anything already
|
|
362
782
|
// carrying a state label is gone before it gets here.
|
|
363
783
|
const { routed, unroutable } = route(await d.tracker.listReady(), project);
|
|
@@ -405,8 +825,8 @@ async function tick(d: Deps): Promise<void> {
|
|
|
405
825
|
|
|
406
826
|
// Two different questions, deliberately two queries. Capacity counts worker
|
|
407
827
|
// *processes*, so a green PR awaiting a human merge must not consume a slot —
|
|
408
|
-
// two of those would otherwise stop the fleet.
|
|
409
|
-
//
|
|
828
|
+
// two of those would otherwise stop the fleet. That same PR's *issue* must
|
|
829
|
+
// still be occupied, which is what `admitCandidates`' busy set is for.
|
|
410
830
|
const live = store.liveRuns(project.name);
|
|
411
831
|
const slots = caps.maxConcurrentWorkers - live.length;
|
|
412
832
|
if (slots <= 0) {
|
|
@@ -414,32 +834,7 @@ async function tick(d: Deps): Promise<void> {
|
|
|
414
834
|
return;
|
|
415
835
|
}
|
|
416
836
|
|
|
417
|
-
const
|
|
418
|
-
|
|
419
|
-
const admitted: { r: Routed; attempt: number }[] = [];
|
|
420
|
-
for (const r of routed) {
|
|
421
|
-
if (admitted.length >= slots) break;
|
|
422
|
-
if (busy.has(r.issue.number)) continue;
|
|
423
|
-
|
|
424
|
-
const prior = store.attemptsFor(project.name, r.issue.number);
|
|
425
|
-
if (prior >= caps.maxAttemptsPerIssue) {
|
|
426
|
-
await safeEscalate(d, {
|
|
427
|
-
tier: 1,
|
|
428
|
-
project: project.name,
|
|
429
|
-
issue: r.issue.number,
|
|
430
|
-
summary: `#${r.issue.number} has used all ${caps.maxAttemptsPerIssue} attempts`,
|
|
431
|
-
detail: [
|
|
432
|
-
r.issue.title,
|
|
433
|
-
r.issue.url,
|
|
434
|
-
"Another attempt almost always means the issue itself is underspecified.",
|
|
435
|
-
"Rewrite the acceptance criteria, or take it off the queue.",
|
|
436
|
-
].join("\n"),
|
|
437
|
-
});
|
|
438
|
-
continue;
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
admitted.push({ r, attempt: prior + 1 });
|
|
442
|
-
}
|
|
837
|
+
const admitted = await admitCandidates(d, routed, slots);
|
|
443
838
|
|
|
444
839
|
if (admitted.length === 0) return;
|
|
445
840
|
|
|
@@ -612,6 +1007,15 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
612
1007
|
const store = openStore(dbPath());
|
|
613
1008
|
const tracker = makeTracker(project);
|
|
614
1009
|
|
|
1010
|
+
// Recorded here, before a single tick runs, so that the deploy an operator
|
|
1011
|
+
// *means* to do never trips the tripwire: installing a new build and
|
|
1012
|
+
// restarting the unit re-records this from the new files. What it catches is
|
|
1013
|
+
// the other thing — the package changing while the daemon that dispatches
|
|
1014
|
+
// work is holding it open, whether that is a worker that wandered out of its
|
|
1015
|
+
// worktree or a human editing the live install "just to test something".
|
|
1016
|
+
const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
|
|
1017
|
+
log(`package integrity baseline: ${integrity.baseline.size} files under ${import.meta.dir}`);
|
|
1018
|
+
|
|
615
1019
|
// Before the first tick, settle what the last process left behind — unless
|
|
616
1020
|
// another daemon is alive (a foreground `daemon --once` beside a running
|
|
617
1021
|
// daemon must not orphan that daemon's real, live workers).
|
|
@@ -641,8 +1045,16 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
641
1045
|
"fails twice, its gates stay red, its branch conflicts, or a tripwire fires.",
|
|
642
1046
|
"Your job when that happens: re-brief the issue (comment what the next worker must do",
|
|
643
1047
|
`differently, then put ${project.queueLabel} back on it), file follow-up issues, or promote to`,
|
|
644
|
-
"tier 2 and let the human decide.
|
|
645
|
-
|
|
1048
|
+
"tier 2 and let the human decide.",
|
|
1049
|
+
// Worded from `authority.merge` rather than fixed, so the standing orders
|
|
1050
|
+
// and the Releases section of the rendered brief cannot disagree about who
|
|
1051
|
+
// is holding the merge button. The daemon still merges nothing itself.
|
|
1052
|
+
project.authority.merge === "orchestrator"
|
|
1053
|
+
? "You never edit product code or push a branch — a worker session does that. Merging is yours: one PR at " +
|
|
1054
|
+
"a time, freshness-checked against the base branch, per the Releases section of your ORCHESTRATOR.md."
|
|
1055
|
+
: "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
|
|
1056
|
+
"human merges.",
|
|
1057
|
+
"Handle each escalation below before the next one.",
|
|
646
1058
|
].join("\n");
|
|
647
1059
|
|
|
648
1060
|
// One orchestrator per daemon run, not per tick: it is a persistent session
|
|
@@ -651,17 +1063,25 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
651
1063
|
// directory, deliberately not a checkout — the orchestrator re-briefs workers
|
|
652
1064
|
// and talks to the tracker, it does not edit product code.
|
|
653
1065
|
let orchestrator: OrchestratorHandle | undefined;
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
1066
|
+
if (project.escalation.orchestrator === "external") {
|
|
1067
|
+
// An operator already runs the brain — typically a visible TUI session that
|
|
1068
|
+
// drains `blocked`/`failed` off the tracker as one of its standing duties.
|
|
1069
|
+
// Starting a second one here would re-triage the same issues from a
|
|
1070
|
+
// transcript nobody is watching, and the two would undo each other.
|
|
1071
|
+
log("orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty");
|
|
1072
|
+
} else {
|
|
1073
|
+
try {
|
|
1074
|
+
orchestrator = await startOrchestrator({ cwd: stateDir(), brief });
|
|
1075
|
+
const transcript = orchestrator.sessionFile();
|
|
1076
|
+
log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
|
|
1077
|
+
} catch (err) {
|
|
1078
|
+
// Loudly, but not fatally: tier-1 escalations degrade to issue comments,
|
|
1079
|
+
// which a human still reads. A dispatcher that refuses to run because its
|
|
1080
|
+
// re-briefing channel is down helps nobody.
|
|
1081
|
+
log(
|
|
1082
|
+
`WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue comments: ${errText(err)}`,
|
|
1083
|
+
);
|
|
1084
|
+
}
|
|
665
1085
|
}
|
|
666
1086
|
|
|
667
1087
|
const escalator = createEscalator(project, tracker, store, orchestrator);
|
|
@@ -671,6 +1091,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
671
1091
|
tracker,
|
|
672
1092
|
store,
|
|
673
1093
|
escalate: (e) => escalator.escalate(e),
|
|
1094
|
+
integrity,
|
|
1095
|
+
// Fresh per daemon run, like the integrity gate: a restart is entitled to
|
|
1096
|
+
// page again about a stall that is still on disk.
|
|
1097
|
+
stall: { paged: false },
|
|
674
1098
|
};
|
|
675
1099
|
|
|
676
1100
|
if (o.once) {
|