omp-conductor 0.5.6 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/package.json +1 -1
- package/src/approval-surface.ts +98 -4
- package/src/board.ts +134 -7
- package/src/briefs/orchestrator.md +30 -9
- package/src/briefs/policy.md +5 -4
- package/src/cli.ts +95 -2
- package/src/daemon.ts +117 -7
- package/src/failure-class.ts +62 -0
- package/src/fleet.ts +19 -1
- package/src/orchestrator-tick.ts +107 -19
- package/src/store.ts +14 -3
- package/src/tracker/github.ts +16 -0
- package/src/types.ts +12 -0
- package/src/unblock.ts +30 -1
- package/src/verbs/actions.ts +8 -0
- package/src/verbs/protocol.ts +29 -0
- package/src/verbs/server.ts +113 -10
- package/src/worktree.ts +24 -1
package/src/cli.ts
CHANGED
|
@@ -23,7 +23,7 @@ import {
|
|
|
23
23
|
} from "./brief-upgrade.ts";
|
|
24
24
|
import { findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
|
|
25
25
|
import { CONDITION_FORMS, parseCondition } from "./decisions.ts";
|
|
26
|
-
import { runDaemon, setPaused } from "./daemon.ts";
|
|
26
|
+
import { isPaused, pausedAt, runDaemon, setPaused } from "./daemon.ts";
|
|
27
27
|
import {
|
|
28
28
|
armTicks,
|
|
29
29
|
clearPaneHalt,
|
|
@@ -58,7 +58,9 @@ import { dbPath, LIVE_STATES, openStore } from "./store.ts";
|
|
|
58
58
|
import { formatTranscriptLine } from "./transcript.ts";
|
|
59
59
|
import { formatVerbLedgerEntry } from "./verbs/ledger.ts";
|
|
60
60
|
import { makeTracker } from "./tracker/github.ts";
|
|
61
|
-
import {
|
|
61
|
+
import { githubVerbActions } from "./verbs/actions.ts";
|
|
62
|
+
import { handleVerbCall, type VerbChannel } from "./verbs/server.ts";
|
|
63
|
+
import { REPORT_KINDS, VERB_NAMES } from "./types.ts";
|
|
62
64
|
import type { ProjectConfig, ReportKind } from "./types.ts";
|
|
63
65
|
import { formatUnblock, unblockIssue } from "./unblock.ts";
|
|
64
66
|
import { upgradeConductor } from "./upgrade.ts";
|
|
@@ -100,6 +102,7 @@ usage:
|
|
|
100
102
|
omp-conductor tail <issue> [--project NAME]
|
|
101
103
|
omp-conductor extend <issue> --turns N [--project NAME]
|
|
102
104
|
omp-conductor unblock <issue> [--force] [--project NAME]
|
|
105
|
+
omp-conductor verb <conductor_*> [--project NAME] [--arg k=v ...]
|
|
103
106
|
omp-conductor daemon [--once] [--port N] [--project NAME]
|
|
104
107
|
omp-conductor pause
|
|
105
108
|
omp-conductor resume
|
|
@@ -168,6 +171,13 @@ usage:
|
|
|
168
171
|
Refuses when the newest attempt's work could not be committed and
|
|
169
172
|
its worktree is the only copy: re-claiming removes that tree. Use
|
|
170
173
|
--force once you have recovered it or accepted the loss.
|
|
174
|
+
verb run one conductor_* verb as the orchestrator, from the CLI: the same
|
|
175
|
+
checks and the same ledger rows a session's call would get. This is
|
|
176
|
+
how an external orchestrator merges, labels, releases or reads PR
|
|
177
|
+
state without a raw gh call that skips every gate. Arguments are
|
|
178
|
+
strings, one per --arg (e.g. --arg prUrl=https://x --arg headSha=y).
|
|
179
|
+
A refusal exits 3. See conductor_pr_merge/conductor_label/
|
|
180
|
+
conductor_release/conductor_pr_update_branch in the brief.
|
|
171
181
|
report hand a rendered report to the daemon's durable outbox. The report is
|
|
172
182
|
persisted before anything is sent, delivered by the daemon with
|
|
173
183
|
bounded retries, and shown by status until it lands. Delivery is
|
|
@@ -732,6 +742,89 @@ try {
|
|
|
732
742
|
break;
|
|
733
743
|
}
|
|
734
744
|
|
|
745
|
+
/**
|
|
746
|
+
* The external-orchestrator half of the verbs (#167): run any
|
|
747
|
+
* `conductor_*` verb from the CLI, through the daemon's own checks and
|
|
748
|
+
* ledger. An orchestrator that merges, labels or releases through this
|
|
749
|
+
* gets exactly the same refusals a session would and writes the same
|
|
750
|
+
* ledger rows — the raw `gh pr merge` a session reaches for instead is
|
|
751
|
+
* invisible to both.
|
|
752
|
+
*
|
|
753
|
+
* Worker-only verbs (conductor_push, conductor_pr_create) refuse with
|
|
754
|
+
* `role-not-allowed` — correct, and deliberately not special-cased here.
|
|
755
|
+
*
|
|
756
|
+
* Accepted limitation: the daemon's in-process one-merge-per-project slot
|
|
757
|
+
* does not span the daemon and a concurrent CLI merge. The execution-time
|
|
758
|
+
* head re-read (`--match-head-commit` in prMergeVerb) is the cross-process
|
|
759
|
+
* guard, so a CLI merge races a daemon merge exactly as two daemon merges
|
|
760
|
+
* would.
|
|
761
|
+
*/
|
|
762
|
+
case "verb": {
|
|
763
|
+
const name = argv[1];
|
|
764
|
+
if (name === undefined || !VERB_NAMES.some((n) => n === name)) {
|
|
765
|
+
process.stderr.write(
|
|
766
|
+
`omp-conductor: unknown verb "${name ?? ""}". Known verbs: ${VERB_NAMES.join(", ")}\n`,
|
|
767
|
+
);
|
|
768
|
+
process.exit(2);
|
|
769
|
+
}
|
|
770
|
+
const cfg = loadConfig();
|
|
771
|
+
const project = findProject(cfg, flag(argv, "project"));
|
|
772
|
+
const args: Record<string, string> = {};
|
|
773
|
+
for (let i = 2; i < argv.length; i++) {
|
|
774
|
+
const token = argv[i];
|
|
775
|
+
if (token === "--arg") {
|
|
776
|
+
const pair = argv[i + 1];
|
|
777
|
+
if (pair === undefined || !pair.includes("=")) {
|
|
778
|
+
process.stderr.write(`omp-conductor: verb --arg needs k=v, got "${pair ?? ""}"\n`);
|
|
779
|
+
process.exit(2);
|
|
780
|
+
}
|
|
781
|
+
const eq = pair.indexOf("=");
|
|
782
|
+
args[pair.slice(0, eq)] = pair.slice(eq + 1);
|
|
783
|
+
i++;
|
|
784
|
+
} else if (token === "--project" || token?.startsWith("--project=") === true) {
|
|
785
|
+
// Consumed by `flag(argv, "project")` above; both spellings skip here.
|
|
786
|
+
if (token === "--project") i++;
|
|
787
|
+
} else {
|
|
788
|
+
process.stderr.write(
|
|
789
|
+
`omp-conductor: verb: unexpected argument "${token ?? ""}" (pass verb arguments as --arg k=v)\n`,
|
|
790
|
+
);
|
|
791
|
+
process.exit(2);
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
const store = openStore(dbPath());
|
|
795
|
+
try {
|
|
796
|
+
const channel: VerbChannel = {
|
|
797
|
+
kind: "orchestrator",
|
|
798
|
+
path: "cli",
|
|
799
|
+
project: project.name,
|
|
800
|
+
role: "orchestrator",
|
|
801
|
+
};
|
|
802
|
+
const reply = await handleVerbCall(
|
|
803
|
+
{
|
|
804
|
+
project: () => findProject(loadConfig(), project.name),
|
|
805
|
+
store,
|
|
806
|
+
tracker: makeTracker(project),
|
|
807
|
+
actions: githubVerbActions(project),
|
|
808
|
+
fleetStop: () =>
|
|
809
|
+
isPaused()
|
|
810
|
+
? "claiming is paused for this fleet (omp-conductor pause, hold or halt)"
|
|
811
|
+
: undefined,
|
|
812
|
+
pausedAt,
|
|
813
|
+
log: (m) => process.stderr.write(`${m}\n`),
|
|
814
|
+
now: () => Date.now(),
|
|
815
|
+
},
|
|
816
|
+
channel,
|
|
817
|
+
{ verb: name, args },
|
|
818
|
+
);
|
|
819
|
+
process.stdout.write(`${reply.text}\n`);
|
|
820
|
+
// A refusal must not read as success to a script calling this.
|
|
821
|
+
if (!reply.ok) process.exitCode = 3;
|
|
822
|
+
} finally {
|
|
823
|
+
store.close();
|
|
824
|
+
}
|
|
825
|
+
break;
|
|
826
|
+
}
|
|
827
|
+
|
|
735
828
|
/**
|
|
736
829
|
* The handover point. Authorship stays with the model; from here the daemon
|
|
737
830
|
* owns delivery, so "I sent the report" stops being a claim the model makes
|
package/src/daemon.ts
CHANGED
|
@@ -102,6 +102,11 @@ const GRAPH_HEALTH_INTERVAL_MS = 60_000;
|
|
|
102
102
|
* report an operator is waiting on must not sit in the outbox for the length of
|
|
103
103
|
* a poll interval, and delivery is owed even while claiming is paused (#123). */
|
|
104
104
|
const REPORT_DELIVERY_INTERVAL_MS = 30_000;
|
|
105
|
+
/** A dispatch-infra run (turn-0 git failure) is requeued so the next tick
|
|
106
|
+
* retries — but only a bounded number of times. Three strikes for one issue
|
|
107
|
+
* means the mirror itself is broken, not unlucky, and the sweep escalates
|
|
108
|
+
* instead of burning a turn-0 run per tick forever (#168, #177). */
|
|
109
|
+
const DISPATCH_INFRA_MAX_STRIKES = 3;
|
|
105
110
|
const DEFAULT_PORT = 8787;
|
|
106
111
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
107
112
|
|
|
@@ -121,8 +126,13 @@ export interface DaemonOpts {
|
|
|
121
126
|
project?: string;
|
|
122
127
|
}
|
|
123
128
|
|
|
124
|
-
/** Everything one tick touches
|
|
125
|
-
*
|
|
129
|
+
/** Everything one tick touches. `project` and `caps` are re-resolved at each
|
|
130
|
+
* tick boundary so an operator's config edit applies on the next tick rather
|
|
131
|
+
* than the next daemon restart (#170); a tick and the runs it admits see one
|
|
132
|
+
* consistent snapshot, and a mid-run edit never changes a live run's labels,
|
|
133
|
+
* model or caps — `handleIssue` destructures them at dispatch time. The rest
|
|
134
|
+
* are resolved once at startup so a tick never re-reads config mid-flight and
|
|
135
|
+
* changes its own limits underneath itself. */
|
|
126
136
|
interface Deps {
|
|
127
137
|
project: ProjectConfig;
|
|
128
138
|
caps: Caps;
|
|
@@ -176,6 +186,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
176
186
|
isPaused()
|
|
177
187
|
? "claiming is paused for this fleet (omp-conductor pause, hold or halt)"
|
|
178
188
|
: undefined,
|
|
189
|
+
pausedAt,
|
|
179
190
|
log,
|
|
180
191
|
now: () => Date.now(),
|
|
181
192
|
};
|
|
@@ -288,6 +299,30 @@ export function isPaused(): boolean {
|
|
|
288
299
|
return existsSync(join(stateDir(), "paused"));
|
|
289
300
|
}
|
|
290
301
|
|
|
302
|
+
/**
|
|
303
|
+
* The epoch-ms timestamp at which the current pause began, read from the same
|
|
304
|
+
* sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
|
|
305
|
+
* `undefined` when the fleet is not paused, or when the file's first line does
|
|
306
|
+
* not parse as a date — a legacy/blank sentinel keeps today's refuse-everything
|
|
307
|
+
* behavior, because a run admitted before an *unknown* pause cannot be proven
|
|
308
|
+
* innocent. {@link isPaused} is the authority on *whether*; this answers
|
|
309
|
+
* *since when*.
|
|
310
|
+
*/
|
|
311
|
+
export function pausedAt(): number | undefined {
|
|
312
|
+
const f = join(stateDir(), "paused");
|
|
313
|
+
if (!existsSync(f)) return undefined;
|
|
314
|
+
try {
|
|
315
|
+
const first = readFileSync(f, "utf8").split("\n")[0]?.trim();
|
|
316
|
+
if (first === undefined || first === "") return undefined;
|
|
317
|
+
const t = Date.parse(first);
|
|
318
|
+
return Number.isNaN(t) ? undefined : t;
|
|
319
|
+
} catch {
|
|
320
|
+
// Unreadable sentinel (permissions, corruption): fail closed like an
|
|
321
|
+
// unparseable line — refuse mutations while the pause is unprovable.
|
|
322
|
+
return undefined;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
291
326
|
export function setPaused(v: boolean): void {
|
|
292
327
|
const f = join(stateDir(), "paused");
|
|
293
328
|
if (v) {
|
|
@@ -1709,8 +1744,19 @@ export async function admitCandidates(
|
|
|
1709
1744
|
slots: number,
|
|
1710
1745
|
): Promise<AdmissionPass> {
|
|
1711
1746
|
const { project, caps, tracker, store } = d;
|
|
1712
|
-
const
|
|
1747
|
+
const activeRuns = store.activeRuns(project.name);
|
|
1748
|
+
const busyIssues = activeRuns.map((r) => r.issue);
|
|
1713
1749
|
const busy = new Set(busyIssues);
|
|
1750
|
+
// issue -> its active run rows, for the pushed-green admission bypass (#175):
|
|
1751
|
+
// only a worker-free pushed-green row may be bypassed, and only when *every*
|
|
1752
|
+
// active run for the issue is worker-free. A live (claimed/running) row still
|
|
1753
|
+
// holds unconditionally.
|
|
1754
|
+
const activeByIssue = new Map<number, RunRecord[]>();
|
|
1755
|
+
for (const run of activeRuns) {
|
|
1756
|
+
const list = activeByIssue.get(run.issue);
|
|
1757
|
+
if (list === undefined) activeByIssue.set(run.issue, [run]);
|
|
1758
|
+
else list.push(run);
|
|
1759
|
+
}
|
|
1714
1760
|
const holds: AdmissionHold[] = [];
|
|
1715
1761
|
const hold = (issue: number, reason: AdmissionHoldReason): void => {
|
|
1716
1762
|
holds.push({ issue, reason });
|
|
@@ -1769,8 +1815,17 @@ export async function admitCandidates(
|
|
|
1769
1815
|
continue;
|
|
1770
1816
|
}
|
|
1771
1817
|
if (busy.has(issue)) {
|
|
1772
|
-
|
|
1773
|
-
|
|
1818
|
+
// A pushed-green row is worker-free by definition (it is not in
|
|
1819
|
+
// LIVE_STATES): its PR is live but no process is writing to its branch.
|
|
1820
|
+
// So an issue whose active runs are ALL pushed-green is not actually
|
|
1821
|
+
// occupied — the corrective attempt the operator unblocked may be
|
|
1822
|
+
// admitted as a continuation of that PR, and the open-PR gate below
|
|
1823
|
+
// decides the identity. Any live row still holds (#175).
|
|
1824
|
+
const allWorkerFree = (activeByIssue.get(issue) ?? []).every((r) => r.state === "pushed-green");
|
|
1825
|
+
if (!allWorkerFree) {
|
|
1826
|
+
hold(issue, "issue-active");
|
|
1827
|
+
continue;
|
|
1828
|
+
}
|
|
1774
1829
|
}
|
|
1775
1830
|
|
|
1776
1831
|
const priorRuns = store.attemptsFor(project.name, issue);
|
|
@@ -1888,7 +1943,8 @@ export async function admitCandidates(
|
|
|
1888
1943
|
latest?.state === "blocked" ||
|
|
1889
1944
|
latest?.state === "failed" ||
|
|
1890
1945
|
latest?.state === "killed" ||
|
|
1891
|
-
latest?.state === "orphaned"
|
|
1946
|
+
latest?.state === "orphaned" ||
|
|
1947
|
+
latest?.state === "pushed-green"
|
|
1892
1948
|
? latest
|
|
1893
1949
|
: undefined;
|
|
1894
1950
|
// The second half asks "is this open PR our retained work", and accepts
|
|
@@ -1966,6 +2022,24 @@ export async function dispatchAdmissions(
|
|
|
1966
2022
|
// ----------------------------------------------------------------------- a tick
|
|
1967
2023
|
|
|
1968
2024
|
export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
2025
|
+
// A config edit takes effect on the next tick, not the next daemon restart
|
|
2026
|
+
// (#170). Re-resolve the project and its caps at the tick boundary so a tick
|
|
2027
|
+
// and every run it admits see one consistent snapshot; a failed read keeps
|
|
2028
|
+
// the boot values rather than wedging the tick, and the next tick tries
|
|
2029
|
+
// again.
|
|
2030
|
+
try {
|
|
2031
|
+
const cfg = loadConfig();
|
|
2032
|
+
const fresh = findProject(cfg, d.project.name);
|
|
2033
|
+
const freshCaps = resolveCaps(fresh, cfg.defaults);
|
|
2034
|
+
if (JSON.stringify(fresh) !== JSON.stringify(d.project) || JSON.stringify(freshCaps) !== JSON.stringify(d.caps)) {
|
|
2035
|
+
log(`config reloaded: project ${d.project.name} changed on disk — applying from this tick`);
|
|
2036
|
+
}
|
|
2037
|
+
d.project = fresh;
|
|
2038
|
+
d.caps = freshCaps;
|
|
2039
|
+
} catch (err) {
|
|
2040
|
+
log(`config reload failed (${errText(err)}) — continuing with the values loaded at boot`);
|
|
2041
|
+
}
|
|
2042
|
+
|
|
1969
2043
|
// Before the pause check, deliberately. This one is not about dispatch: the
|
|
1970
2044
|
// orchestrator is a different process, and it can be wedged while this fleet
|
|
1971
2045
|
// is paused — which is exactly the state the reference fleet was in when the
|
|
@@ -2643,6 +2717,14 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
2643
2717
|
}
|
|
2644
2718
|
if (run.state === "failed" && facts.pr === "open") {
|
|
2645
2719
|
facts.checks = await tracker.checkConclusions(run.prUrl);
|
|
2720
|
+
// When a check failed with a reachable log, pull its tail so the
|
|
2721
|
+
// table can tell an infra outage (#177) from a real test failure by
|
|
2722
|
+
// the log's own words. First failure wins; a log that cannot be
|
|
2723
|
+
// fetched is left undefined and classification stays conservative.
|
|
2724
|
+
const firstFailure = facts.checks.find((c) => c.state === "failure" && c.link !== undefined);
|
|
2725
|
+
if (firstFailure?.link !== undefined) {
|
|
2726
|
+
facts.failingLog = await tracker.checkLog(firstFailure.link);
|
|
2727
|
+
}
|
|
2646
2728
|
}
|
|
2647
2729
|
}
|
|
2648
2730
|
} catch (err) {
|
|
@@ -2730,6 +2812,26 @@ async function recoverRun(
|
|
|
2730
2812
|
}
|
|
2731
2813
|
|
|
2732
2814
|
if (recovery === "requeue") {
|
|
2815
|
+
// A dispatch-infra requeue that keeps landing on the same issue means the
|
|
2816
|
+
// mirror for its repo is persistently broken — a ref-lock that retry already
|
|
2817
|
+
// exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
|
|
2818
|
+
// with no chance of success, so after a bounded number of strikes this
|
|
2819
|
+
// escalates to a human instead (#168, #177).
|
|
2820
|
+
if (cls === "dispatch-infra" && store.classCountFor(project.name, run.issue, "dispatch-infra") >= DISPATCH_INFRA_MAX_STRIKES) {
|
|
2821
|
+
await safeEscalate(d, {
|
|
2822
|
+
tier: 1,
|
|
2823
|
+
project: project.name,
|
|
2824
|
+
issue: run.issue,
|
|
2825
|
+
summary: `[dispatch-infra] #${run.issue}: the mirror for ${run.repo} is failing persistently — ${evidence}`,
|
|
2826
|
+
detail: [
|
|
2827
|
+
`The dispatcher could not provision a worktree for #${run.issue} ${DISPATCH_INFRA_MAX_STRIKES} times in a row, all before the worker's first turn.`,
|
|
2828
|
+
"The mirror on this host needs attention (check disk, SSH/HTTPS credentials, and the mirror root).",
|
|
2829
|
+
].join("\n"),
|
|
2830
|
+
});
|
|
2831
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
2832
|
+
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
2833
|
+
return;
|
|
2834
|
+
}
|
|
2733
2835
|
// Only when the tracker still shows this issue as ours to hand back. An
|
|
2734
2836
|
// issue that is closed, or has no state label, was resolved by another route
|
|
2735
2837
|
// and requeueing it would dispatch work nobody asked for.
|
|
@@ -2772,7 +2874,15 @@ async function recoverRun(
|
|
|
2772
2874
|
);
|
|
2773
2875
|
}
|
|
2774
2876
|
if (run.lastError !== undefined && cls !== "question") detail.push(run.lastError);
|
|
2775
|
-
|
|
2877
|
+
// #172: an unwritten transcript is "the run died before it flushed", not a
|
|
2878
|
+
// link to a file the operator will open and find missing.
|
|
2879
|
+
detail.push(
|
|
2880
|
+
run.sessionFile === undefined
|
|
2881
|
+
? "Session: (no transcript)"
|
|
2882
|
+
: existsSync(run.sessionFile)
|
|
2883
|
+
? `Session: ${run.sessionFile}`
|
|
2884
|
+
: `Session: ${run.sessionFile} (file was never written — the run died before its transcript was flushed)`,
|
|
2885
|
+
);
|
|
2776
2886
|
// The class and the run are in the summary, which is what the notifications
|
|
2777
2887
|
// ledger dedupes on — so one class escalates once per run rather than every
|
|
2778
2888
|
// five minutes.
|
package/src/failure-class.ts
CHANGED
|
@@ -22,6 +22,10 @@ export interface ClassifyFacts {
|
|
|
22
22
|
pr?: "open" | "merged" | "closed";
|
|
23
23
|
mergeable?: "conflicting" | "clean" | "unknown";
|
|
24
24
|
checks?: { name: string; state: string; link?: string }[];
|
|
25
|
+
/** Tail (ANSI-stripped) of the first failed check's log, when one was
|
|
26
|
+
* reachable. Lets the table tell an infrastructure outage (#177) from a
|
|
27
|
+
* deterministic test failure by the log's own words. */
|
|
28
|
+
failingLog?: string;
|
|
25
29
|
}
|
|
26
30
|
|
|
27
31
|
export interface Classification {
|
|
@@ -50,6 +54,24 @@ const INFRA_CHECK_STATES: Record<string, true> = {
|
|
|
50
54
|
/** States that mean "this check has a verdict and it is good". */
|
|
51
55
|
const SUCCESS_CHECK_STATES: Record<string, true> = { success: true, neutral: true };
|
|
52
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Substrings in a failed check's log that prove the failure was infrastructure,
|
|
59
|
+
* not the diff (#177). Each is a registry/docker/runner fault a worker cannot
|
|
60
|
+
* have introduced: a rate limit, an image-manifest resolution failure, a runner
|
|
61
|
+
* being torn down under the job, or a DNS failure. Matched lowercased against
|
|
62
|
+
* the log tail.
|
|
63
|
+
*
|
|
64
|
+
* Deliberately *not* matching bare `failed to solve:` — a docker build failure
|
|
65
|
+
* often prints it with a real resolution error, so the closing words carry the
|
|
66
|
+
* signal, not the phrase.
|
|
67
|
+
*/
|
|
68
|
+
const INFRA_LOG_SIGNATURES = [
|
|
69
|
+
"429 too many requests",
|
|
70
|
+
"failed to resolve source metadata for",
|
|
71
|
+
"the runner has received a shutdown signal",
|
|
72
|
+
"could not resolve host",
|
|
73
|
+
];
|
|
74
|
+
|
|
53
75
|
function normalise(state: string): string {
|
|
54
76
|
return state.trim().toLowerCase();
|
|
55
77
|
}
|
|
@@ -161,6 +183,28 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
|
|
|
161
183
|
}
|
|
162
184
|
}
|
|
163
185
|
|
|
186
|
+
// Dispatch infra: the conductor's own git path failed before the worker's
|
|
187
|
+
// first turn — a mirror ref-lock (#168), a dead mirror, a transiently
|
|
188
|
+
// unreachable host. The worker never touched the issue, so this should not
|
|
189
|
+
// charge an implementation attempt. `lastError` here is written only by the
|
|
190
|
+
// dispatch catch (`errText(err)` = the stack, which prefixes the thrown
|
|
191
|
+
// "git … exited N: …" message with "Error: ") and by state messages — never
|
|
192
|
+
// by a worker — so that shape is proof of the daemon's own git, not a
|
|
193
|
+
// worker's failed push.
|
|
194
|
+
if (
|
|
195
|
+
run.state === "failed" &&
|
|
196
|
+
run.turns === 0 &&
|
|
197
|
+
!hasArtifacts &&
|
|
198
|
+
run.lastError !== undefined &&
|
|
199
|
+
/^(?:Error: )?git .+ exited \d+/s.test(run.lastError)
|
|
200
|
+
) {
|
|
201
|
+
return {
|
|
202
|
+
cls: "dispatch-infra",
|
|
203
|
+
recovery: "requeue",
|
|
204
|
+
evidence: `dispatch failed in the conductor's own git path before turn 1: ${run.lastError.split("\n")[0]}`,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
164
208
|
if (run.state === "blocked") {
|
|
165
209
|
return {
|
|
166
210
|
cls: "question",
|
|
@@ -215,6 +259,24 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
|
|
|
215
259
|
const checks = facts.checks ?? [];
|
|
216
260
|
const unresolved = checks.filter((c) => !SUCCESS_CHECK_STATES[normalise(c.state)] === true);
|
|
217
261
|
if (checks.length > 0 && unresolved.length > 0) {
|
|
262
|
+
// A failed check whose *log* smells like infrastructure — a registry 429,
|
|
263
|
+
// a runner shutdown, a DNS failure (#177). The check has a verdict, so
|
|
264
|
+
// the check-state table above cannot call it infra; only the log can. It
|
|
265
|
+
// beats ci-deterministic because charging an implementation attempt for a
|
|
266
|
+
// rate limit is exactly the waste that class exists to prevent.
|
|
267
|
+
if (facts.failingLog !== undefined) {
|
|
268
|
+
const lower = facts.failingLog.toLowerCase();
|
|
269
|
+
for (const signature of INFRA_LOG_SIGNATURES) {
|
|
270
|
+
if (lower.includes(signature)) {
|
|
271
|
+
const check = checks.find((c) => normalise(c.state) === "failure");
|
|
272
|
+
return {
|
|
273
|
+
cls: "ci-infra",
|
|
274
|
+
recovery: "rerun-checks",
|
|
275
|
+
evidence: `infrastructure signature in ${check?.name === undefined ? "failed check" : check.name} log: "${signature}"`,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
218
280
|
const failing = unresolved.filter((c) => normalise(c.state) === "failure");
|
|
219
281
|
if (failing.length > 0) {
|
|
220
282
|
return {
|
package/src/fleet.ts
CHANGED
|
@@ -29,7 +29,7 @@ import { homedir } from "node:os";
|
|
|
29
29
|
import { dirname, join } from "node:path";
|
|
30
30
|
import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
31
31
|
import { planUsageLine, readPlanUsage, sharedUsageSource } from "./usage.ts";
|
|
32
|
-
import { readApprovalSurface } from "./approval-surface.ts";
|
|
32
|
+
import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
|
|
33
33
|
import { inspectBriefLayout } from "./brief-upgrade.ts";
|
|
34
34
|
import { dbPath, openStore } from "./store.ts";
|
|
35
35
|
import { renderBriefForProject } from "./setup.ts";
|
|
@@ -1303,6 +1303,24 @@ export async function probeTelegramHealth(
|
|
|
1303
1303
|
if (approval.kind === "missing") {
|
|
1304
1304
|
return { kind: "degraded", detail: `${username}; inbound configured; ${approval.reason}` };
|
|
1305
1305
|
}
|
|
1306
|
+
// Last, and only once the surface is answerable: whether it is *quiet*.
|
|
1307
|
+
//
|
|
1308
|
+
// The order is the order an operator should fix things in, and one row carries
|
|
1309
|
+
// one remedy. A missing token means nothing outbound works, so the approval
|
|
1310
|
+
// surface is not worth discussing; a missing approval surface means an
|
|
1311
|
+
// amendment cannot be asked, which outranks noise; an interactive profile is
|
|
1312
|
+
// real but strictly the least severe — the fleet works, it is just loud. And
|
|
1313
|
+
// the two lower checks are not independent of each other: setting `notifyMode`
|
|
1314
|
+
// to fix the approval surface is what *arms* the `agent_end` notify post this
|
|
1315
|
+
// one warns about, so a fleet that fixes them in the other order would see the
|
|
1316
|
+
// noise appear as the reward for fixing the silence.
|
|
1317
|
+
const profile = readDaemonProfile(accessPath);
|
|
1318
|
+
if (profile.kind === "interactive") {
|
|
1319
|
+
return {
|
|
1320
|
+
kind: "degraded",
|
|
1321
|
+
detail: `${username}; inbound configured; interactive profile relays assistant text — /telegram set profile daemon`,
|
|
1322
|
+
};
|
|
1323
|
+
}
|
|
1306
1324
|
return { kind: "ok", detail: `${username}; inbound configured; telegram_ask available` };
|
|
1307
1325
|
}
|
|
1308
1326
|
|
package/src/orchestrator-tick.ts
CHANGED
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
bridgeTokenBound,
|
|
53
53
|
hasBotToken,
|
|
54
54
|
readApprovalSurface,
|
|
55
|
+
readDaemonProfile,
|
|
55
56
|
TELEGRAM_APPROVAL_TOOL,
|
|
56
57
|
} from "./approval-surface.ts";
|
|
57
58
|
import {
|
|
@@ -363,12 +364,27 @@ export const TICK_SCOPE_CONSTRAINTS: { readonly [K in ReportScope]: string } = {
|
|
|
363
364
|
/**
|
|
364
365
|
* The delivery clause, appended to every default tick prompt.
|
|
365
366
|
*
|
|
366
|
-
*
|
|
367
|
-
*
|
|
368
|
-
*
|
|
369
|
-
*
|
|
370
|
-
*
|
|
371
|
-
*
|
|
367
|
+
* A tick's end-of-turn text does not reach the operator as a *report*, and the
|
|
368
|
+
* rule's job is to stop a session believing otherwise: on 2026-08-06 the fleet
|
|
369
|
+
* this extension runs produced a release report and two tier-2 escalations as
|
|
370
|
+
* end-of-turn text, and not one of the three reached anybody.
|
|
371
|
+
*
|
|
372
|
+
* This comment used to explain that by saying end-of-turn text streams to
|
|
373
|
+
* Telegram only on a turn that *began* as an inbound Telegram message. That
|
|
374
|
+
* premise is false, and the correction matters because it flips the reason:
|
|
375
|
+
* omp-telegram's `agent_end` handler posts `finalText` to the notify chat on
|
|
376
|
+
* every run that did *not* come from Telegram, whenever `notifyMode` is set —
|
|
377
|
+
* and the fleet must set it, because {@link readApprovalSurface} needs it for
|
|
378
|
+
* the approval surface. So a tick's closing prose *was* reaching the operator
|
|
379
|
+
* all along: not as a delivered report, as an unlogged, unretried, untracked
|
|
380
|
+
* ping in the middle of whatever else was in that chat. Both the void and the
|
|
381
|
+
* ping are the same fault seen from two sides — text is not a delivery channel.
|
|
382
|
+
*
|
|
383
|
+
* omp-telegram 0.11.0's `profile: "daemon"` removes it in the other direction:
|
|
384
|
+
* explicit-only outbound, and the idle/final notify post suppressed outright, so
|
|
385
|
+
* a tick's visible text goes nowhere at all. That is the configuration the fleet
|
|
386
|
+
* runs (see {@link TICK_NARRATION_RULE}, which warns when it is absent), and it
|
|
387
|
+
* makes this rule's demand literal rather than merely prudent.
|
|
372
388
|
*
|
|
373
389
|
* The clause used to name `telegram_send`, and that held — but it was still an
|
|
374
390
|
* instruction where a mechanism was needed: the same miss recurs whenever the
|
|
@@ -384,6 +400,16 @@ export const TICK_SCOPE_CONSTRAINTS: { readonly [K in ReportScope]: string } = {
|
|
|
384
400
|
export const TICK_DELIVERY_RULE =
|
|
385
401
|
"This tick was injected locally, not sent from Telegram, so your end-of-turn text does NOT reach your operator. Hand anything reportable this turn to the durable outbox by running `omp-conductor report --text \"<the whole report>\"` (add `--kind digest` for the daily digest) and confirming it printed a report id; the daemon then owns delivery and retries until it lands. Never claim a report was sent otherwise, and never use telegram_send for a report -- that path leaves no record that it went out.";
|
|
386
402
|
|
|
403
|
+
/** The {@link TICK_DELIVERY_RULE} variant for a fleet whose bridge actually
|
|
404
|
+
* delivers the tick's ending text (#169). That is a narrower class than the
|
|
405
|
+
* approval surface being ready: `notifyMode: "always"` has to be set, and the
|
|
406
|
+
* profile has to be `interactive` — a `daemon` profile suppresses the
|
|
407
|
+
* `agent_end` notify post, so its end-of-turn text reaches nobody and the
|
|
408
|
+
* durable-outbox rule above is the honest one there. Everything here keeps the
|
|
409
|
+
* outbox as the message; only the opening claim about delivery differs. */
|
|
410
|
+
export const TICK_DELIVERY_RULE_BRIDGED =
|
|
411
|
+
"Your end-of-turn text IS delivered to the operator by the Telegram bridge (notifyMode: always). Still hand anything reportable to the durable outbox with omp-conductor report — and then close with at most one short line. The report is the message; never restate it in your closing text.";
|
|
412
|
+
|
|
387
413
|
/** Re-exported so the tick's own contract stays readable from one file: the
|
|
388
414
|
* constant itself lives beside the check that decides whether it is callable. */
|
|
389
415
|
export { TELEGRAM_APPROVAL_TOOL };
|
|
@@ -418,6 +444,38 @@ export const TICK_APPROVAL_UNAVAILABLE_RULE =
|
|
|
418
444
|
`If you have an amendment to propose, deliver the question with telegram_send and wait for your operator's reply on a later turn; ` +
|
|
419
445
|
`never apply an amendment, or record one as approved, without an explicit answer you actually received.`;
|
|
420
446
|
|
|
447
|
+
/**
|
|
448
|
+
* Appended to every tick — the shipped prompt or the operator's own — composed
|
|
449
|
+
* on a bridge that is not in omp-telegram's `profile: "daemon"`.
|
|
450
|
+
*
|
|
451
|
+
* The running commentary an operator sees is *mechanical*, not disobedience, and
|
|
452
|
+
* that is why a prompt line is the wrong permanent fix and the right interim one.
|
|
453
|
+
* Two verified paths carry visible text out without anyone calling a tool:
|
|
454
|
+
* `outbound.ts` `onTurnEnd()` finalizes one real Telegram message per assistant
|
|
455
|
+
* turn for as long as the chat is marked active — so a multi-step answer arrives
|
|
456
|
+
* as several messages, and a message that lands mid-tick keeps the chat active
|
|
457
|
+
* for the rest of that run, relaying every subsequent tick-internal turn — and
|
|
458
|
+
* the `agent_end` handler posts the closing text of every *local* run to the
|
|
459
|
+
* notify chat whenever `notifyMode` is set. The fleet has to set `notifyMode`
|
|
460
|
+
* (see {@link TICK_APPROVAL_UNAVAILABLE_RULE}), so the correctly-askable fleet is
|
|
461
|
+
* exactly the narrating one. `profile: "daemon"` closes both at the transport:
|
|
462
|
+
* explicit-only outbound, notify post suppressed.
|
|
463
|
+
*
|
|
464
|
+
* So this rule is a stopgap with an expiry date, and it says so: it names the
|
|
465
|
+
* one command that removes it. Until then the only defence is discipline the
|
|
466
|
+
* turn can actually exercise — no visible text between tool calls, and one
|
|
467
|
+
* `telegram_send` for a human answer rather than prose that leaks a turn at a
|
|
468
|
+
* time. It fires *because* the mechanism is absent, so a fleet that has run
|
|
469
|
+
* `/telegram set profile daemon` never sees it.
|
|
470
|
+
*
|
|
471
|
+
* Appended to a configured `message` too, for {@link TICK_APPROVAL_UNAVAILABLE_RULE}'s
|
|
472
|
+
* reason exactly: an operator's prompt owns the reporting contract, but it cannot
|
|
473
|
+
* consent on the orchestrator's behalf to what the transport does with its text.
|
|
474
|
+
* Transport truth is not the operator's prompt to waive.
|
|
475
|
+
*/
|
|
476
|
+
export const TICK_NARRATION_RULE =
|
|
477
|
+
"This session's Telegram bridge is NOT in daemon profile, so visible assistant text can auto-relay to your operator's chat (per-turn messages while a Telegram conversation is active, and end-of-run notify posts on local runs). Until the operator runs `/telegram set profile daemon`: produce no visible text between tool calls, and answer any human message with a single telegram_send call only.";
|
|
478
|
+
|
|
421
479
|
function frictionLabel(kind: FrictionSignal["kind"]): string {
|
|
422
480
|
if (kind.startsWith("admission:")) return `admission hold ${kind.slice("admission:".length)}`;
|
|
423
481
|
if (kind === "feedback:escalation-should-digest") return "escalations classified as digest material";
|
|
@@ -1147,6 +1205,29 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1147
1205
|
refreshComposedBriefBestEffort();
|
|
1148
1206
|
|
|
1149
1207
|
const scope = resolveTickScope();
|
|
1208
|
+
// The transport contract, read once from the same file at the same moment so
|
|
1209
|
+
// the approval line, the delivery rule and the narration line cannot disagree
|
|
1210
|
+
// (#169, #179). No access file means no fleet channel to judge; a session
|
|
1211
|
+
// with no bound bridge token means omp-telegram posts nothing, so both halves
|
|
1212
|
+
// fail toward "not delivered" — the durable outbox instruction — rather than
|
|
1213
|
+
// asserting a delivery the bridge cannot make.
|
|
1214
|
+
const approval =
|
|
1215
|
+
config.accessFile === undefined
|
|
1216
|
+
? undefined
|
|
1217
|
+
: session.bridgeTokenAtStart
|
|
1218
|
+
? readApprovalSurface(config.accessFile)
|
|
1219
|
+
: ({
|
|
1220
|
+
kind: "missing",
|
|
1221
|
+
reason:
|
|
1222
|
+
`${TELEGRAM_APPROVAL_TOOL} unavailable on local ticks: omp-telegram had no bot token when this ` +
|
|
1223
|
+
"session started, so it bound none and its tools stay dead however complete the access file looks " +
|
|
1224
|
+
"now — run `/telegram on` in this session, or restart it, to rebind the bridge",
|
|
1225
|
+
} as const);
|
|
1226
|
+
const profile =
|
|
1227
|
+
config.accessFile === undefined || !session.bridgeTokenAtStart
|
|
1228
|
+
? undefined
|
|
1229
|
+
: readDaemonProfile(config.accessFile);
|
|
1230
|
+
|
|
1150
1231
|
// A configured message owns the ordinary reporting and delivery clauses.
|
|
1151
1232
|
// Mechanical evidence is different: release-policy drift and repeated
|
|
1152
1233
|
// operational friction must not disappear because an operator customized the
|
|
@@ -1157,7 +1238,8 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1157
1238
|
session.scopeFallbackLogged = true;
|
|
1158
1239
|
pi.logger.info(`[omp-conductor] tick reporting scope: using ${DEFAULT_REPORT_SCOPE} — ${scope.fallback}`);
|
|
1159
1240
|
}
|
|
1160
|
-
|
|
1241
|
+
const bridged = approval?.kind === "ready" && approval.notifyMode === "always" && profile?.kind === "interactive";
|
|
1242
|
+
content = `${defaultTickMessage(new Date(), scope.briefPath, scope.policyPath)}\n${TICK_SCOPE_CONSTRAINTS[scope.scope]}\n${bridged ? TICK_DELIVERY_RULE_BRIDGED : TICK_DELIVERY_RULE}`;
|
|
1161
1243
|
}
|
|
1162
1244
|
let frictionStore: Store | undefined;
|
|
1163
1245
|
let frictionSignals: FrictionSignal[] = [];
|
|
@@ -1227,18 +1309,6 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1227
1309
|
// `telegram_ask` and `telegram_send` dead until `/telegram on`. Trusting the
|
|
1228
1310
|
// file alone there puts the tick straight back to mandating a call its surface
|
|
1229
1311
|
// cannot make, which is #114 exactly.
|
|
1230
|
-
const approval =
|
|
1231
|
-
config.accessFile === undefined
|
|
1232
|
-
? undefined
|
|
1233
|
-
: session.bridgeTokenAtStart
|
|
1234
|
-
? readApprovalSurface(config.accessFile)
|
|
1235
|
-
: ({
|
|
1236
|
-
kind: "missing",
|
|
1237
|
-
reason:
|
|
1238
|
-
`${TELEGRAM_APPROVAL_TOOL} unavailable on local ticks: omp-telegram had no bot token when this ` +
|
|
1239
|
-
"session started, so it bound none and its tools stay dead however complete the access file looks " +
|
|
1240
|
-
"now — run `/telegram on` in this session, or restart it, to rebind the bridge",
|
|
1241
|
-
} as const);
|
|
1242
1312
|
if (approval?.kind === "missing") {
|
|
1243
1313
|
content = `${content}\n${TICK_APPROVAL_UNAVAILABLE_RULE}`;
|
|
1244
1314
|
if (!session.approvalToolMissingLogged) {
|
|
@@ -1254,6 +1324,24 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1254
1324
|
}
|
|
1255
1325
|
}
|
|
1256
1326
|
|
|
1327
|
+
// The other half of transport truth, read from the same file at the same
|
|
1328
|
+
// moment: whether visible text stays out of the operator's chat. Appended
|
|
1329
|
+
// after the approval line because it is the weaker instruction of the two —
|
|
1330
|
+
// the approval rule stops an invented approval, this one asks the turn to keep
|
|
1331
|
+
// quiet — and a prompt should end on the sentence that must not be missed.
|
|
1332
|
+
//
|
|
1333
|
+
// Gated on `bridgeTokenAtStart` for the same reason the approval read is, and
|
|
1334
|
+
// the reasoning is the mirror image of it: with no token bound, omp-telegram
|
|
1335
|
+
// sends *nothing*, so the two relay paths this rule warns about cannot fire
|
|
1336
|
+
// either. A narration warning on a dead bridge would describe a hazard that
|
|
1337
|
+
// does not exist, on top of an approval warning that already names the one
|
|
1338
|
+
// command worth running (`/telegram on`) — two remedies on one prompt, and the
|
|
1339
|
+
// operator acts on neither. So the rule fires only where the leak is real: a
|
|
1340
|
+
// live bridge whose profile is not `daemon`.
|
|
1341
|
+
//
|
|
1342
|
+
// No `accessFile` means no fleet bridge to judge, exactly as above.
|
|
1343
|
+
if (profile?.kind === "interactive") content = `${content}\n${TICK_NARRATION_RULE}`;
|
|
1344
|
+
|
|
1257
1345
|
try {
|
|
1258
1346
|
pi.sendMessage(
|
|
1259
1347
|
{ customType: TICK_CUSTOM_TYPE, content, display: true, attribution: "user" },
|