agent-coord-mcp 0.26.5 → 0.26.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/prefix.js +64 -0
- package/dist/prefix.js.map +1 -0
- package/dist/server.js +32 -2
- package/dist/server.js.map +1 -1
- package/dist/tools/attention.js +73 -0
- package/dist/tools/attention.js.map +1 -0
- package/dist/tools/away.js +122 -0
- package/dist/tools/away.js.map +1 -0
- package/dist/tools/events.js +171 -0
- package/dist/tools/events.js.map +1 -0
- package/dist/tools/index.js +3 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/records.js +651 -0
- package/dist/tools/records.js.map +1 -0
- package/dist/tools/registry.js +45 -5
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/rotate.js +143 -0
- package/dist/tools/rotate.js.map +1 -0
- package/dist/tools/shared.js +9 -0
- package/dist/tools/shared.js.map +1 -1
- package/dist/tools/stall.js +294 -0
- package/dist/tools/stall.js.map +1 -0
- package/dist/tools/transport.js +92 -16
- package/dist/tools/transport.js.map +1 -1
- package/dist/tools/worktrees.js +294 -0
- package/dist/tools/worktrees.js.map +1 -0
- package/package.json +2 -2
- package/scripts/check-test-count.mjs +1 -1
- package/scripts/coord-attention-clock.mjs +122 -0
- package/scripts/coord-stall-clock.mjs +125 -0
- package/src/prefix.ts +72 -0
- package/src/server.ts +138 -3
- package/src/tools/attention.ts +91 -0
- package/src/tools/away.ts +121 -0
- package/src/tools/events.ts +199 -0
- package/src/tools/index.ts +3 -0
- package/src/tools/records.ts +747 -0
- package/src/tools/registry.ts +45 -5
- package/src/tools/rotate.ts +180 -0
- package/src/tools/shared.ts +24 -0
- package/src/tools/stall.ts +311 -0
- package/src/tools/transport.ts +99 -3
- package/src/tools/worktrees.ts +311 -0
package/src/tools/registry.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { detachAgentTool } from "./transport.js";
|
|
2
|
+
import { readAway, secondCoordinatorRefusal } from "./away.js";
|
|
2
3
|
import { randomUUID } from "node:crypto";
|
|
3
4
|
import { existsSync, openSync, watch } from "node:fs";
|
|
4
5
|
import { promises as fsp } from "node:fs";
|
|
@@ -60,6 +61,7 @@ import {
|
|
|
60
61
|
type Cursor,
|
|
61
62
|
type Source,
|
|
62
63
|
type TransportMarker,
|
|
64
|
+
roomFeedOf,
|
|
63
65
|
sourceFile,
|
|
64
66
|
getOffset,
|
|
65
67
|
setOffset,
|
|
@@ -123,6 +125,13 @@ export async function registerTool(args: { agentId: string; project?: string; ro
|
|
|
123
125
|
const roleUpdate = resolveRoleUpdate(args.agentId, before[args.agentId], args.role);
|
|
124
126
|
if (!roleUpdate.ok) return { ok: false as const, error: roleUpdate.error };
|
|
125
127
|
|
|
128
|
+
// 4.2 — a SECOND coordinator may not register while the first is away.
|
|
129
|
+
// Checked on the RESOLVED role, not the string the caller passed: the refusal
|
|
130
|
+
// has to see what the registry will actually record, or a spelling slips past
|
|
131
|
+
// the guard and lands as `coordinator` anyway.
|
|
132
|
+
const second = secondCoordinatorRefusal(readAway(), args.agentId, roleUpdate.roleId);
|
|
133
|
+
if (second) return { ok: false as const, error: second };
|
|
134
|
+
|
|
126
135
|
const reg = await updateJson<AgentRegistry>(AGENTS_FILE, {}, (current) => {
|
|
127
136
|
const now = Date.now();
|
|
128
137
|
const existing = current[args.agentId];
|
|
@@ -250,10 +259,29 @@ export async function listAgentsTool() {
|
|
|
250
259
|
|
|
251
260
|
const reg = await updateJson<AgentRegistry>(AGENTS_FILE, {}, (current) => {
|
|
252
261
|
for (const [id, entry] of Object.entries(current)) {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
262
|
+
// A LIVE TRANSPORT PROTECTS AN AGENT FROM EVICTION. IT DOES NOT STAMP IT.
|
|
263
|
+
//
|
|
264
|
+
// This loop used to write `entry.lastHeartbeat = now` here, and
|
|
265
|
+
// `agents.json` is ONE store shared by every fleet on the machine while
|
|
266
|
+
// `list_agents` takes no project argument — so a call from one fleet
|
|
267
|
+
// rewrote every other fleet's timestamps. Another fleet did not ask for
|
|
268
|
+
// that, cannot see it, and it is not ours to write.
|
|
269
|
+
//
|
|
270
|
+
// kit#105 removed the fabricated value from the RESPONSE; the WRITE
|
|
271
|
+
// stayed, so every consumer reading the file directly still saw
|
|
272
|
+
// freshness this call had invented.
|
|
273
|
+
//
|
|
274
|
+
// AND IT MASKED STALL DETECTION. `stall_check` reads `lastHeartbeat` to
|
|
275
|
+
// find agents that have gone quiet. Because any `list_agents` call
|
|
276
|
+
// refreshed every live-transport agent, that clock could never age past
|
|
277
|
+
// the threshold, so no-heartbeat could not fire for exactly the agents
|
|
278
|
+
// most likely to be stuck — alive, attached, and doing nothing.
|
|
279
|
+
//
|
|
280
|
+
// Nothing is lost: the pusher calls `heartbeat` every 60s
|
|
281
|
+
// (scripts/coord-pusher.mjs:181), so an attached agent has a REAL
|
|
282
|
+
// heartbeat. This stamp only ever overwrote a true value with a
|
|
283
|
+
// simultaneous one.
|
|
284
|
+
if (liveTransports.has(id)) continue;
|
|
257
285
|
if (now - entry.lastHeartbeat > EVICT_MS) {
|
|
258
286
|
evicted.push(id);
|
|
259
287
|
delete current[id];
|
|
@@ -300,7 +328,19 @@ export async function listAgentsTool() {
|
|
|
300
328
|
...heartbeatFields,
|
|
301
329
|
capabilities: merged.length > 0 ? merged : undefined,
|
|
302
330
|
transport: transport
|
|
303
|
-
? {
|
|
331
|
+
? {
|
|
332
|
+
kind: transport.transport,
|
|
333
|
+
tmuxTarget: transport.tmuxTarget,
|
|
334
|
+
pid: transport.pid,
|
|
335
|
+
// `attached` alone said nothing about WHAT is attached. A pusher
|
|
336
|
+
// started `--no-room` delivers DMs only, and its marker used to be
|
|
337
|
+
// indistinguishable from a full one — so an agent could sit with
|
|
338
|
+
// its room feed off while status read healthy (worker-2).
|
|
339
|
+
//
|
|
340
|
+
// Absent is UNKNOWN, never "on": an older pusher's marker cannot
|
|
341
|
+
// answer, and answering for it is how the original defect worked.
|
|
342
|
+
rooms: roomFeedOf(transport),
|
|
343
|
+
}
|
|
304
344
|
: undefined,
|
|
305
345
|
};
|
|
306
346
|
});
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* `rotate` — handover-to-self across a context clear (Phase 5 Task 5).
|
|
3
|
+
*
|
|
4
|
+
* THE PACKET IS A SNAPSHOT, AND A CONTEXT RESET IS PRECISELY WHEN NOBODY CAN
|
|
5
|
+
* CHECK IT. After `/clear` the agent has no memory to contradict the packet
|
|
6
|
+
* with: whatever it says becomes the world. So the packet is written from
|
|
7
|
+
* TOOLS AND `gh`, never from chat memory — memory is the one source that
|
|
8
|
+
* cannot be re-derived after the reset it is meant to survive — and on the
|
|
9
|
+
* far side it is RECONCILED against live state before any work is done.
|
|
10
|
+
*
|
|
11
|
+
* A packet that is merely READ on resume is a stale world restored with
|
|
12
|
+
* confidence. That is the failure this verb exists to prevent, so `missionHint`
|
|
13
|
+
* is named a HINT in the type and treated as one in code: it is the only field
|
|
14
|
+
* that cannot be verified, and it never gets to assert anything.
|
|
15
|
+
*/
|
|
16
|
+
import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
17
|
+
import { execFileSync } from "node:child_process";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
import { z } from "zod";
|
|
20
|
+
import { ROOT } from "../store.js";
|
|
21
|
+
|
|
22
|
+
/*
|
|
23
|
+
* 5.4 — JOB IDS ARE AN ALLOWLIST.
|
|
24
|
+
*
|
|
25
|
+
* `archive-done` is deliberately ABSENT and stays absent: it is an arithmetic
|
|
26
|
+
* job over DONE.md that belongs to Groundwork and an existing QUEUE item, and
|
|
27
|
+
* a rotation is the worst possible moment to run one. Rotation exists to carry
|
|
28
|
+
* state ACROSS a reset intact; a verb that also rewrites the records it is
|
|
29
|
+
* carrying cannot be checked afterwards by the agent that ran it.
|
|
30
|
+
*/
|
|
31
|
+
export const ROTATE_JOBS = ["reseed-only", "phase-boundary"] as const;
|
|
32
|
+
export type RotateJob = (typeof ROTATE_JOBS)[number];
|
|
33
|
+
|
|
34
|
+
export type OpenPr = { n: number; headRefOid: string };
|
|
35
|
+
export type RotatePacket = {
|
|
36
|
+
agentId: string;
|
|
37
|
+
job: RotateJob;
|
|
38
|
+
rooms: string[];
|
|
39
|
+
name: string;
|
|
40
|
+
/** A HINT, never an assertion — the one field nothing can verify. */
|
|
41
|
+
missionHint: string;
|
|
42
|
+
atSha: string;
|
|
43
|
+
openPrs: OpenPr[];
|
|
44
|
+
at: string;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const packetFile = (agentId: string) => path.join(ROOT, "rotate", `${agentId}.json`);
|
|
48
|
+
|
|
49
|
+
export type RepoFacts = { dirty: string; sha: string; openPrs: OpenPr[] };
|
|
50
|
+
const realFacts = (repo: string): RepoFacts => {
|
|
51
|
+
const git = (args: string[]) => execFileSync("git", args, { cwd: repo, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
|
|
52
|
+
const out = execFileSync("gh", ["pr", "list", "--state", "open", "--json", "number,headRefOid"], { cwd: repo, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
53
|
+
return {
|
|
54
|
+
dirty: git(["status", "--porcelain"]),
|
|
55
|
+
sha: git(["rev-parse", "HEAD"]),
|
|
56
|
+
openPrs: (JSON.parse(out) as { number: number; headRefOid: string }[]).map((p) => ({ n: p.number, headRefOid: p.headRefOid })),
|
|
57
|
+
};
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
export const rotateSchema = {
|
|
61
|
+
agentId: z.string().min(1),
|
|
62
|
+
job: z.string().min(1),
|
|
63
|
+
repo: z.string().optional(),
|
|
64
|
+
rooms: z.array(z.string()).optional(),
|
|
65
|
+
missionHint: z.string().optional(),
|
|
66
|
+
write: z.boolean().optional(),
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
export async function rotateTool(
|
|
70
|
+
args: { agentId: string; job: string; repo?: string; rooms?: string[]; missionHint?: string; write?: boolean },
|
|
71
|
+
facts: (repo: string) => RepoFacts = realFacts,
|
|
72
|
+
) {
|
|
73
|
+
const repo = args.repo ?? process.cwd();
|
|
74
|
+
|
|
75
|
+
// 5.4 — unknown job refused BY NAME, and `archive-done` gets its own reason
|
|
76
|
+
// so the refusal reads as a decision rather than a typo.
|
|
77
|
+
if (!(ROTATE_JOBS as readonly string[]).includes(args.job)) {
|
|
78
|
+
const extra = args.job === "archive-done" ? ` 'archive-done' is deliberately not a rotation job: it rewrites the records the rotation is carrying, and a reset is the one moment nobody can check the result. It belongs to Groundwork and its existing QUEUE item.` : "";
|
|
79
|
+
return { ok: false as const, error: `'${args.job}' is not a rotate job. Allowed: ${ROTATE_JOBS.join(", ")}.${extra}` };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
let f: RepoFacts;
|
|
83
|
+
try {
|
|
84
|
+
f = facts(repo);
|
|
85
|
+
} catch (e) {
|
|
86
|
+
return { ok: false as const, error: `could not read live state (${String((e as Error).message).split("\n")[0]}) — a packet built from anything but live state is chat memory with a filename.` };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// 5.2 — MID-SLICE DIRTY REFUSES THE CLEAR.
|
|
90
|
+
//
|
|
91
|
+
// Uncommitted work is the one thing a packet cannot carry: it is not in the
|
|
92
|
+
// repo, not on a branch, and not in any tool's answer, so after `/clear` no
|
|
93
|
+
// reconciliation can discover it ever existed. It does not get lost loudly —
|
|
94
|
+
// it gets lost silently, which is why this is a refusal and not a warning.
|
|
95
|
+
if (f.dirty) {
|
|
96
|
+
const n = f.dirty.split("\n").filter(Boolean).length;
|
|
97
|
+
return {
|
|
98
|
+
ok: false as const,
|
|
99
|
+
error: `${n} uncommitted change(s) — refusing to rotate mid-slice. Uncommitted work is the one thing a packet cannot carry: after the clear, nothing can discover it existed. Commit it or stash it deliberately, then rotate.`,
|
|
100
|
+
dirty: f.dirty.split("\n").filter(Boolean),
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const packet: RotatePacket = {
|
|
105
|
+
agentId: args.agentId,
|
|
106
|
+
job: args.job as RotateJob,
|
|
107
|
+
rooms: args.rooms ?? [],
|
|
108
|
+
name: args.agentId,
|
|
109
|
+
missionHint: args.missionHint ?? "",
|
|
110
|
+
atSha: f.sha,
|
|
111
|
+
openPrs: f.openPrs,
|
|
112
|
+
at: new Date().toISOString(),
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
if (!args.write) return { ok: true as const, written: false as const, packet, note: `packet built from live state; pass write:true to persist it before /clear.` };
|
|
116
|
+
mkdirSync(path.dirname(packetFile(args.agentId)), { recursive: true });
|
|
117
|
+
writeFileSync(packetFile(args.agentId), `${JSON.stringify(packet, null, 2)}\n`);
|
|
118
|
+
return { ok: true as const, written: true as const, packet, path: packetFile(args.agentId) };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/* ── 5.3 / 5.5 — reconcile before working, or refuse to work ───────────────── */
|
|
122
|
+
|
|
123
|
+
export type Divergence = { field: string; packet: string; live: string; note: string };
|
|
124
|
+
|
|
125
|
+
export const rotateReconcileSchema = { agentId: z.string().min(1), repo: z.string().optional() };
|
|
126
|
+
|
|
127
|
+
export async function rotateReconcileTool(
|
|
128
|
+
args: { agentId: string; repo?: string },
|
|
129
|
+
facts: (repo: string) => RepoFacts = realFacts,
|
|
130
|
+
) {
|
|
131
|
+
const repo = args.repo ?? process.cwd();
|
|
132
|
+
const f0 = packetFile(args.agentId);
|
|
133
|
+
if (!existsSync(f0)) return { ok: false as const, error: `no rotate packet for '${args.agentId}'. A reseeded agent with no packet has nothing to reconcile against and must not infer its state — ask for a GO.` };
|
|
134
|
+
|
|
135
|
+
let packet: RotatePacket;
|
|
136
|
+
try {
|
|
137
|
+
packet = JSON.parse(readFileSync(f0, "utf8")) as RotatePacket;
|
|
138
|
+
} catch (e) {
|
|
139
|
+
return { ok: false as const, error: `packet for '${args.agentId}' is unreadable (${String((e as Error).message).split("\n")[0]}) — an unparseable packet is not an empty one; do not proceed as if there were no prior state.` };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
let f: RepoFacts;
|
|
143
|
+
try {
|
|
144
|
+
f = facts(repo);
|
|
145
|
+
} catch (e) {
|
|
146
|
+
// NOT RECONCILED IS NOT RECONCILED-CLEAN.
|
|
147
|
+
return { ok: false as const, error: `could not read live state to reconcile (${String((e as Error).message).split("\n")[0]}) — NOT checked, which is not the same as checked and matching.`, packet };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const live = new Map(f.openPrs.map((p) => [p.n, p.headRefOid]));
|
|
151
|
+
const divergences: Divergence[] = [];
|
|
152
|
+
|
|
153
|
+
// 5.5 — THE PACKET'S OPEN PRs ARE CLAIMS, NOT FACTS.
|
|
154
|
+
//
|
|
155
|
+
// A PR that merged during the reset is the dangerous direction: the packet
|
|
156
|
+
// says "open", the agent resumes and keeps working a branch that is already
|
|
157
|
+
// in main, and every one of its next steps is coherent and wrong.
|
|
158
|
+
for (const p of packet.openPrs ?? []) {
|
|
159
|
+
if (!live.has(p.n)) divergences.push({ field: `pr#${p.n}`, packet: "open", live: "not open", note: `#${p.n} is no longer open — it merged or closed during the reset. Do NOT resume work on it as open.` });
|
|
160
|
+
else if (live.get(p.n) !== p.headRefOid) divergences.push({ field: `pr#${p.n}`, packet: p.headRefOid, live: String(live.get(p.n)), note: `#${p.n} advanced during the reset — the packet's head is stale; re-read before acting.` });
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
if (packet.atSha && f.sha !== packet.atSha)
|
|
164
|
+
divergences.push({ field: "atSha", packet: packet.atSha, live: f.sha, note: `the repo moved during the reset; anything the packet said about the tree may be stale.` });
|
|
165
|
+
|
|
166
|
+
// The verdict names its POPULATION, and `missionHint` is excluded from it on
|
|
167
|
+
// purpose: it is unverifiable, so counting it as "reconciled" would be a
|
|
168
|
+
// clean report over something never checked.
|
|
169
|
+
const verdict = {
|
|
170
|
+
reconciled: (packet.openPrs?.length ?? 0) + (packet.atSha ? 1 : 0),
|
|
171
|
+
divergences,
|
|
172
|
+
unverifiable: ["missionHint"],
|
|
173
|
+
missionHint: packet.missionHint,
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
if (divergences.length)
|
|
177
|
+
return { ok: false as const, error: `${divergences.length} divergence(s) between packet and live state — REFUSING to report ready. A packet read without reconciliation is a stale world restored with confidence.`, verdict, packet };
|
|
178
|
+
|
|
179
|
+
return { ok: true as const, ready: true as const, verdict, packet, note: `packet matches live state. 'missionHint' is a HINT and was NOT verified — treat it as a prompt, never as an instruction you have confirmed.` };
|
|
180
|
+
}
|
package/src/tools/shared.ts
CHANGED
|
@@ -97,8 +97,32 @@ export type TransportMarker = {
|
|
|
97
97
|
// it for a session restart + re-attach. Absent on markers written by older
|
|
98
98
|
// versions (treated as "unknown, skip", deliberately mirroring scriptMtime).
|
|
99
99
|
serverBuildMtime?: number;
|
|
100
|
+
// Does this transport carry ROOM traffic, or DMs only?
|
|
101
|
+
//
|
|
102
|
+
// A pusher started with `--no-room` delivers inbox messages and nothing
|
|
103
|
+
// else, and until this field existed the marker looked identical to a full
|
|
104
|
+
// one — so `status` said `attached: true` and an agent sat with its room
|
|
105
|
+
// feed off while every reading said healthy. That is how worker-2 missed
|
|
106
|
+
// its channel traffic.
|
|
107
|
+
//
|
|
108
|
+
// ABSENT MEANS UNKNOWN, NEVER "ON" — deliberately mirroring scriptMtime and
|
|
109
|
+
// serverBuildMtime above. A marker from an older pusher cannot tell us, and
|
|
110
|
+
// reporting an unasked question as full capability is the defect this field
|
|
111
|
+
// exists to remove.
|
|
112
|
+
rooms?: boolean;
|
|
100
113
|
};
|
|
101
114
|
|
|
115
|
+
/**
|
|
116
|
+
* How to REPORT a transport's room capability.
|
|
117
|
+
*
|
|
118
|
+
* Three states, not two. `undefined` is a marker written before the field
|
|
119
|
+
* existed: it cannot answer, and "unknown" is the only truthful report. The
|
|
120
|
+
* defect being fixed is precisely a two-state reading of a three-state world —
|
|
121
|
+
* `attached: true` covered "full", "DM-only", and "cannot say" alike.
|
|
122
|
+
*/
|
|
123
|
+
export const roomFeedOf = (m: { rooms?: boolean } | null | undefined): "on" | "off" | "unknown" =>
|
|
124
|
+
!m || m.rooms === undefined ? "unknown" : m.rooms ? "on" : "off";
|
|
125
|
+
|
|
102
126
|
export type AgentRegistry = Record<string, AgentEntry>;
|
|
103
127
|
|
|
104
128
|
export type {
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* `stall_check` — a predicate over the board and the bus, plus a HALT flag.
|
|
3
|
+
*
|
|
4
|
+
* Stall has been "notice the room" until now, which means it is noticed when
|
|
5
|
+
* somebody happens to look. The measured cost elsewhere: ~42 hours of dead merge
|
|
6
|
+
* path across four incidents in three days, with `main` advancing throughout so
|
|
7
|
+
* every commit-based liveness check read green.
|
|
8
|
+
*
|
|
9
|
+
* A MISS IS SILENT TO THE DUTY OFFICER AND NEVER SILENT TO THE RECORD.
|
|
10
|
+
*
|
|
11
|
+
* This is the whole design constraint (3.3b). "HIT DMs, MISS silent" answers the
|
|
12
|
+
* noise question and makes a DEAD CLOCK look exactly like a healthy fleet — the
|
|
13
|
+
* absence-read-as-evidence rule, inside the verb built to watch for absence. So
|
|
14
|
+
* every run leaves a mark, HIT or MISS, and "no alert" becomes distinguishable
|
|
15
|
+
* from "nothing ran". A check that only speaks when it fires cannot be told from
|
|
16
|
+
* a broken one.
|
|
17
|
+
*/
|
|
18
|
+
import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
19
|
+
import { execFileSync } from "node:child_process";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { z } from "zod";
|
|
22
|
+
import { parseWorkDoc, workstreamsV1RowsOf } from "@davidbalzan/groundwork-seam";
|
|
23
|
+
import { ROOT, AGENTS_FILE, readJson } from "../store.js";
|
|
24
|
+
import { loadLiveTransports } from "./registry.js";
|
|
25
|
+
|
|
26
|
+
const BOARD_DOC = "docs/WORKSTREAMS.md";
|
|
27
|
+
const STALL_MS = 30 * 60 * 1000;
|
|
28
|
+
|
|
29
|
+
const runFile = () => path.join(ROOT, "stall-check.json");
|
|
30
|
+
const haltFile = () => path.join(ROOT, "halt.json");
|
|
31
|
+
|
|
32
|
+
export type StallHit =
|
|
33
|
+
| { kind: "no-heartbeat"; agentId: string; stream: string; minutes: number }
|
|
34
|
+
| { kind: "no-vcs-activity"; agentId: string; branch: string; minutes: number };
|
|
35
|
+
|
|
36
|
+
// ---------- halt ----------
|
|
37
|
+
|
|
38
|
+
export const setHaltSchema = {
|
|
39
|
+
reason: z.string().min(1),
|
|
40
|
+
by: z.string().min(1),
|
|
41
|
+
clear: z.boolean().optional(),
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* A NAMED state, never a mood.
|
|
46
|
+
*
|
|
47
|
+
* "Production feels down" is not a halt: a halt blocks every claim in the fleet,
|
|
48
|
+
* so the thing that sets it must be nameable and therefore arguable — a board
|
|
49
|
+
* cutover, a cited `BLOCKER:`, a documented red pipeline. The reason is required
|
|
50
|
+
* for that reason, not for the log.
|
|
51
|
+
*/
|
|
52
|
+
export async function setHaltTool(args: { reason: string; by: string; clear?: boolean }) {
|
|
53
|
+
mkdirSync(ROOT, { recursive: true });
|
|
54
|
+
if (args.clear) {
|
|
55
|
+
writeFileSync(haltFile(), JSON.stringify({ halted: false, clearedBy: args.by, clearedAt: Date.now(), lastReason: args.reason }, null, 2));
|
|
56
|
+
return { ok: true as const, halted: false, clearedBy: args.by };
|
|
57
|
+
}
|
|
58
|
+
const state = { halted: true, reason: args.reason, by: args.by, at: Date.now() };
|
|
59
|
+
writeFileSync(haltFile(), JSON.stringify(state, null, 2));
|
|
60
|
+
return { ok: true as const, ...state };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function haltState(): { halted: boolean; reason?: string; by?: string; at?: number } {
|
|
64
|
+
try {
|
|
65
|
+
const raw = JSON.parse(readFileSync(haltFile(), "utf8"));
|
|
66
|
+
return raw?.halted ? raw : { halted: false };
|
|
67
|
+
} catch {
|
|
68
|
+
return { halted: false };
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// ---------- the run mark ----------
|
|
73
|
+
|
|
74
|
+
/** Every run leaves this, HIT or MISS. It is what makes a dead clock visible. */
|
|
75
|
+
/**
|
|
76
|
+
* Record a run that FAILED.
|
|
77
|
+
*
|
|
78
|
+
* Acceptance from the queue item, and the clause most easily skipped: "a
|
|
79
|
+
* scheduled check reports its FETCH FAILURES, or a broken check is
|
|
80
|
+
* indistinguishable from a quiet registry". Without this, a clock that fires
|
|
81
|
+
* every 30 minutes and throws every time leaves NO marks at all — identical on
|
|
82
|
+
* disk to a clock that was never installed.
|
|
83
|
+
*/
|
|
84
|
+
export function markRunFailure(reason: string): void {
|
|
85
|
+
mkdirSync(ROOT, { recursive: true });
|
|
86
|
+
let history: RunMark[] = [];
|
|
87
|
+
try {
|
|
88
|
+
history = JSON.parse(readFileSync(runFile(), "utf8")).history ?? [];
|
|
89
|
+
} catch {
|
|
90
|
+
/* first run */
|
|
91
|
+
}
|
|
92
|
+
history.push({ at: Date.now(), hits: 0, checked: 0, failed: reason });
|
|
93
|
+
writeFileSync(runFile(), JSON.stringify({ history: history.slice(-200) }, null, 2));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
type RunMark = { at: number; hits: number; checked: number; failed?: string };
|
|
97
|
+
|
|
98
|
+
function markRun(result: { hits: StallHit[]; checked: number }) {
|
|
99
|
+
mkdirSync(ROOT, { recursive: true });
|
|
100
|
+
let history: RunMark[] = [];
|
|
101
|
+
try {
|
|
102
|
+
history = JSON.parse(readFileSync(runFile(), "utf8")).history ?? [];
|
|
103
|
+
} catch {
|
|
104
|
+
/* first run */
|
|
105
|
+
}
|
|
106
|
+
history.push({ at: Date.now(), hits: result.hits.length, checked: result.checked });
|
|
107
|
+
// A RUN OF MISSES MUST BE VISIBLE AS RUNS, not as absence — so the marks are a
|
|
108
|
+
// list, not a single timestamp. "Ten quiet checks" and "one check ten hours ago"
|
|
109
|
+
// are different states and only the first is a healthy fleet.
|
|
110
|
+
writeFileSync(runFile(), JSON.stringify({ history: history.slice(-200) }, null, 2));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export const lastRanSchema = { maxAgeMinutes: z.number().optional() };
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Is the clock alive? Readable by a human or another check, which is the point —
|
|
117
|
+
* MISS is silent to the duty officer, not to the record.
|
|
118
|
+
*/
|
|
119
|
+
export async function stallClockStatusTool(args: { maxAgeMinutes?: number }) {
|
|
120
|
+
const maxAge = (args.maxAgeMinutes ?? 60) * 60 * 1000;
|
|
121
|
+
let history: RunMark[] = [];
|
|
122
|
+
try {
|
|
123
|
+
history = JSON.parse(readFileSync(runFile(), "utf8")).history ?? [];
|
|
124
|
+
} catch {
|
|
125
|
+
return {
|
|
126
|
+
ok: false as const,
|
|
127
|
+
error:
|
|
128
|
+
"stall_check has NEVER run — no run mark exists. That is not a quiet fleet, it is an unwatched one: a check that only speaks when it fires cannot be told from a broken one.",
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
const last = history[history.length - 1];
|
|
132
|
+
// A RUN THAT FAILED IS NOT A RUN THAT PASSED. Without this the clock reads
|
|
133
|
+
// "fresh" off a mark it wrote while erroring — the age is honest and the
|
|
134
|
+
// health is not, which is the same shape as a fresh heartbeat from a stuck
|
|
135
|
+
// agent.
|
|
136
|
+
const failures = history.filter((h) => h.failed);
|
|
137
|
+
const lastFailed = last?.failed;
|
|
138
|
+
const age = Date.now() - (last?.at ?? 0);
|
|
139
|
+
// `>=`, NOT `>`, AND THE DIFFERENCE IS A REAL RACE RATHER THAN PEDANTRY.
|
|
140
|
+
//
|
|
141
|
+
// With `>`, a window of 0 and a mark written in the SAME MILLISECOND gives
|
|
142
|
+
// `0 > 0` = false: the clock reads FRESH at the instant it was asked to treat
|
|
143
|
+
// everything as stale. It passed locally and on one CI run and failed on
|
|
144
|
+
// another — green in two environments, red in one — because it depended on at
|
|
145
|
+
// least a millisecond elapsing.
|
|
146
|
+
//
|
|
147
|
+
// The window having ELAPSED is the condition, so equality is inside it: a
|
|
148
|
+
// 0-minute window means nothing is ever fresh, which is what a caller asking
|
|
149
|
+
// for one means.
|
|
150
|
+
const stale = age >= maxAge;
|
|
151
|
+
const misses = history.filter((h) => !h.failed && h.hits === 0).length;
|
|
152
|
+
const hits = history.filter((h) => !h.failed && h.hits > 0).length;
|
|
153
|
+
return {
|
|
154
|
+
// A FAILING CLOCK IS NOT A HEALTHY ONE. It writes marks on schedule, so the
|
|
155
|
+
// age looks fresh while nothing is being measured — a fresh heartbeat from
|
|
156
|
+
// a stuck agent, one level up.
|
|
157
|
+
ok: !stale && !lastFailed,
|
|
158
|
+
...(stale
|
|
159
|
+
? {
|
|
160
|
+
error: `stall_check last ran ${Math.round(age / 60000)}m ago, past the ${args.maxAgeMinutes ?? 60}m window — THE CLOCK IS STOPPED. No alerts is not the same as no stalls.`,
|
|
161
|
+
}
|
|
162
|
+
: lastFailed
|
|
163
|
+
? {
|
|
164
|
+
error: `the clock is RUNNING but its last run FAILED: ${lastFailed}. It is firing on schedule and measuring nothing, which reads as fresh and is not.`,
|
|
165
|
+
}
|
|
166
|
+
: {}),
|
|
167
|
+
lastRanMinutesAgo: Math.round(age / 60000),
|
|
168
|
+
runs: history.length,
|
|
169
|
+
misses,
|
|
170
|
+
hits,
|
|
171
|
+
failures: failures.length,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// ---------- the predicate ----------
|
|
176
|
+
|
|
177
|
+
export const stallCheckSchema = { repo: z.string().optional(), stallMinutes: z.number().optional() };
|
|
178
|
+
|
|
179
|
+
export async function stallCheckTool(args: { repo?: string; stallMinutes?: number }) {
|
|
180
|
+
const repo = args.repo ?? process.cwd();
|
|
181
|
+
const limit = (args.stallMinutes ?? 30) * 60 * 1000;
|
|
182
|
+
const board = path.join(repo, BOARD_DOC);
|
|
183
|
+
if (!existsSync(board)) return { ok: false as const, error: `no ${BOARD_DOC} under '${repo}'` };
|
|
184
|
+
|
|
185
|
+
const liveTransports = await loadLiveTransports();
|
|
186
|
+
const rows = workstreamsV1RowsOf(parseWorkDoc(readFileSync(board, "utf8")));
|
|
187
|
+
const inFlight = rows.filter((r) => /🚧/.test(r.status));
|
|
188
|
+
const reg = await readJson<Record<string, { lastHeartbeat: number }>>(AGENTS_FILE, {});
|
|
189
|
+
const now = Date.now();
|
|
190
|
+
const hits: StallHit[] = [];
|
|
191
|
+
/** Rows whose VCS activity could not be measured. NOT stall claims. */
|
|
192
|
+
const unmeasurable: { agentId: string; value: string; why: string }[] = [];
|
|
193
|
+
|
|
194
|
+
for (const row of inFlight) {
|
|
195
|
+
const agentId = row.owner.replace(/[`*]/g, "").trim();
|
|
196
|
+
const entry = reg[agentId];
|
|
197
|
+
if (entry) {
|
|
198
|
+
const age = now - entry.lastHeartbeat;
|
|
199
|
+
// A HEARTBEAT IS ONLY EVIDENCE WHERE SOMETHING WRITES ONE.
|
|
200
|
+
//
|
|
201
|
+
// `heartbeat` is called by the PUSHER, never by the agent. Only
|
|
202
|
+
// `coord-pusher.mjs` (the REMOTE pusher) calls it, and it must: a remote
|
|
203
|
+
// marker cannot be pid-probed across machines, so its liveness IS the
|
|
204
|
+
// heartbeat. `hooks/tmux-pusher.mjs` — what this fleet actually runs —
|
|
205
|
+
// never calls it, because a LOCAL marker's liveness is `isPidAlive`.
|
|
206
|
+
//
|
|
207
|
+
// So for a local transport there is no heartbeat SOURCE at all. Before
|
|
208
|
+
// kit#137, `list_agents` stamped these agents and that fabrication was
|
|
209
|
+
// the only thing keeping the field moving; removing it left the field
|
|
210
|
+
// honest and empty. Measured: three attached, working agents at an
|
|
211
|
+
// IDENTICAL 44.7m — the uniform signature of one shared cause, not three
|
|
212
|
+
// stalls.
|
|
213
|
+
//
|
|
214
|
+
// WHY NOT JUST MAKE tmux-pusher HEARTBEAT: because the signal would mean
|
|
215
|
+
// "the pusher process is alive", which `isPidAlive` already answers for
|
|
216
|
+
// local markers. During the 17-hour stall every transport was live the
|
|
217
|
+
// whole time, so a pusher heartbeat would have read FRESH for all 17
|
|
218
|
+
// hours. It would restore a field without restoring a detector — and the
|
|
219
|
+
// case this verb exists for is exactly the one it would miss. The vcs
|
|
220
|
+
// half is what catches "alive and not progressing"; saying so is more
|
|
221
|
+
// honest than a green field.
|
|
222
|
+
//
|
|
223
|
+
// Unknown is not stalled (kit#138). This does NOT narrow Task 3.5: an
|
|
224
|
+
// agent with no transport, or a REMOTE one where the heartbeat genuinely
|
|
225
|
+
// is the liveness mechanism, still HITs on a dead heartbeat.
|
|
226
|
+
const marker = liveTransports.get(agentId);
|
|
227
|
+
if (age > limit && marker && marker.transport === "tmux-push") {
|
|
228
|
+
unmeasurable.push({
|
|
229
|
+
agentId,
|
|
230
|
+
value: marker.transport,
|
|
231
|
+
why: `heartbeat is ${Math.round(age / 60000)}m old, but nothing writes heartbeats for a local 'tmux-push' transport — hooks/tmux-pusher.mjs does not call heartbeat, and this marker's liveness is its pid. There is no heartbeat SOURCE here, so the age measures nothing about this agent`,
|
|
232
|
+
});
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
if (age > limit) {
|
|
236
|
+
hits.push({ kind: "no-heartbeat", agentId, stream: row.stream.slice(0, 60), minutes: Math.round(age / 60000) });
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
// A FRESH HEARTBEAT IS NOT PROGRESS. An agent can be alive and stuck, which is
|
|
241
|
+
// the case "notice the room" never catches: the pane is responsive, so nobody
|
|
242
|
+
// looks. Ask the branch instead.
|
|
243
|
+
//
|
|
244
|
+
// RESOLVE A REF, AND SAY SO WHEN THE VALUE IS NOT ONE.
|
|
245
|
+
//
|
|
246
|
+
// `git log -1 <value>` accepts a PATHSPEC exactly as readily as a ref, and
|
|
247
|
+
// the old guard only required a `/` — which every path has. So the board's
|
|
248
|
+
// `Branch · Worktree` cells, which hold PATHS, were fed to git and silently
|
|
249
|
+
// measured as paths: the aide was reported at 2,271 minutes, the age of
|
|
250
|
+
// `docs/phases/phase5`'s last commit, not of any activity by that agent.
|
|
251
|
+
// Verified: `docs/phases/phase5` does not resolve as a ref, yet
|
|
252
|
+
// `git log -1 --format=%cI docs/phases/phase5` returns a date.
|
|
253
|
+
//
|
|
254
|
+
// Other rows read plausibly only by coincidence — a path that happens to be
|
|
255
|
+
// committed often looks like an active branch.
|
|
256
|
+
//
|
|
257
|
+
// UNKNOWN IS NOT STALLED, which this verb already gets right for
|
|
258
|
+
// heartbeats. An unresolvable value is REPORTED as unmeasurable rather than
|
|
259
|
+
// skipped in silence: a silent `continue` and a healthy agent produce the
|
|
260
|
+
// same output, which is the failure this whole verb exists to avoid.
|
|
261
|
+
const raw = (row.branchWorktree.match(/`([^`]+)`/)?.[1] ?? "").trim();
|
|
262
|
+
if (!raw) {
|
|
263
|
+
unmeasurable.push({ agentId, value: "", why: "no value in the Branch · Worktree cell" });
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
let sha = "";
|
|
267
|
+
try {
|
|
268
|
+
sha = execFileSync("git", ["rev-parse", "--verify", "--quiet", `${raw}^{commit}`], {
|
|
269
|
+
cwd: repo, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"],
|
|
270
|
+
}).trim();
|
|
271
|
+
} catch {
|
|
272
|
+
sha = "";
|
|
273
|
+
}
|
|
274
|
+
if (!sha) {
|
|
275
|
+
unmeasurable.push({
|
|
276
|
+
agentId,
|
|
277
|
+
value: raw,
|
|
278
|
+
why: `'${raw}' does not resolve as a git ref — it is a path or glob, and \`git log <path>\` would silently report that PATH's last commit as this agent's activity`,
|
|
279
|
+
});
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
try {
|
|
283
|
+
// The resolved SHA, and `--`: a ref can then never be re-read as a
|
|
284
|
+
// pathspec, which is the ambiguity that produced the wrong number.
|
|
285
|
+
const iso = execFileSync("git", ["log", "-1", "--format=%cI", sha, "--"], {
|
|
286
|
+
cwd: repo, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"],
|
|
287
|
+
}).trim();
|
|
288
|
+
const age = now - Date.parse(iso);
|
|
289
|
+
if (age > limit) {
|
|
290
|
+
hits.push({ kind: "no-vcs-activity", agentId, branch: raw, minutes: Math.round(age / 60000) });
|
|
291
|
+
}
|
|
292
|
+
} catch {
|
|
293
|
+
unmeasurable.push({ agentId, value: raw, why: "resolved as a ref but its log could not be read" });
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const result = { hits, checked: inFlight.length, unmeasurable };
|
|
298
|
+
markRun(result);
|
|
299
|
+
return {
|
|
300
|
+
ok: true as const,
|
|
301
|
+
...result,
|
|
302
|
+
halted: haltState().halted,
|
|
303
|
+
// MISS is silent to the DUTY OFFICER — the caller decides whether to DM — and
|
|
304
|
+
// never silent to the record, which markRun just wrote.
|
|
305
|
+
dm: hits.length > 0,
|
|
306
|
+
note:
|
|
307
|
+
hits.length === 0
|
|
308
|
+
? `MISS — ${inFlight.length} in-flight row(s), none stalled${unmeasurable.length ? `; ${unmeasurable.length} row(s) UNMEASURABLE for VCS activity (${unmeasurable.map((u) => u.agentId).join(", ")}) — reported, not counted as healthy` : ""}. No DM. The run IS recorded: read it with stall_clock_status, because no alert and nothing running look identical from here.`
|
|
309
|
+
: undefined,
|
|
310
|
+
};
|
|
311
|
+
}
|