@mergeline/agent 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -3
- package/dist/cli.js +57 -13
- package/dist/engines.js +126 -0
- package/dist/repo.js +3 -1
- package/dist/service.js +29 -4
- package/dist/watch.js +115 -27
- package/dist/worker.js +1098 -0
- package/engine/prompt.md +58 -0
- package/package.json +2 -1
package/dist/worker.js
ADDED
|
@@ -0,0 +1,1098 @@
|
|
|
1
|
+
// `board watch --engine <name>`: the worker. It owns the board's protocol and
|
|
2
|
+
// the engine (engines.ts) only writes code (round 8, D1 to D6):
|
|
3
|
+
//
|
|
4
|
+
// claim -> the spec from origin/main -> a fresh worktree at ../.wt/<ticket>
|
|
5
|
+
// -> .mergeline-run/ (the ticket in, the result out) -> the engine, fenced,
|
|
6
|
+
// while the worker heartbeats -> commit what it left, the `test` command,
|
|
7
|
+
// push, the PR -> submit, ask or release -> remove the worktree and branch.
|
|
8
|
+
//
|
|
9
|
+
// The engine never holds the agent key and cannot push: its environment has
|
|
10
|
+
// no BOARD_*, no GitHub token, no SSH agent, empty board and gh config
|
|
11
|
+
// folders, and a push URL for origin that does not exist. Every file it
|
|
12
|
+
// leaves is untrusted text. What this does not stop (the same OS user) is
|
|
13
|
+
// in docs/security.md.
|
|
14
|
+
//
|
|
15
|
+
// One worker per agent and machine (the lock), one board session for its
|
|
16
|
+
// whole life. It stops when the queue is empty (unless --follow), after
|
|
17
|
+
// --max-runs claims that got a ticket, or after 3 failed runs (releases) in a
|
|
18
|
+
// row (exit 1). After a failed run it waits --interval before claiming again.
|
|
19
|
+
// A paused board, or one it cannot reach after its first contact, is waited
|
|
20
|
+
// out (watch.ts, untilAnswered); during a call that ends a run only a second
|
|
21
|
+
// signal ends that wait, and the ticket is then left In progress.
|
|
22
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
23
|
+
import { createHash } from "node:crypto";
|
|
24
|
+
import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readlinkSync, readSync, rmSync, statSync, writeFileSync, } from "node:fs";
|
|
25
|
+
import { homedir, tmpdir } from "node:os";
|
|
26
|
+
import { join, resolve } from "node:path";
|
|
27
|
+
import { BoardError, BoardPaused } from "./service.js";
|
|
28
|
+
import { hasWork, isUnreachable, label, sleep, takeLock, untilAnswered } from "./watch.js";
|
|
29
|
+
const FAILURE_STOP = 3;
|
|
30
|
+
const RUN_DIR = ".mergeline-run";
|
|
31
|
+
const RESULT_MAX = 64 * 1024;
|
|
32
|
+
const POLL_MS = 5000;
|
|
33
|
+
const NOTE_GAP_MS = 30_000;
|
|
34
|
+
const NOTE_CHARS = 500;
|
|
35
|
+
const PLAN_CHARS = 2000;
|
|
36
|
+
/** The most read from plan.md, and from progress.log at each poll. */
|
|
37
|
+
const READ_MAX = 64 * 1024;
|
|
38
|
+
/** How much of a command's output the worker keeps for a question. */
|
|
39
|
+
const OUTPUT_KEEP = 256 * 1024;
|
|
40
|
+
const KILL_GRACE_MS = 2000;
|
|
41
|
+
/** The board's answers to a heartbeat that mean the ticket is no longer this session's. */
|
|
42
|
+
const REFUSALS = [403, 404, 409];
|
|
43
|
+
const FENCED_VARS = ["GH_TOKEN", "GITHUB_TOKEN", "GH_ENTERPRISE_TOKEN", "GITHUB_ENTERPRISE_TOKEN", "SSH_AUTH_SOCK"];
|
|
44
|
+
const clip = (s, n) => (s.length > n ? s.slice(0, n) : s);
|
|
45
|
+
/** Reads .mergeline-run/result.json; anything but one of the three forms is no result, with why. */
|
|
46
|
+
export function readResult(file) {
|
|
47
|
+
const st = lstatOrNull(file);
|
|
48
|
+
if (!st)
|
|
49
|
+
return { outcome: "none", why: "no result.json" };
|
|
50
|
+
if (!st.isFile())
|
|
51
|
+
return { outcome: "none", why: "result.json is not a regular file" };
|
|
52
|
+
if (st.size > RESULT_MAX)
|
|
53
|
+
return { outcome: "none", why: "result.json is over 64 KB" };
|
|
54
|
+
let data;
|
|
55
|
+
try {
|
|
56
|
+
data = JSON.parse(readFileSync(file, "utf8"));
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return { outcome: "none", why: "result.json is not valid JSON" };
|
|
60
|
+
}
|
|
61
|
+
if (!data || typeof data !== "object" || Array.isArray(data))
|
|
62
|
+
return { outcome: "none", why: "result.json is not a JSON object" };
|
|
63
|
+
const r = data;
|
|
64
|
+
const field = { done: "summary", ask: "question", release: "reason" };
|
|
65
|
+
const max = { done: 2000, ask: 5000, release: 5000 };
|
|
66
|
+
const outcome = r.outcome;
|
|
67
|
+
if (typeof outcome !== "string" || !(outcome in field)) {
|
|
68
|
+
return { outcome: "none", why: `unknown outcome ${JSON.stringify(outcome ?? null)}` };
|
|
69
|
+
}
|
|
70
|
+
const o = outcome;
|
|
71
|
+
const text = r[field[o]];
|
|
72
|
+
if (typeof text !== "string" || !text.trim())
|
|
73
|
+
return { outcome: "none", why: `"${field[o]}" must be a non-empty string` };
|
|
74
|
+
const value = clip(text.trim(), max[o]);
|
|
75
|
+
if (o === "done")
|
|
76
|
+
return { outcome: "done", summary: value };
|
|
77
|
+
if (o === "ask")
|
|
78
|
+
return { outcome: "ask", question: value };
|
|
79
|
+
return { outcome: "release", reason: value };
|
|
80
|
+
}
|
|
81
|
+
/** A subject's most characters, the ticket's label included. */
|
|
82
|
+
const SUBJECT_MAX = 72;
|
|
83
|
+
/** The engine's text without control characters; the body keeps its line breaks (a CRLF is one). */
|
|
84
|
+
function engineText(text, keepLines) {
|
|
85
|
+
const t = text.replace(/\r\n/g, "\n");
|
|
86
|
+
return (keepLines ? t.replace(/[\u0000-\u0009\u000b-\u001f\u007f-\u009f]/g, "") : t.replace(/[\u0000-\u001f\u007f-\u009f]/g, "")).trim();
|
|
87
|
+
}
|
|
88
|
+
/** The first sentence of the summary's first line, as a subject after `<label>: `, cut to SUBJECT_MAX in all. */
|
|
89
|
+
function firstSentence(summary, room) {
|
|
90
|
+
const line = engineText(summary.split("\n")[0] ?? "", false);
|
|
91
|
+
const end = /[.!?] /.exec(line);
|
|
92
|
+
const sentence = (end ? line.slice(0, end.index + 1) : line).trim();
|
|
93
|
+
const chars = Array.from(sentence);
|
|
94
|
+
if (chars.length <= room)
|
|
95
|
+
return sentence;
|
|
96
|
+
const head = chars.slice(0, room - 1).join("");
|
|
97
|
+
const space = chars[room - 1] === " " ? head.length : head.lastIndexOf(" ");
|
|
98
|
+
return (space > 0 ? head.slice(0, space) : head).trimEnd() + "…";
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* The worker's commit of what the engine left uncommitted (R3.4): a subject from the engine's summary (done) or a
|
|
102
|
+
* fixed one before a question (ask), and a body with the engine's text and who committed it.
|
|
103
|
+
*/
|
|
104
|
+
export function commitMessage(label, engine, result) {
|
|
105
|
+
const by = `Committed by the worker: ${engine} left this uncommitted.`;
|
|
106
|
+
if (result.outcome === "ask") {
|
|
107
|
+
const question = engineText(result.question, true);
|
|
108
|
+
return { subject: `${label}: work so far, before a question`, body: `${question ? `Asked on the board: ${question}\n\n` : ""}${by}` };
|
|
109
|
+
}
|
|
110
|
+
const prefix = `${label}: `;
|
|
111
|
+
const sentence = firstSentence(result.summary, SUBJECT_MAX - Array.from(prefix).length);
|
|
112
|
+
const summary = engineText(result.summary, true);
|
|
113
|
+
return {
|
|
114
|
+
subject: sentence ? prefix + sentence : `${label}: work the engine left uncommitted`,
|
|
115
|
+
body: `${summary ? `${summary}\n\n` : ""}${by}`,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
/** origin's URLs in the repository at cwd (R3.2): raw config values and git's own reading of them. */
|
|
119
|
+
export function originUrls(cwd, env = withoutBoard(process.env)) {
|
|
120
|
+
const lines = (...args) => {
|
|
121
|
+
const r = spawnSync("git", args, { cwd, env, encoding: "utf8" });
|
|
122
|
+
return r.status === 0 ? r.stdout.split("\n").map((l) => l.trim()).filter(Boolean) : [];
|
|
123
|
+
};
|
|
124
|
+
const pushurls = lines("config", "--get-all", "remote.origin.pushurl");
|
|
125
|
+
const urls = [...lines("config", "--get-all", "remote.origin.url"), ...lines("remote", "get-url", "--all", "origin")];
|
|
126
|
+
const resolved = lines("remote", "get-url", "--push", "--all", "origin");
|
|
127
|
+
// With an explicit pushurl, get-url --push gives the pushurls (as rewritten); without one, the fetch URLs.
|
|
128
|
+
return pushurls.length
|
|
129
|
+
? { pushurls: [...new Set([...pushurls, ...resolved])], urls: [...new Set(urls)] }
|
|
130
|
+
: { pushurls: [], urls: [...new Set([...urls, ...resolved])] };
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* The engine's environment: the worker's, fenced (D2, D13). Pushes to origin's URLs are rewritten to a path
|
|
134
|
+
* that does not exist (R3.2): `url.<nowhere>.insteadOf` for each explicit pushurl (git ignores pushInsteadOf
|
|
135
|
+
* for a remote that has one), `url.<nowhere>.pushInsteadOf` for each fetch URL, so the engine can still fetch.
|
|
136
|
+
* A push to any other URL, an `origin` of another repository included, is not touched.
|
|
137
|
+
*/
|
|
138
|
+
export function fencedEnv(base, emptyDir, ghDir, origin) {
|
|
139
|
+
const env = {};
|
|
140
|
+
for (const [k, v] of Object.entries(base)) {
|
|
141
|
+
if (k.startsWith("BOARD_") || FENCED_VARS.includes(k))
|
|
142
|
+
continue;
|
|
143
|
+
env[k] = v;
|
|
144
|
+
}
|
|
145
|
+
env.BOARD_CONFIG_DIR = emptyDir;
|
|
146
|
+
env.BOARD_STATE_DIR = emptyDir;
|
|
147
|
+
env.GH_CONFIG_DIR = ghDir;
|
|
148
|
+
const nowhere = join(emptyDir, "no-such-remote");
|
|
149
|
+
const entries = [
|
|
150
|
+
...origin.pushurls.map((u) => [`url.${nowhere}.insteadOf`, u]),
|
|
151
|
+
...origin.urls.map((u) => [`url.${nowhere}.pushInsteadOf`, u]),
|
|
152
|
+
["credential.helper", ""],
|
|
153
|
+
];
|
|
154
|
+
let n = Number(env.GIT_CONFIG_COUNT) > 0 ? Number(env.GIT_CONFIG_COUNT) : 0;
|
|
155
|
+
for (const [key, value] of entries) {
|
|
156
|
+
env[`GIT_CONFIG_KEY_${n}`] = key;
|
|
157
|
+
env[`GIT_CONFIG_VALUE_${n}`] = value;
|
|
158
|
+
n++;
|
|
159
|
+
}
|
|
160
|
+
env.GIT_CONFIG_COUNT = String(n);
|
|
161
|
+
return env;
|
|
162
|
+
}
|
|
163
|
+
/** The worker's environment minus BOARD_*, for its own git and gh. */
|
|
164
|
+
function withoutBoard(base) {
|
|
165
|
+
return Object.fromEntries(Object.entries(base).filter(([k]) => !k.startsWith("BOARD_")));
|
|
166
|
+
}
|
|
167
|
+
/** The worker's main checkout: its top folder, or why it cannot run there. */
|
|
168
|
+
export function mainCheckout(cwd) {
|
|
169
|
+
const out = spawnSyncText("git", ["rev-parse", "--path-format=absolute", "--show-toplevel", "--git-dir", "--git-common-dir"], cwd);
|
|
170
|
+
if (out === null)
|
|
171
|
+
return { error: "board watch --engine runs in a git checkout of the repository; this folder is not one" };
|
|
172
|
+
const [top, gitDir, common] = out.split("\n");
|
|
173
|
+
if (!top || !gitDir || !common)
|
|
174
|
+
return { error: "board watch --engine runs in a git checkout of the repository; this folder is not one" };
|
|
175
|
+
if (resolve(gitDir) !== resolve(common)) {
|
|
176
|
+
return { error: `board watch --engine runs in the repository's main checkout, not in a linked worktree (this is one of ${resolve(common, "..")})` };
|
|
177
|
+
}
|
|
178
|
+
if (spawnSyncText("git", ["remote", "get-url", "origin"], top) === null) {
|
|
179
|
+
return { error: "board watch --engine needs a git checkout with an origin remote; this one has none" };
|
|
180
|
+
}
|
|
181
|
+
return { repo: top };
|
|
182
|
+
}
|
|
183
|
+
function spawnSyncText(cmd, args, cwd) {
|
|
184
|
+
const r = spawnSync(cmd, args, { cwd, encoding: "utf8" });
|
|
185
|
+
return r.status === 0 ? r.stdout.trim() : null;
|
|
186
|
+
}
|
|
187
|
+
/** Thrown when the board refused a heartbeat: the run stops with no other call for the ticket. */
|
|
188
|
+
class Refused extends Error {
|
|
189
|
+
}
|
|
190
|
+
/** A second signal ended the wait for a call that ends a run: the ticket stays In progress. */
|
|
191
|
+
class Abandoned extends Error {
|
|
192
|
+
}
|
|
193
|
+
/** Thrown when a git config file changed after the engine started: no more git, and the worker stops. */
|
|
194
|
+
class ConfigChanged extends Error {
|
|
195
|
+
file;
|
|
196
|
+
what;
|
|
197
|
+
constructor(file,
|
|
198
|
+
/** What changed, by key and without values (they can hold tokens). */
|
|
199
|
+
what) {
|
|
200
|
+
super(`${file} changed during the run`);
|
|
201
|
+
this.file = file;
|
|
202
|
+
this.what = what;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
/** The most keys a config change names. */
|
|
206
|
+
const KEYS_NAMED = 10;
|
|
207
|
+
/**
|
|
208
|
+
* A config file's entries, read with `git config --file <file> --list` (which reads the file and runs nothing).
|
|
209
|
+
* A missing file has none; never a FIFO or a device.
|
|
210
|
+
*/
|
|
211
|
+
export function configEntries(file, env = process.env) {
|
|
212
|
+
const l = lstatOrNull(file);
|
|
213
|
+
if (!l)
|
|
214
|
+
return new Map();
|
|
215
|
+
const st = l.isSymbolicLink() ? statOrNull(file) : l;
|
|
216
|
+
if (!st?.isFile())
|
|
217
|
+
return "replaced";
|
|
218
|
+
const r = spawnSync("git", ["config", "--file", file, "--list", "-z"], {
|
|
219
|
+
cwd: tmpdir(),
|
|
220
|
+
encoding: "utf8",
|
|
221
|
+
env: withoutBoard(env),
|
|
222
|
+
timeout: 10_000,
|
|
223
|
+
});
|
|
224
|
+
if (r.status !== 0)
|
|
225
|
+
return "unreadable";
|
|
226
|
+
const entries = new Map();
|
|
227
|
+
for (const item of r.stdout.split("\0")) {
|
|
228
|
+
if (!item)
|
|
229
|
+
continue;
|
|
230
|
+
const nl = item.indexOf("\n");
|
|
231
|
+
const key = nl < 0 ? item : item.slice(0, nl);
|
|
232
|
+
const value = nl < 0 ? "" : item.slice(nl + 1);
|
|
233
|
+
const values = entries.get(key) ?? [];
|
|
234
|
+
values.push(value);
|
|
235
|
+
entries.set(key, values);
|
|
236
|
+
}
|
|
237
|
+
return entries;
|
|
238
|
+
}
|
|
239
|
+
/** What changed between two readings of a config file: the keys added, removed or changed, never a value. */
|
|
240
|
+
export function describeConfigChange(before, after) {
|
|
241
|
+
if (after === "replaced" || before === "replaced")
|
|
242
|
+
return "replaced";
|
|
243
|
+
if (after === "unreadable")
|
|
244
|
+
return "no longer readable as git config";
|
|
245
|
+
if (before === "unreadable")
|
|
246
|
+
return "changed";
|
|
247
|
+
const same = (a, b) => a.length === b.length && a.every((v, i) => v === b[i]);
|
|
248
|
+
const groups = [
|
|
249
|
+
["added", [...after.keys()].filter((k) => !before.has(k))],
|
|
250
|
+
["removed", [...before.keys()].filter((k) => !after.has(k))],
|
|
251
|
+
["changed", [...after.keys()].filter((k) => before.has(k) && !same(before.get(k), after.get(k)))],
|
|
252
|
+
];
|
|
253
|
+
const total = groups.reduce((n, [, keys]) => n + keys.length, 0);
|
|
254
|
+
if (!total)
|
|
255
|
+
return "changed, but no key was added, removed or changed";
|
|
256
|
+
let left = KEYS_NAMED;
|
|
257
|
+
const parts = [];
|
|
258
|
+
for (const [verb, keys] of groups) {
|
|
259
|
+
if (!keys.length || !left)
|
|
260
|
+
continue;
|
|
261
|
+
// A key's subsection can be a URL with a token in it (url."https://user:token@host/".insteadof).
|
|
262
|
+
const named = keys.slice(0, left).map((k) => k.replace(/\/\/[^/@\s]*@/g, "//***@"));
|
|
263
|
+
left -= named.length;
|
|
264
|
+
parts.push(`${verb} ${named.join(", ")}`);
|
|
265
|
+
}
|
|
266
|
+
return parts.join("; ") + (total > KEYS_NAMED ? ` and ${total - KEYS_NAMED} more` : "");
|
|
267
|
+
}
|
|
268
|
+
/** The git config files that can change what the worker's git does in a worktree. */
|
|
269
|
+
export function gitConfigFiles(commonConfig, worktree, env = process.env) {
|
|
270
|
+
const home = env.HOME || homedir();
|
|
271
|
+
const xdg = env.XDG_CONFIG_HOME || join(home, ".config");
|
|
272
|
+
const files = [commonConfig, join(worktree, ".git"), join(home, ".gitconfig"), join(xdg, "git", "config")];
|
|
273
|
+
if (env.GIT_CONFIG_GLOBAL)
|
|
274
|
+
files.push(resolve(env.GIT_CONFIG_GLOBAL));
|
|
275
|
+
return [...new Set(files)];
|
|
276
|
+
}
|
|
277
|
+
/** What a file is now, for comparing: missing, where a link points, not a regular file, or its hash. */
|
|
278
|
+
function fileState(file) {
|
|
279
|
+
const l = lstatOrNull(file);
|
|
280
|
+
if (!l)
|
|
281
|
+
return "missing";
|
|
282
|
+
let prefix = "";
|
|
283
|
+
let st = l;
|
|
284
|
+
if (l.isSymbolicLink()) {
|
|
285
|
+
try {
|
|
286
|
+
prefix = `link to ${readlinkSync(file)}: `;
|
|
287
|
+
}
|
|
288
|
+
catch {
|
|
289
|
+
prefix = "link: ";
|
|
290
|
+
}
|
|
291
|
+
st = statOrNull(file);
|
|
292
|
+
if (!st)
|
|
293
|
+
return `${prefix}dangling`;
|
|
294
|
+
}
|
|
295
|
+
// Never open a FIFO or a device: the engine could put one there.
|
|
296
|
+
if (!st.isFile())
|
|
297
|
+
return `${prefix}not a file (${st.mode}, ${st.ino})`;
|
|
298
|
+
try {
|
|
299
|
+
return prefix + createHash("sha256").update(readFileSync(file)).digest("hex");
|
|
300
|
+
}
|
|
301
|
+
catch {
|
|
302
|
+
return `${prefix}unreadable`;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
function lstatOrNull(file) {
|
|
306
|
+
try {
|
|
307
|
+
return lstatSync(file);
|
|
308
|
+
}
|
|
309
|
+
catch {
|
|
310
|
+
return null;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
function statOrNull(file) {
|
|
314
|
+
try {
|
|
315
|
+
return statSync(file);
|
|
316
|
+
}
|
|
317
|
+
catch {
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
/** Up to max bytes of a regular file from offset (never a link, a FIFO or a device: the engine wrote it). */
|
|
322
|
+
function readFrom(file, offset, max) {
|
|
323
|
+
const st = lstatOrNull(file);
|
|
324
|
+
if (!st?.isFile() || st.size <= offset)
|
|
325
|
+
return Buffer.alloc(0);
|
|
326
|
+
let fd;
|
|
327
|
+
try {
|
|
328
|
+
fd = openSync(file, "r");
|
|
329
|
+
}
|
|
330
|
+
catch {
|
|
331
|
+
return Buffer.alloc(0);
|
|
332
|
+
}
|
|
333
|
+
try {
|
|
334
|
+
const buf = Buffer.alloc(Math.min(max, st.size - offset));
|
|
335
|
+
const n = readSync(fd, buf, 0, buf.length, offset);
|
|
336
|
+
return buf.subarray(0, n);
|
|
337
|
+
}
|
|
338
|
+
catch {
|
|
339
|
+
return Buffer.alloc(0);
|
|
340
|
+
}
|
|
341
|
+
finally {
|
|
342
|
+
closeSync(fd);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
/** The last lines of a command's output, for a question. */
|
|
346
|
+
export function tail(text, lines = 40, chars = 4000) {
|
|
347
|
+
const end = text.replace(/\s+$/, "").split("\n").slice(-lines).join("\n");
|
|
348
|
+
return end.length > chars ? end.slice(-chars) : end;
|
|
349
|
+
}
|
|
350
|
+
/** Runs the worker until a stop condition; returns the exit code for the CLI. */
|
|
351
|
+
export async function work(opts) {
|
|
352
|
+
const { board, engine, log, repo } = opts;
|
|
353
|
+
takeLock(opts.lockFile);
|
|
354
|
+
const state = { stopped: false };
|
|
355
|
+
/** Set by a second signal: it ends even the waits for a call that ends a run. */
|
|
356
|
+
const hard = { stopped: false };
|
|
357
|
+
/** The signal that stopped the worker, for the release of a run it stopped. */
|
|
358
|
+
let stopSignal = null;
|
|
359
|
+
/** The process the worker would kill on a signal: the engine only. */
|
|
360
|
+
let engineChild = null;
|
|
361
|
+
/** The signal that stopped the worker while an engine ran: that run is released. */
|
|
362
|
+
let engineStopped = null;
|
|
363
|
+
const onSignal = (signal) => {
|
|
364
|
+
if (state.stopped)
|
|
365
|
+
hard.stopped = true;
|
|
366
|
+
state.stopped = true;
|
|
367
|
+
stopSignal ??= signal;
|
|
368
|
+
log(`${signal}: stopping${engineChild ? " the engine" : " after the current step"}`);
|
|
369
|
+
if (engineChild) {
|
|
370
|
+
engineStopped ??= signal;
|
|
371
|
+
killGroup(engineChild);
|
|
372
|
+
}
|
|
373
|
+
};
|
|
374
|
+
process.on("SIGINT", onSignal);
|
|
375
|
+
process.on("SIGTERM", onSignal);
|
|
376
|
+
/** fn's result once the board answers (paused or unreachable boards are waited out); null when stopped meanwhile. */
|
|
377
|
+
const whenAwake = (fn, strict = false) => untilAnswered(fn, { intervalSec: opts.intervalSec, log, stop: state, strict });
|
|
378
|
+
/**
|
|
379
|
+
* The same for a call that ends a run (or that the run cannot go on without): the first signal does not
|
|
380
|
+
* end the wait, a second one does, and then the ticket stays In progress (Abandoned).
|
|
381
|
+
*/
|
|
382
|
+
const settle = async (ticketLabel, fn) => {
|
|
383
|
+
// Tried once even after two signals: a board that answers gets the ticket back.
|
|
384
|
+
try {
|
|
385
|
+
return await fn();
|
|
386
|
+
}
|
|
387
|
+
catch (err) {
|
|
388
|
+
if (!(err instanceof BoardPaused) && !isUnreachable(err))
|
|
389
|
+
throw err;
|
|
390
|
+
}
|
|
391
|
+
const result = await untilAnswered(fn, { intervalSec: opts.intervalSec, log, stop: hard });
|
|
392
|
+
if (result === null) {
|
|
393
|
+
log(`stopped while waiting for the board: ${ticketLabel} is left In progress and its lease will run out`);
|
|
394
|
+
throw new Abandoned();
|
|
395
|
+
}
|
|
396
|
+
return result;
|
|
397
|
+
};
|
|
398
|
+
const gitDir = resolve(repo, spawnSyncText("git", ["rev-parse", "--git-common-dir"], repo) ?? ".git");
|
|
399
|
+
const configFile = join(gitDir, "config");
|
|
400
|
+
/**
|
|
401
|
+
* The git config as it was before the engine started (each file's state and its entries, never logged);
|
|
402
|
+
* while set, every git command of the worker checks it first.
|
|
403
|
+
*/
|
|
404
|
+
let configBefore = null;
|
|
405
|
+
/** The worktree's .git file of the current run: not a config file, so a change of it is "replaced". */
|
|
406
|
+
let worktreeGit = null;
|
|
407
|
+
const checkConfig = () => {
|
|
408
|
+
if (!configBefore)
|
|
409
|
+
return;
|
|
410
|
+
for (const [file, was] of configBefore) {
|
|
411
|
+
if (fileState(file) === was.state)
|
|
412
|
+
continue;
|
|
413
|
+
const what = file === worktreeGit ? "replaced" : describeConfigChange(was.entries, configEntries(file));
|
|
414
|
+
throw new ConfigChanged(file, what);
|
|
415
|
+
}
|
|
416
|
+
};
|
|
417
|
+
/** Set when a run saw the git config change: the worker stops after it (exit 1). */
|
|
418
|
+
let configStop = null;
|
|
419
|
+
/** Runs a command; the worker's own git gets no hooks. */
|
|
420
|
+
const exec = (cmd, args, o) => new Promise((done) => {
|
|
421
|
+
const child = spawn(cmd, args, {
|
|
422
|
+
cwd: o.cwd,
|
|
423
|
+
env: o.env ?? process.env,
|
|
424
|
+
detached: !!o.group,
|
|
425
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
426
|
+
});
|
|
427
|
+
o.track?.(child);
|
|
428
|
+
let stdout = "";
|
|
429
|
+
let stderr = "";
|
|
430
|
+
let output = "";
|
|
431
|
+
const keep = (d) => {
|
|
432
|
+
output += d;
|
|
433
|
+
if (output.length > OUTPUT_KEEP)
|
|
434
|
+
output = output.slice(-OUTPUT_KEEP / 2);
|
|
435
|
+
};
|
|
436
|
+
let killed = false;
|
|
437
|
+
let timer;
|
|
438
|
+
if (o.timeoutMs !== undefined) {
|
|
439
|
+
timer = setTimeout(() => {
|
|
440
|
+
killed = true;
|
|
441
|
+
killGroup(child);
|
|
442
|
+
}, Math.max(0, o.timeoutMs));
|
|
443
|
+
}
|
|
444
|
+
child.stdout.on("data", (d) => {
|
|
445
|
+
stdout += d;
|
|
446
|
+
keep(d);
|
|
447
|
+
if (o.passStdout)
|
|
448
|
+
process.stdout.write(d);
|
|
449
|
+
});
|
|
450
|
+
child.stderr.on("data", (d) => {
|
|
451
|
+
stderr += d;
|
|
452
|
+
keep(d);
|
|
453
|
+
if (o.passStdout)
|
|
454
|
+
process.stderr.write(d);
|
|
455
|
+
});
|
|
456
|
+
let finished = false;
|
|
457
|
+
const finish = (code) => {
|
|
458
|
+
if (finished)
|
|
459
|
+
return;
|
|
460
|
+
finished = true;
|
|
461
|
+
if (timer)
|
|
462
|
+
clearTimeout(timer);
|
|
463
|
+
o.track?.(null);
|
|
464
|
+
done({ code, stdout, stderr, output, killed });
|
|
465
|
+
};
|
|
466
|
+
child.on("error", (err) => {
|
|
467
|
+
stderr += `could not start ${cmd}: ${err.message}`;
|
|
468
|
+
output += `could not start ${cmd}: ${err.message}`;
|
|
469
|
+
finish(127);
|
|
470
|
+
});
|
|
471
|
+
child.on("close", (code, signal) => finish(code ?? (signal ? 128 : 1)));
|
|
472
|
+
});
|
|
473
|
+
/**
|
|
474
|
+
* The worker's own git: no hooks, no fsmonitor, no BOARD_*, in its own process group (Ctrl+C at the
|
|
475
|
+
* terminal does not cut a push short); after the engine has started, only while the git config is
|
|
476
|
+
* as it was.
|
|
477
|
+
*/
|
|
478
|
+
const gitEnv = withoutBoard(process.env);
|
|
479
|
+
const git = async (cwd, ...args) => {
|
|
480
|
+
checkConfig();
|
|
481
|
+
return exec("git", ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", ...args], { cwd, env: gitEnv, group: true });
|
|
482
|
+
};
|
|
483
|
+
const removeWorktree = async (path, branch) => {
|
|
484
|
+
await git(repo, "worktree", "remove", "--force", path);
|
|
485
|
+
// Its .git file is gone with it, by the worker's own hand: no longer one to compare.
|
|
486
|
+
configBefore?.delete(join(path, ".git"));
|
|
487
|
+
if (existsSync(path))
|
|
488
|
+
rmSync(path, { recursive: true, force: true });
|
|
489
|
+
await git(repo, "worktree", "prune");
|
|
490
|
+
if (branch)
|
|
491
|
+
await git(repo, "branch", "-D", branch);
|
|
492
|
+
};
|
|
493
|
+
/**
|
|
494
|
+
* Keeps the commits of the local branch agent/<id> that origin does not have (an earlier run's, stopped or
|
|
495
|
+
* killed) as refs/mergeline/kept/<id>/<UTC time>, before the branch is reset. Never pushed, never deleted.
|
|
496
|
+
*/
|
|
497
|
+
const keepUnpushed = async (ticket) => {
|
|
498
|
+
const local = `refs/heads/${ticket.branch}`;
|
|
499
|
+
const tip = await git(repo, "rev-parse", "--verify", "-q", `${local}^{commit}`);
|
|
500
|
+
if (tip.code !== 0)
|
|
501
|
+
return {};
|
|
502
|
+
const sha = tip.stdout.trim();
|
|
503
|
+
const list = await git(repo, "rev-list", sha, "--not", "--remotes=origin");
|
|
504
|
+
if (list.code !== 0)
|
|
505
|
+
return { error: `could not look for unpushed commits on ${ticket.branch}: ${oneLine(list.stderr)}` };
|
|
506
|
+
const count = list.stdout.split("\n").filter(Boolean).length;
|
|
507
|
+
if (!count)
|
|
508
|
+
return {};
|
|
509
|
+
const time = new Date().toISOString().replace(/\.\d+Z$/, "Z").replace(/[-:]/g, "");
|
|
510
|
+
let last = "";
|
|
511
|
+
for (let i = 1; i <= 5; i++) {
|
|
512
|
+
const ref = `refs/mergeline/kept/${ticket.id}/${time}${i > 1 ? `-${i}` : ""}`;
|
|
513
|
+
// The empty old value: only when the ref does not exist yet.
|
|
514
|
+
const made = await git(repo, "update-ref", ref, sha, "");
|
|
515
|
+
if (made.code === 0)
|
|
516
|
+
return { ref, count };
|
|
517
|
+
last = oneLine(made.stderr);
|
|
518
|
+
}
|
|
519
|
+
return { error: `could not keep the unpushed commits on ${ticket.branch}: ${last}` };
|
|
520
|
+
};
|
|
521
|
+
/** One claimed ticket, from its spec to its cleanup. Returns whether it was a failed run. */
|
|
522
|
+
const runTicket = async (n, ticket, resumed) => {
|
|
523
|
+
const lbl = label(ticket);
|
|
524
|
+
const say = (line) => log(`run ${n}: ${lbl}: ${line}`);
|
|
525
|
+
log(`run ${n}: ${resumed ? "resumed" : "claimed"} ${lbl} (${ticket.branch})`);
|
|
526
|
+
// The id names a folder and the branch is pushed to: both are checked before either is used.
|
|
527
|
+
const unsafe = !/^[a-z0-9]+$/.test(ticket.id)
|
|
528
|
+
? `the ticket id ${JSON.stringify(ticket.id)} is not a safe folder name`
|
|
529
|
+
: ticket.branch !== `agent/${ticket.id}`
|
|
530
|
+
? `the ticket's branch ${JSON.stringify(ticket.branch)} is not agent/${ticket.id}`
|
|
531
|
+
: null;
|
|
532
|
+
if (unsafe) {
|
|
533
|
+
try {
|
|
534
|
+
await settle(lbl, () => board.release(ticket.id, unsafe));
|
|
535
|
+
say(`released: ${unsafe}`);
|
|
536
|
+
}
|
|
537
|
+
catch (err) {
|
|
538
|
+
if (err instanceof Abandoned)
|
|
539
|
+
return "failed";
|
|
540
|
+
say(`not released (${err instanceof Error ? err.message : String(err)}): ${unsafe}`);
|
|
541
|
+
}
|
|
542
|
+
return "failed";
|
|
543
|
+
}
|
|
544
|
+
/** The board said the ticket is no longer this session's (403, 404, 409): no other call for it. */
|
|
545
|
+
let refused = null;
|
|
546
|
+
/** The board turned a heartbeat down for another reason (a 4xx but 429): the run stops and is released. */
|
|
547
|
+
let rejected = null;
|
|
548
|
+
/** The child a refused heartbeat kills (the engine or the test command). */
|
|
549
|
+
let current = null;
|
|
550
|
+
const beat = async (note) => {
|
|
551
|
+
if (refused || rejected)
|
|
552
|
+
return;
|
|
553
|
+
try {
|
|
554
|
+
await board.heartbeat(ticket.id, note);
|
|
555
|
+
}
|
|
556
|
+
catch (err) {
|
|
557
|
+
const status = err instanceof BoardError && !(err instanceof BoardPaused) ? err.status : 0;
|
|
558
|
+
if (REFUSALS.includes(status)) {
|
|
559
|
+
refused ??= err.message;
|
|
560
|
+
if (current)
|
|
561
|
+
killGroup(current);
|
|
562
|
+
}
|
|
563
|
+
else if (status >= 400 && status < 500 && status !== 429) {
|
|
564
|
+
rejected ??= `the board turned down a heartbeat (${status}): ${err.message}`;
|
|
565
|
+
if (current)
|
|
566
|
+
killGroup(current);
|
|
567
|
+
}
|
|
568
|
+
else {
|
|
569
|
+
log(`run ${n}: ${lbl}: heartbeat failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
};
|
|
573
|
+
const held = () => {
|
|
574
|
+
if (refused)
|
|
575
|
+
throw new Refused(refused);
|
|
576
|
+
if (rejected) {
|
|
577
|
+
const why = rejected;
|
|
578
|
+
rejected = null;
|
|
579
|
+
throw new Error(why);
|
|
580
|
+
}
|
|
581
|
+
};
|
|
582
|
+
const beats = setInterval(() => void beat(), opts.beatSec * 1000);
|
|
583
|
+
const scratch = mkdtempSync(join(tmpdir(), "mergeline-run-"));
|
|
584
|
+
const emptyDir = join(scratch, "board");
|
|
585
|
+
const ghDir = join(scratch, "gh");
|
|
586
|
+
mkdirSync(emptyDir, { mode: 0o700 });
|
|
587
|
+
mkdirSync(ghDir, { mode: 0o700 });
|
|
588
|
+
const fence = fencedEnv(process.env, emptyDir, ghDir, originUrls(repo));
|
|
589
|
+
const path = resolve(repo, "..", ".wt", ticket.id);
|
|
590
|
+
let worktree = false;
|
|
591
|
+
let usage = null;
|
|
592
|
+
/** Set once a push of the run went through. */
|
|
593
|
+
let pushedOk = false;
|
|
594
|
+
const stopFor = (err) => {
|
|
595
|
+
configStop = { file: err.file, what: err.what, path, branch: ticket.branch, id: ticket.id, pushed: pushedOk };
|
|
596
|
+
};
|
|
597
|
+
const release = async (reason) => {
|
|
598
|
+
if (refused)
|
|
599
|
+
throw new Refused(refused);
|
|
600
|
+
await settle(lbl, () => board.release(ticket.id, reason, usage ?? undefined));
|
|
601
|
+
say(`released: ${reason}`);
|
|
602
|
+
return "failed";
|
|
603
|
+
};
|
|
604
|
+
const ask = async (question) => {
|
|
605
|
+
held();
|
|
606
|
+
await settle(lbl, () => board.askHuman(ticket.id, question));
|
|
607
|
+
say(`asked: ${question.split("\n")[0]}`);
|
|
608
|
+
if (usage) {
|
|
609
|
+
const u = usage;
|
|
610
|
+
try {
|
|
611
|
+
// The ticket is no longer In progress: a second signal just leaves the usage unreported.
|
|
612
|
+
const done = await untilAnswered(() => board.reportUsage(ticket.id, u), { intervalSec: opts.intervalSec, log, stop: hard });
|
|
613
|
+
if (done === null)
|
|
614
|
+
say("usage not reported: stopped while waiting for the board");
|
|
615
|
+
}
|
|
616
|
+
catch (err) {
|
|
617
|
+
say(`usage not reported: ${err instanceof Error ? err.message : String(err)}`);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
return "ok";
|
|
621
|
+
};
|
|
622
|
+
try {
|
|
623
|
+
// The spec, from origin/main only.
|
|
624
|
+
const fetched = await git(repo, "fetch", "origin", "main");
|
|
625
|
+
if (fetched.code !== 0)
|
|
626
|
+
return await release(`git fetch origin main failed: ${oneLine(fetched.stderr)}`);
|
|
627
|
+
const sha = (await git(repo, "rev-parse", "origin/main")).stdout.trim();
|
|
628
|
+
const specFile = `${ticket.spec_path}/spec.md`;
|
|
629
|
+
const spec = await git(repo, "show", `origin/main:${specFile}`);
|
|
630
|
+
if (spec.code !== 0 || !ticket.spec_path) {
|
|
631
|
+
return await ask(`There is no ${specFile} on main (${sha}). Push the spec or fix the ticket's spec path, then answer.`);
|
|
632
|
+
}
|
|
633
|
+
held();
|
|
634
|
+
await settle(lbl, () => board.recordSpec(ticket.id, sha));
|
|
635
|
+
const testCommand = await repoTest(git, repo);
|
|
636
|
+
// A fresh worktree.
|
|
637
|
+
await removeWorktree(path);
|
|
638
|
+
const onOrigin = (await git(repo, "ls-remote", "--heads", "origin", ticket.branch)).stdout.trim() !== "";
|
|
639
|
+
if (onOrigin) {
|
|
640
|
+
const f = await git(repo, "fetch", "origin", `+refs/heads/${ticket.branch}:refs/remotes/origin/${ticket.branch}`);
|
|
641
|
+
if (f.code !== 0)
|
|
642
|
+
return await release(`git fetch origin ${ticket.branch} failed: ${oneLine(f.stderr)}`);
|
|
643
|
+
}
|
|
644
|
+
// Commits of an earlier run that origin does not have (a stopped or killed worker) are kept aside, not used.
|
|
645
|
+
const kept = await keepUnpushed(ticket);
|
|
646
|
+
if (kept.error)
|
|
647
|
+
return await release(kept.error);
|
|
648
|
+
if (kept.ref)
|
|
649
|
+
say(`kept ${kept.count} unpushed commit(s) of an earlier run as ${kept.ref}`);
|
|
650
|
+
const from = onOrigin ? `origin/${ticket.branch}` : "origin/main";
|
|
651
|
+
const added = await git(repo, "worktree", "add", "--no-track", "-B", ticket.branch, path, from);
|
|
652
|
+
if (added.code !== 0)
|
|
653
|
+
return await release(`git worktree add failed: ${oneLine(added.stderr)}`);
|
|
654
|
+
worktree = true;
|
|
655
|
+
const base = (await git(path, "rev-parse", "HEAD")).stdout.trim();
|
|
656
|
+
// The run folder.
|
|
657
|
+
const runDir = join(path, RUN_DIR);
|
|
658
|
+
mkdirSync(runDir, { recursive: true });
|
|
659
|
+
writeFileSync(join(runDir, ".gitignore"), "*\n");
|
|
660
|
+
writeFileSync(join(runDir, "ticket.md"), ticketFile(ticket, lbl, onOrigin, sha, spec.stdout));
|
|
661
|
+
// The engine.
|
|
662
|
+
const originUrl = (await git(repo, "remote", "get-url", "origin")).stdout.trim();
|
|
663
|
+
const deadline = Date.now() + opts.timeoutSec * 1000;
|
|
664
|
+
const { cmd, args } = engine.command(opts.prompt, opts.engineArgs, { cwd: path, env: fence, scratch });
|
|
665
|
+
held();
|
|
666
|
+
if (state.stopped)
|
|
667
|
+
return await release(`board watch was stopped (${stopSignal ?? "SIGTERM"})`);
|
|
668
|
+
worktreeGit = join(path, ".git");
|
|
669
|
+
configBefore = new Map(gitConfigFiles(configFile, path).map((f) => [f, { state: fileState(f), entries: f === worktreeGit ? "replaced" : configEntries(f) }]));
|
|
670
|
+
say(`${engine.name} started in ../.wt/${ticket.id}`);
|
|
671
|
+
let planLogged = false;
|
|
672
|
+
let progressAt = 0;
|
|
673
|
+
let partial = Buffer.alloc(0);
|
|
674
|
+
let pending = [];
|
|
675
|
+
let lastNote = 0;
|
|
676
|
+
/** New complete lines of progress.log, read from where the last read stopped, at most READ_MAX at a time. */
|
|
677
|
+
const progressLines = (final) => {
|
|
678
|
+
const chunk = readFrom(join(runDir, "progress.log"), progressAt, READ_MAX);
|
|
679
|
+
progressAt += chunk.length;
|
|
680
|
+
let data = Buffer.concat([partial, chunk]);
|
|
681
|
+
const nl = data.lastIndexOf(10);
|
|
682
|
+
if (final || data.length >= READ_MAX) {
|
|
683
|
+
partial = Buffer.alloc(0);
|
|
684
|
+
}
|
|
685
|
+
else {
|
|
686
|
+
partial = nl >= 0 ? data.subarray(nl + 1) : data;
|
|
687
|
+
data = nl >= 0 ? data.subarray(0, nl + 1) : Buffer.alloc(0);
|
|
688
|
+
}
|
|
689
|
+
return data.toString("utf8").split("\n").map((l) => l.trim()).filter(Boolean);
|
|
690
|
+
};
|
|
691
|
+
const readFolder = async (final) => {
|
|
692
|
+
if (refused || rejected)
|
|
693
|
+
return;
|
|
694
|
+
if (!planLogged) {
|
|
695
|
+
const plan = readFrom(join(runDir, "plan.md"), 0, READ_MAX).toString("utf8").trim();
|
|
696
|
+
if (plan) {
|
|
697
|
+
planLogged = true;
|
|
698
|
+
try {
|
|
699
|
+
await whenAwake(() => board.logEvent(ticket.id, "plan", clip(plan, PLAN_CHARS)));
|
|
700
|
+
}
|
|
701
|
+
catch (err) {
|
|
702
|
+
say(`plan not logged: ${err instanceof Error ? err.message : String(err)}`);
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
pending.push(...progressLines(final));
|
|
707
|
+
if (pending.join(" / ").length > NOTE_CHARS)
|
|
708
|
+
pending = [clip(pending.join(" / "), NOTE_CHARS)];
|
|
709
|
+
if (pending.length && (final || Date.now() - lastNote >= NOTE_GAP_MS)) {
|
|
710
|
+
const note = clip(pending.join(" / "), NOTE_CHARS);
|
|
711
|
+
pending = [];
|
|
712
|
+
lastNote = Date.now();
|
|
713
|
+
await beat(note);
|
|
714
|
+
}
|
|
715
|
+
};
|
|
716
|
+
let polling = false;
|
|
717
|
+
const poll = setInterval(() => {
|
|
718
|
+
if (polling)
|
|
719
|
+
return;
|
|
720
|
+
polling = true;
|
|
721
|
+
void readFolder(false).finally(() => (polling = false));
|
|
722
|
+
}, POLL_MS);
|
|
723
|
+
const ran = await exec(cmd, args, {
|
|
724
|
+
cwd: path,
|
|
725
|
+
env: fence,
|
|
726
|
+
group: true,
|
|
727
|
+
passStdout: true,
|
|
728
|
+
timeoutMs: deadline - Date.now(),
|
|
729
|
+
track: (c) => {
|
|
730
|
+
current = c;
|
|
731
|
+
engineChild = c;
|
|
732
|
+
},
|
|
733
|
+
}).finally(() => clearInterval(poll));
|
|
734
|
+
while (polling)
|
|
735
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
736
|
+
if (ran.code === 127 && ran.stderr.startsWith("could not start"))
|
|
737
|
+
say(oneLine(ran.stderr));
|
|
738
|
+
held();
|
|
739
|
+
await readFolder(true);
|
|
740
|
+
held();
|
|
741
|
+
usage = engine.usage(ran.stdout);
|
|
742
|
+
if (engineStopped)
|
|
743
|
+
return await release(`board watch was stopped (${engineStopped})`);
|
|
744
|
+
checkConfig();
|
|
745
|
+
let result = ran.killed
|
|
746
|
+
? { outcome: "none", why: `the engine ran past the timeout (${opts.timeoutSec}s) and was stopped` }
|
|
747
|
+
: readResult(join(runDir, "result.json"));
|
|
748
|
+
if (result.outcome === "none") {
|
|
749
|
+
say(`no result from the engine: ${result.why}`);
|
|
750
|
+
const error = engine.error(ran.stdout);
|
|
751
|
+
return await release(`the engine ended without a result (exit ${ran.code})${error ? `: ${error}` : ""}`);
|
|
752
|
+
}
|
|
753
|
+
if (result.outcome === "release")
|
|
754
|
+
return await release(result.reason);
|
|
755
|
+
// Commit what the engine left.
|
|
756
|
+
const status = await git(path, "status", "--porcelain");
|
|
757
|
+
if (status.stdout.trim()) {
|
|
758
|
+
await git(path, "add", "-A");
|
|
759
|
+
const msg = commitMessage(lbl, engine.name, result);
|
|
760
|
+
const c = await git(path, "commit", "-m", msg.subject, "-m", msg.body);
|
|
761
|
+
if (c.code !== 0)
|
|
762
|
+
return await release(`could not commit what the engine left: ${oneLine(c.stderr || c.stdout)}`);
|
|
763
|
+
}
|
|
764
|
+
const head = async () => (await git(path, "rev-parse", "HEAD")).stdout.trim();
|
|
765
|
+
if (result.outcome === "done") {
|
|
766
|
+
const ahead = Number((await git(path, "rev-list", "--count", "origin/main..HEAD")).stdout.trim());
|
|
767
|
+
if (!ahead)
|
|
768
|
+
return await release("the engine said done but the branch has nothing beyond main");
|
|
769
|
+
}
|
|
770
|
+
const runTest = async () => {
|
|
771
|
+
if (!testCommand)
|
|
772
|
+
return null;
|
|
773
|
+
held();
|
|
774
|
+
say(`running the test command: ${testCommand}`);
|
|
775
|
+
const t = await exec("sh", ["-c", testCommand], {
|
|
776
|
+
cwd: path,
|
|
777
|
+
env: fence,
|
|
778
|
+
group: true,
|
|
779
|
+
timeoutMs: deadline - Date.now(),
|
|
780
|
+
track: (c) => (current = c),
|
|
781
|
+
});
|
|
782
|
+
held();
|
|
783
|
+
checkConfig();
|
|
784
|
+
if (t.code === 0 && !t.killed)
|
|
785
|
+
return null;
|
|
786
|
+
const how = t.killed ? "ran past the timeout" : `failed (exit ${t.code})`;
|
|
787
|
+
return `The test command \`${testCommand}\` ${how} after the engine said done. The end of its output:\n\n\`\`\`\n${tail(t.output)}\n\`\`\``;
|
|
788
|
+
};
|
|
789
|
+
if (result.outcome === "done") {
|
|
790
|
+
const failed = await runTest();
|
|
791
|
+
if (failed)
|
|
792
|
+
result = { outcome: "ask", question: failed };
|
|
793
|
+
}
|
|
794
|
+
// The push: always for done (nothing to push is fine), for a question only when there is work.
|
|
795
|
+
if (result.outcome === "done" || (await head()) !== base) {
|
|
796
|
+
const push = async () => {
|
|
797
|
+
held();
|
|
798
|
+
await beat();
|
|
799
|
+
held();
|
|
800
|
+
return git(path, "push", originUrl, `HEAD:refs/heads/${ticket.branch}`);
|
|
801
|
+
};
|
|
802
|
+
let pushed = await push();
|
|
803
|
+
if (pushed.code === 0)
|
|
804
|
+
pushedOk = true;
|
|
805
|
+
else {
|
|
806
|
+
say(`the push was rejected; merging origin/${ticket.branch}`);
|
|
807
|
+
const f = await git(path, "fetch", originUrl, `+refs/heads/${ticket.branch}:refs/remotes/origin/${ticket.branch}`);
|
|
808
|
+
const merged = f.code === 0 ? await git(path, "merge", "--no-edit", `origin/${ticket.branch}`) : f;
|
|
809
|
+
const lost = `The run's commits are not pushed; the worker keeps them under refs/mergeline/kept/${ticket.id}/.`;
|
|
810
|
+
if (merged.code !== 0) {
|
|
811
|
+
const conflicts = (await git(path, "diff", "--name-only", "--diff-filter=U")).stdout.trim().split("\n").filter(Boolean);
|
|
812
|
+
await git(path, "merge", "--abort");
|
|
813
|
+
const what = conflicts.length
|
|
814
|
+
? `Merging origin/${ticket.branch} (it moved on origin during the run) conflicts in: ${conflicts.join(", ")}.`
|
|
815
|
+
: `Bringing in origin/${ticket.branch} (it moved on origin during the run) failed: ${oneLine(merged.stderr || merged.stdout)}.`;
|
|
816
|
+
return await ask(withBefore(result, `${what} ${lost}`));
|
|
817
|
+
}
|
|
818
|
+
if (result.outcome === "done") {
|
|
819
|
+
const failed = await runTest();
|
|
820
|
+
if (failed)
|
|
821
|
+
result = { outcome: "ask", question: failed };
|
|
822
|
+
}
|
|
823
|
+
pushed = await push();
|
|
824
|
+
if (pushed.code === 0)
|
|
825
|
+
pushedOk = true;
|
|
826
|
+
else {
|
|
827
|
+
return await ask(withBefore(result, `The push to ${ticket.branch} was rejected twice: ${oneLine(pushed.stderr)}. ${lost}`));
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
if (result.outcome === "ask")
|
|
832
|
+
return await ask(result.question);
|
|
833
|
+
// The PR. gh reads the checkout's git config too.
|
|
834
|
+
held();
|
|
835
|
+
const summary = result.summary;
|
|
836
|
+
const ghEnv = withoutBoard(process.env);
|
|
837
|
+
checkConfig();
|
|
838
|
+
const view = await exec("gh", ["pr", "view", ticket.branch, "--json", "url,state"], { cwd: repo, env: ghEnv, group: true });
|
|
839
|
+
let prUrl = "";
|
|
840
|
+
if (view.code === 0) {
|
|
841
|
+
try {
|
|
842
|
+
const pr = JSON.parse(view.stdout);
|
|
843
|
+
if (pr.state === "OPEN" && typeof pr.url === "string")
|
|
844
|
+
prUrl = pr.url;
|
|
845
|
+
}
|
|
846
|
+
catch {
|
|
847
|
+
// not an open PR
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
if (!prUrl) {
|
|
851
|
+
const body = `Ticket ${lbl} (${ticket.id}).\nSpec: ${specFile} at ${sha}.\n\n${summary}`;
|
|
852
|
+
checkConfig();
|
|
853
|
+
const created = await exec("gh", ["pr", "create", "--base", "main", "--head", ticket.branch, "--title", ticket.title, "--body", body], {
|
|
854
|
+
cwd: repo,
|
|
855
|
+
env: ghEnv,
|
|
856
|
+
group: true,
|
|
857
|
+
});
|
|
858
|
+
prUrl = created.stdout.trim().split("\n").at(-1)?.trim() ?? "";
|
|
859
|
+
if (created.code !== 0 || !/^https?:\/\/\S+$/.test(prUrl)) {
|
|
860
|
+
return await release(`could not open the PR: ${oneLine(created.stderr || created.stdout) || `exit ${created.code}`}; the branch is pushed`);
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
held();
|
|
864
|
+
await settle(lbl, () => board.submitForReview(ticket.id, prUrl, summary, usage ?? undefined));
|
|
865
|
+
say(`submitted for review: ${prUrl}`);
|
|
866
|
+
return "ok";
|
|
867
|
+
}
|
|
868
|
+
catch (err) {
|
|
869
|
+
if (err instanceof Abandoned)
|
|
870
|
+
return "failed";
|
|
871
|
+
if (err instanceof Refused) {
|
|
872
|
+
say(`the board refused the heartbeat (${err.message}); stopped without pushing`);
|
|
873
|
+
return "failed";
|
|
874
|
+
}
|
|
875
|
+
if (err instanceof ConfigChanged) {
|
|
876
|
+
stopFor(err);
|
|
877
|
+
try {
|
|
878
|
+
return await release(`the git config changed during the run (${err.file}: ${err.what}); nothing was pushed`);
|
|
879
|
+
}
|
|
880
|
+
catch (e) {
|
|
881
|
+
if (!(e instanceof Abandoned))
|
|
882
|
+
say(`not released: ${e instanceof Error ? e.message : String(e)}`);
|
|
883
|
+
return "failed";
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
887
|
+
say(`the run failed: ${message}`);
|
|
888
|
+
if (refused || (err instanceof BoardError && REFUSALS.includes(err.status)))
|
|
889
|
+
return "failed";
|
|
890
|
+
try {
|
|
891
|
+
return await release(`the worker failed: ${clip(message, 400)}`);
|
|
892
|
+
}
|
|
893
|
+
catch {
|
|
894
|
+
return "failed";
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
finally {
|
|
898
|
+
clearInterval(beats);
|
|
899
|
+
engineChild = null;
|
|
900
|
+
rmSync(scratch, { recursive: true, force: true });
|
|
901
|
+
try {
|
|
902
|
+
// A changed git config: no more git at all; the worktree and the branch stay for the person to inspect.
|
|
903
|
+
if (!configStop) {
|
|
904
|
+
// A run that pushed nothing (released, asked, failed): its commits are kept before the branch goes.
|
|
905
|
+
const kept = pushedOk ? {} : await keepUnpushed(ticket);
|
|
906
|
+
if (kept.ref)
|
|
907
|
+
say(`kept ${kept.count} unpushed commit(s) of this run as ${kept.ref}`);
|
|
908
|
+
if (kept.error)
|
|
909
|
+
say(`${kept.error}; the local branch ${ticket.branch} is left`);
|
|
910
|
+
const branch = kept.error ? undefined : ticket.branch;
|
|
911
|
+
if (worktree || existsSync(path))
|
|
912
|
+
await removeWorktree(path, branch);
|
|
913
|
+
else if (branch)
|
|
914
|
+
await git(repo, "branch", "-D", branch);
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
catch (err) {
|
|
918
|
+
if (!(err instanceof ConfigChanged))
|
|
919
|
+
throw err;
|
|
920
|
+
stopFor(err);
|
|
921
|
+
}
|
|
922
|
+
configBefore = null;
|
|
923
|
+
worktreeGit = null;
|
|
924
|
+
}
|
|
925
|
+
};
|
|
926
|
+
let runs = 0;
|
|
927
|
+
let failures = 0;
|
|
928
|
+
let waitingLogged = false;
|
|
929
|
+
let heldLogged = false;
|
|
930
|
+
let kitLogged = false;
|
|
931
|
+
const givenBack = new Set();
|
|
932
|
+
try {
|
|
933
|
+
if ((await whenAwake(() => board.login(), true)) === null)
|
|
934
|
+
return 0;
|
|
935
|
+
log(`working ${board.agentName}'s queue with ${engine.name} in ${repo}`);
|
|
936
|
+
while (!state.stopped) {
|
|
937
|
+
if (runs >= opts.maxRuns) {
|
|
938
|
+
const mine = await whenAwake(() => board.listMine());
|
|
939
|
+
if (mine === null)
|
|
940
|
+
break;
|
|
941
|
+
if (hasWork(mine)) {
|
|
942
|
+
log(`stopping after ${runs} runs (--max-runs) with work still in the queue`);
|
|
943
|
+
return 1;
|
|
944
|
+
}
|
|
945
|
+
log(`queue empty after ${runs} run(s); stopping`);
|
|
946
|
+
return 0;
|
|
947
|
+
}
|
|
948
|
+
let claimed;
|
|
949
|
+
try {
|
|
950
|
+
claimed = await whenAwake(() => board.claim({ runner: "worker", engine: engine.name }));
|
|
951
|
+
}
|
|
952
|
+
catch (err) {
|
|
953
|
+
const mine = err instanceof BoardError && err.status === 409 && /already working on ticket ([a-z0-9]+)/.exec(err.message);
|
|
954
|
+
if (mine) {
|
|
955
|
+
// This session still holds a ticket a run of it ended without giving back (its last call failed):
|
|
956
|
+
// its worktree is gone, so give it back, once.
|
|
957
|
+
const id = mine[1];
|
|
958
|
+
if (givenBack.has(id))
|
|
959
|
+
throw err;
|
|
960
|
+
givenBack.add(id);
|
|
961
|
+
log(`${id} is still held by this worker from an earlier run; releasing it`);
|
|
962
|
+
try {
|
|
963
|
+
await settle(id, () => board.release(id, "board watch: an earlier run of this worker ended without giving the ticket back"));
|
|
964
|
+
}
|
|
965
|
+
catch (e) {
|
|
966
|
+
if (e instanceof Abandoned)
|
|
967
|
+
break;
|
|
968
|
+
throw e;
|
|
969
|
+
}
|
|
970
|
+
continue;
|
|
971
|
+
}
|
|
972
|
+
const held = err instanceof BoardError && err.status === 409 && /held by another session/.test(err.message);
|
|
973
|
+
if (!held)
|
|
974
|
+
throw err;
|
|
975
|
+
if (!heldLogged) {
|
|
976
|
+
const m = /ticket (\S+) \("((?:[^"\\]|\\.)*)"\)/.exec(err.message);
|
|
977
|
+
const who = m ? label({ id: m[1], title: m[2] }) : "its ticket";
|
|
978
|
+
log(`${who} is held by another session of ${board.agentName}; claiming again every ${opts.intervalSec}s`);
|
|
979
|
+
}
|
|
980
|
+
heldLogged = true;
|
|
981
|
+
await sleep(opts.intervalSec * 1000, state);
|
|
982
|
+
continue;
|
|
983
|
+
}
|
|
984
|
+
if (claimed === null)
|
|
985
|
+
break;
|
|
986
|
+
heldLogged = false;
|
|
987
|
+
if (claimed.kit_outdated && !kitLogged) {
|
|
988
|
+
log(claimed.kit_outdated);
|
|
989
|
+
kitLogged = true;
|
|
990
|
+
}
|
|
991
|
+
if (!claimed.ticket) {
|
|
992
|
+
if (!opts.follow) {
|
|
993
|
+
log(`queue empty after ${runs} run(s); stopping`);
|
|
994
|
+
return 0;
|
|
995
|
+
}
|
|
996
|
+
if (!waitingLogged)
|
|
997
|
+
log(`queue empty; checking every ${opts.intervalSec}s (Ctrl+C to stop)`);
|
|
998
|
+
waitingLogged = true;
|
|
999
|
+
await sleep(opts.intervalSec * 1000, state);
|
|
1000
|
+
continue;
|
|
1001
|
+
}
|
|
1002
|
+
waitingLogged = false;
|
|
1003
|
+
runs++;
|
|
1004
|
+
const outcome = await runTicket(runs, claimed.ticket, !!claimed.resumed);
|
|
1005
|
+
if (configStop) {
|
|
1006
|
+
const { file, what, path, branch, id, pushed } = configStop;
|
|
1007
|
+
log(`stopping: ${file} changed while run ${runs} was going on (${what}), so the worker ran no more git for it.`);
|
|
1008
|
+
log(`Inspect your git config (${file}; also core.fsmonitor, core.sshCommand, url.*.insteadOf, include.path) before you start it again.`);
|
|
1009
|
+
log(pushed
|
|
1010
|
+
? `The run's branch ${branch} was pushed before the change; its worktree is left at ${path}.`
|
|
1011
|
+
: `The run's commits are not pushed: they are on the local branch ${branch} (the worktree is left at ${path}), and the next start keeps them under refs/mergeline/kept/${id}/ before it starts the ticket again.`);
|
|
1012
|
+
log("A change of your own while the worker runs (git branch -D of a branch with an upstream, git config, git push -u) stops it the same way.");
|
|
1013
|
+
return 1;
|
|
1014
|
+
}
|
|
1015
|
+
if (outcome === "failed") {
|
|
1016
|
+
failures++;
|
|
1017
|
+
if (failures >= FAILURE_STOP) {
|
|
1018
|
+
log(`${failures} runs in a row failed; stopping`);
|
|
1019
|
+
return 1;
|
|
1020
|
+
}
|
|
1021
|
+
if (!state.stopped)
|
|
1022
|
+
await sleep(opts.intervalSec * 1000, state);
|
|
1023
|
+
}
|
|
1024
|
+
else {
|
|
1025
|
+
failures = 0;
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
return 0;
|
|
1029
|
+
}
|
|
1030
|
+
finally {
|
|
1031
|
+
process.off("SIGINT", onSignal);
|
|
1032
|
+
process.off("SIGTERM", onSignal);
|
|
1033
|
+
rmSync(opts.lockFile, { force: true });
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
/** Kills a child's whole process group (it was started detached), then for sure. */
|
|
1037
|
+
function killGroup(child) {
|
|
1038
|
+
const kill = (signal) => {
|
|
1039
|
+
if (child.pid === undefined)
|
|
1040
|
+
return;
|
|
1041
|
+
try {
|
|
1042
|
+
process.kill(-child.pid, signal);
|
|
1043
|
+
}
|
|
1044
|
+
catch {
|
|
1045
|
+
try {
|
|
1046
|
+
child.kill(signal);
|
|
1047
|
+
}
|
|
1048
|
+
catch {
|
|
1049
|
+
// gone
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
};
|
|
1053
|
+
kill("SIGTERM");
|
|
1054
|
+
setTimeout(() => kill("SIGKILL"), KILL_GRACE_MS).unref();
|
|
1055
|
+
}
|
|
1056
|
+
const oneLine = (s) => clip(s.replace(/\s+/g, " ").trim(), 500);
|
|
1057
|
+
/** A question for a run that could not push, after what the engine itself asked (if it asked). */
|
|
1058
|
+
function withBefore(result, what) {
|
|
1059
|
+
return result.outcome === "ask" ? `${result.question}\n\n${what}` : `The engine said done, but its work could not be pushed. ${what}`;
|
|
1060
|
+
}
|
|
1061
|
+
/** `test` from .mergeline.json on origin/main (never from the worktree), or null. */
|
|
1062
|
+
async function repoTest(git, repo) {
|
|
1063
|
+
const shown = await git(repo, "show", "origin/main:.mergeline.json");
|
|
1064
|
+
if (shown.code !== 0)
|
|
1065
|
+
return null;
|
|
1066
|
+
try {
|
|
1067
|
+
const test = JSON.parse(shown.stdout).test;
|
|
1068
|
+
return typeof test === "string" && test.trim() ? test.trim() : null;
|
|
1069
|
+
}
|
|
1070
|
+
catch {
|
|
1071
|
+
return null;
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
function ticketFile(t, lbl, onOrigin, sha, spec) {
|
|
1075
|
+
const notes = t.notes ?? [];
|
|
1076
|
+
const round = 1 + notes.filter((n) => n.kind === "review").length;
|
|
1077
|
+
return [
|
|
1078
|
+
`# ${t.title}`,
|
|
1079
|
+
"",
|
|
1080
|
+
`Ticket: ${lbl} (${t.id})`,
|
|
1081
|
+
`Branch: ${t.branch}, ${onOrigin ? `from origin/${t.branch}` : "new, from origin/main"}`,
|
|
1082
|
+
`Review round: ${round}`,
|
|
1083
|
+
`Spec: ${t.spec_path}/spec.md at ${sha}`,
|
|
1084
|
+
"",
|
|
1085
|
+
"## Spec",
|
|
1086
|
+
"",
|
|
1087
|
+
spec.replace(/\s+$/, ""),
|
|
1088
|
+
"",
|
|
1089
|
+
"## Answer",
|
|
1090
|
+
"",
|
|
1091
|
+
t.answer?.trim() ? t.answer.trim() : "None.",
|
|
1092
|
+
"",
|
|
1093
|
+
"## Notes, oldest first",
|
|
1094
|
+
"",
|
|
1095
|
+
...(notes.length ? notes.map((n) => `- ${n.kind}, ${n.at}: ${n.text}`) : ["None."]),
|
|
1096
|
+
"",
|
|
1097
|
+
].join("\n");
|
|
1098
|
+
}
|