@saadnvd1/agent-os 0.3.466 → 0.3.468
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/components/SessionList/useDoneAction.ts +2 -1
- package/lib/db/migrations.ts +31 -0
- package/lib/done/index.ts +1 -8
- package/lib/done/plan.ts +13 -2
- package/lib/done/worktree.ts +55 -33
- package/lib/orchestrator/armed.ts +2 -2
- package/lib/orchestrator/brief.ts +1 -1
- package/lib/orchestrator/escalate.ts +7 -7
- package/lib/orchestrator/gates.ts +36 -16
- package/lib/orchestrator/signoff.ts +1 -1
- package/lib/tasks/finish.ts +8 -2
- package/lib/worktree-removal.ts +115 -0
- package/lib/worktrees.ts +30 -27
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -908,10 +908,13 @@ it may do on its own, and the lines that always come back to you as asks.
|
|
|
908
908
|
`BLOCKED:` or waiting, the diff stays in scope (no secrets, not only
|
|
909
909
|
lockfiles, nothing outside the repo, within the card) and its stack parent
|
|
910
910
|
has merged, and the PR body's Code review section names that commit. `land` judges each item again at its own head right before
|
|
911
|
-
merging it.
|
|
912
|
-
as an ask, and the orchestrator stops merging that task.
|
|
913
|
-
|
|
914
|
-
|
|
911
|
+
merging it. A gate failing again on a different commit, or a repo with no
|
|
912
|
+
CI, goes to you as an ask, and the orchestrator stops merging that task.
|
|
913
|
+
A failure counts once per task, gate and head commit, however many times
|
|
914
|
+
and from wherever that commit is judged (the armed watcher, CI finishing,
|
|
915
|
+
`sign_off`, `done`, `land`); a pass in between doesn't reset it. A review
|
|
916
|
+
that blocks counts when its verdict arrives, whether or not `sign_off`
|
|
917
|
+
ever reads it.
|
|
915
918
|
- **Arm to merge:** `arm` (or **Arm merge** on a task's row) makes AgentOS
|
|
916
919
|
merge the task's PR itself the moment every `sign_off` gate passes at its
|
|
917
920
|
head, starting the review once CI is green, so nothing polls `sign_off`.
|
|
@@ -27,7 +27,8 @@ export function useDoneAction() {
|
|
|
27
27
|
id,
|
|
28
28
|
duration: 8000,
|
|
29
29
|
...(!outcome.merged &&
|
|
30
|
-
outcome.worktree.action !== "removed" &&
|
|
30
|
+
outcome.worktree.action !== "removed" &&
|
|
31
|
+
outcome.worktree.action !== "gone" && {
|
|
31
32
|
action: {
|
|
32
33
|
label: "Undo",
|
|
33
34
|
onClick: () => unarchive.mutate(session.id),
|
package/lib/db/migrations.ts
CHANGED
|
@@ -1327,6 +1327,37 @@ const migrations: Migration[] = [
|
|
|
1327
1327
|
installChangeTriggers(db, TABLES_WATCHED_58);
|
|
1328
1328
|
},
|
|
1329
1329
|
},
|
|
1330
|
+
{
|
|
1331
|
+
id: 59,
|
|
1332
|
+
name: "gate_failure_commits",
|
|
1333
|
+
up: (db) => {
|
|
1334
|
+
// Each head a gate failed on, once: a gate's failure counts once per
|
|
1335
|
+
// commit however many times that commit is judged (the armed watcher,
|
|
1336
|
+
// CI finishing, sign_off, done, land), and the second commit it fails
|
|
1337
|
+
// on goes to Saad. The key is what makes it once, across restarts and
|
|
1338
|
+
// concurrent evaluations.
|
|
1339
|
+
db.exec(`
|
|
1340
|
+
CREATE TABLE IF NOT EXISTS orchestrator_gate_failure_commits (
|
|
1341
|
+
session_id TEXT NOT NULL,
|
|
1342
|
+
gate TEXT NOT NULL,
|
|
1343
|
+
sha TEXT NOT NULL,
|
|
1344
|
+
workspace_id TEXT NOT NULL,
|
|
1345
|
+
reason TEXT,
|
|
1346
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
1347
|
+
PRIMARY KEY (session_id, gate, sha)
|
|
1348
|
+
);
|
|
1349
|
+
INSERT OR IGNORE INTO orchestrator_gate_failure_commits (session_id, gate, sha, workspace_id, reason)
|
|
1350
|
+
SELECT session_id, gate, last_sha, workspace_id, last_reason
|
|
1351
|
+
FROM orchestrator_gate_failures WHERE last_sha IS NOT NULL;
|
|
1352
|
+
CREATE TRIGGER IF NOT EXISTS gate_failures_clear_commits
|
|
1353
|
+
AFTER DELETE ON orchestrator_gate_failures
|
|
1354
|
+
BEGIN
|
|
1355
|
+
DELETE FROM orchestrator_gate_failure_commits
|
|
1356
|
+
WHERE session_id = OLD.session_id AND gate = OLD.gate;
|
|
1357
|
+
END;
|
|
1358
|
+
`);
|
|
1359
|
+
},
|
|
1360
|
+
},
|
|
1330
1361
|
];
|
|
1331
1362
|
|
|
1332
1363
|
// `upTo`: stop after this id (tests that start from an older database).
|
package/lib/done/index.ts
CHANGED
|
@@ -7,7 +7,6 @@
|
|
|
7
7
|
* and archives the session. What it will do is decided first (plan.ts).
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import fs from "fs";
|
|
11
10
|
import { db, queries, type Session } from "../db";
|
|
12
11
|
import { stopChat } from "../chat/runner";
|
|
13
12
|
import { hostExec } from "../hosts";
|
|
@@ -69,6 +68,7 @@ async function stopAgent(s: Session): Promise<void> {
|
|
|
69
68
|
|
|
70
69
|
function worktreeLine(f: WorktreeFate): string {
|
|
71
70
|
if (f.action === "removed") return `worktree removed (${f.why})`;
|
|
71
|
+
if (f.action === "gone") return `worktree ${f.why}`;
|
|
72
72
|
if (f.action === "kept") return `worktree kept at ${f.path}: ${f.why}`;
|
|
73
73
|
return "no worktree";
|
|
74
74
|
}
|
|
@@ -141,7 +141,6 @@ export async function doneSession(
|
|
|
141
141
|
`${s.name} runs on ${peer.hostName}: only you can mark it done, from its row.`
|
|
142
142
|
);
|
|
143
143
|
if (peer) return doneOnPeer(peer, s);
|
|
144
|
-
const hadWorktree = !!s.worktree_path && fs.existsSync(s.worktree_path);
|
|
145
144
|
// A sign-off just before this (the orchestrator's sign_off, then done)
|
|
146
145
|
// is still cleaning up in the background: read what it leaves.
|
|
147
146
|
await mergeSettled(s.id);
|
|
@@ -173,12 +172,6 @@ export async function doneSession(
|
|
|
173
172
|
await stopAgent(s);
|
|
174
173
|
worktree = await settleWorktree(s, plan.merged);
|
|
175
174
|
}
|
|
176
|
-
if (
|
|
177
|
-
hadWorktree &&
|
|
178
|
-
worktree.action === "none" &&
|
|
179
|
-
!fs.existsSync(s.worktree_path!)
|
|
180
|
-
)
|
|
181
|
-
worktree = { action: "removed", why: "its branch is merged" };
|
|
182
175
|
archiveSession(s.id);
|
|
183
176
|
const head = merged ?? `Done: ${s.name}.`;
|
|
184
177
|
const extra = notes.length ? `, ${notes.join(", ")}` : "";
|
package/lib/done/plan.ts
CHANGED
|
@@ -78,6 +78,14 @@ async function taskPlan(s: Session): Promise<DonePlan> {
|
|
|
78
78
|
return cleanup(s, "done", pr, null);
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
// A task merged before done looked (a sign-off, an armed merge): its PR's
|
|
82
|
+
// head, so commits that went in with it aren't counted as unsaved once the
|
|
83
|
+
// cleanup has deleted its branch on origin.
|
|
84
|
+
async function mergedAs(s: Session): Promise<MergedAs> {
|
|
85
|
+
const pr = await prFor(s, true).catch(() => null);
|
|
86
|
+
return { prHead: pr?.state === "MERGED" ? (pr.head ?? null) : null };
|
|
87
|
+
}
|
|
88
|
+
|
|
81
89
|
export async function planDone(
|
|
82
90
|
s: Session,
|
|
83
91
|
callerId?: string | null
|
|
@@ -90,7 +98,8 @@ export async function planDone(
|
|
|
90
98
|
if ((await statusOf(s)).status === "running")
|
|
91
99
|
return refuse(`${s.name} is still working: wait for it, or stop it first`);
|
|
92
100
|
if (s.task_status === "running") return taskPlan(s);
|
|
93
|
-
if (s.task_status === "merged")
|
|
101
|
+
if (s.task_status === "merged")
|
|
102
|
+
return cleanup(s, null, null, await mergedAs(s));
|
|
94
103
|
return cleanup(s, null, null, null);
|
|
95
104
|
}
|
|
96
105
|
|
|
@@ -105,6 +114,8 @@ export function planLine(p: DonePlan): string {
|
|
|
105
114
|
? `remove its worktree (${w.why})`
|
|
106
115
|
: w.action === "kept"
|
|
107
116
|
? `keep its worktree: ${w.why}`
|
|
108
|
-
:
|
|
117
|
+
: w.action === "gone"
|
|
118
|
+
? `its worktree is ${w.why}`
|
|
119
|
+
: "no worktree";
|
|
109
120
|
return `stop it, ${tree}, archive it`;
|
|
110
121
|
}
|
package/lib/done/worktree.ts
CHANGED
|
@@ -10,17 +10,15 @@ import { getDefaultBranch } from "../git";
|
|
|
10
10
|
import { git as runGit } from "../git-exec";
|
|
11
11
|
import { mergePolicy } from "../tasks/merge-policy";
|
|
12
12
|
import { getProject } from "../projects";
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
isAgentOSWorktree,
|
|
16
|
-
removalRunning,
|
|
17
|
-
removalSettled,
|
|
18
|
-
} from "../worktrees";
|
|
13
|
+
import { deleteWorktree, isAgentOSWorktree } from "../worktrees";
|
|
14
|
+
import { removalRunning, removalSettled } from "../worktree-removal";
|
|
19
15
|
import { describeChanges, unsavedChanges } from "../worktree-placed";
|
|
20
16
|
|
|
21
17
|
export type WorktreeFate =
|
|
22
18
|
| { action: "none" }
|
|
23
19
|
| { action: "removed"; why: string }
|
|
20
|
+
// Removed, or being removed, by something else (a merge's cleanup).
|
|
21
|
+
| { action: "gone"; why: string }
|
|
24
22
|
| { action: "kept"; why: string; path: string };
|
|
25
23
|
|
|
26
24
|
const git = (cwd: string, ...args: string[]) =>
|
|
@@ -104,30 +102,63 @@ export type FatePreview =
|
|
|
104
102
|
type Judged = Pick<Session, "worktree_path" | "base_branch"> &
|
|
105
103
|
Partial<Pick<Session, "project_id">>;
|
|
106
104
|
|
|
105
|
+
// Gone, or still going: removed by whatever else got to it first.
|
|
106
|
+
const gone = (merged: MergedAs | null, still = false) => ({
|
|
107
|
+
action: "gone" as const,
|
|
108
|
+
why: `${still ? "still being" : "already"} removed${merged ? " by the merge cleanup" : ""}`,
|
|
109
|
+
});
|
|
110
|
+
|
|
107
111
|
// Decides without touching anything, after any removal already running
|
|
108
|
-
// (a
|
|
109
|
-
//
|
|
112
|
+
// (a merge's cleanup, from any process) has finished: a tree read while
|
|
113
|
+
// git deletes it lists every deleted file as a change.
|
|
110
114
|
export async function worktreeFate(
|
|
111
115
|
session: Judged,
|
|
112
|
-
merged: MergedAs | null
|
|
116
|
+
merged: MergedAs | null,
|
|
117
|
+
waitMs = 60_000
|
|
113
118
|
): Promise<FatePreview> {
|
|
114
119
|
const path = session.worktree_path;
|
|
120
|
+
if (!path) return { action: "none" };
|
|
115
121
|
for (;;) {
|
|
116
|
-
if (
|
|
122
|
+
if (!(await removalSettled(path, waitMs))) return gone(merged, true);
|
|
123
|
+
if (!fs.existsSync(path)) return gone(merged);
|
|
117
124
|
const fate = await judge(session, merged);
|
|
118
|
-
|
|
125
|
+
// A removal that started while it read: read what it leaves.
|
|
126
|
+
if (!removalRunning(path) && fs.existsSync(path)) return fate;
|
|
119
127
|
}
|
|
120
128
|
}
|
|
121
129
|
|
|
130
|
+
// What the branch's commits would lose: why they keep the worktree, or
|
|
131
|
+
// null when every one is merged, pushed or not its own.
|
|
132
|
+
async function commitsKeep(
|
|
133
|
+
path: string,
|
|
134
|
+
repo: string,
|
|
135
|
+
session: Judged,
|
|
136
|
+
merged: MergedAs | null
|
|
137
|
+
): Promise<string | null> {
|
|
138
|
+
if (merged) {
|
|
139
|
+
const stray = await strayCommits(path, merged).catch(() => null);
|
|
140
|
+
if (stray === null) return "its commits can't be checked";
|
|
141
|
+
if (stray > 0)
|
|
142
|
+
return `${stray} commit${stray === 1 ? " is" : "s are"} on it that the merged PR didn't include and no remote has`;
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
const base = session.base_branch || (await getDefaultBranch(repo));
|
|
146
|
+
const own = await ownCommits(path, base).catch(() => null);
|
|
147
|
+
if (own === null) return `its base ${base} can't be found to compare`;
|
|
148
|
+
if (own > 0)
|
|
149
|
+
return `its branch has ${own} commit${own === 1 ? "" : "s"} not merged into ${base}`;
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
|
|
122
153
|
// Uncommitted work always keeps it; a merged branch goes only if the merge
|
|
123
154
|
// settings delete worktrees and every commit on it was in the merge or is
|
|
124
|
-
// on a remote; an unmerged one only if it has no commits of its own.
|
|
155
|
+
// on a remote; an unmerged one only if it has no commits of its own. A
|
|
156
|
+
// kept one names every reason, the commits as well as the changes.
|
|
125
157
|
async function judge(
|
|
126
158
|
session: Judged,
|
|
127
159
|
merged: MergedAs | null
|
|
128
160
|
): Promise<FatePreview> {
|
|
129
|
-
const path = session.worktree_path
|
|
130
|
-
if (!path || !fs.existsSync(path)) return { action: "none" };
|
|
161
|
+
const path = session.worktree_path!;
|
|
131
162
|
const keep = (why: string) => ({ action: "kept" as const, why, path });
|
|
132
163
|
if (!isAgentOSWorktree(path)) return keep("it isn't an AgentOS worktree");
|
|
133
164
|
if (merged && session.project_id) {
|
|
@@ -141,27 +172,18 @@ async function judge(
|
|
|
141
172
|
// nobody has touched since isn't work; anything else is.
|
|
142
173
|
const unsaved = await unsavedChanges(path).catch(() => null);
|
|
143
174
|
if (!unsaved) return keep("its uncommitted changes can't be checked");
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
`${stray} commit${stray === 1 ? " is" : "s are"} on it that the merged PR didn't include and no remote has`
|
|
152
|
-
);
|
|
153
|
-
return { action: "remove", why: "its branch is merged", repo };
|
|
154
|
-
}
|
|
155
|
-
const base = session.base_branch || (await getDefaultBranch(repo));
|
|
156
|
-
const own = await ownCommits(path, base).catch(() => null);
|
|
157
|
-
if (own === null) return keep(`its base ${base} can't be found to compare`);
|
|
158
|
-
if (own > 0)
|
|
159
|
-
return keep(
|
|
160
|
-
`its branch has ${own} commit${own === 1 ? "" : "s"} not merged into ${base}`
|
|
161
|
-
);
|
|
175
|
+
const why = [
|
|
176
|
+
unsaved.length
|
|
177
|
+
? `it has uncommitted changes (${describeChanges(unsaved)})`
|
|
178
|
+
: null,
|
|
179
|
+
await commitsKeep(path, repo, session, merged),
|
|
180
|
+
].filter((w) => w !== null);
|
|
181
|
+
if (why.length) return keep(why.join(", and "));
|
|
162
182
|
return {
|
|
163
183
|
action: "remove",
|
|
164
|
-
why:
|
|
184
|
+
why: merged
|
|
185
|
+
? "its branch is merged"
|
|
186
|
+
: "its branch has no commits of its own",
|
|
165
187
|
repo,
|
|
166
188
|
};
|
|
167
189
|
}
|
|
@@ -229,8 +229,8 @@ async function mergeOne(
|
|
|
229
229
|
}
|
|
230
230
|
const seen = `${pr.head}:${failures(verdict.text)}`;
|
|
231
231
|
if (task.merge_armed_seen === seen) return;
|
|
232
|
-
// Counted
|
|
233
|
-
// goes to Saad.
|
|
232
|
+
// Counted as sign_off would count it: once per head (the database keys
|
|
233
|
+
// it), and a gate failing on a second commit goes to Saad.
|
|
234
234
|
if (!stillArmed(workspaceId, task.id)) return;
|
|
235
235
|
const counted = await deps.judge(workspaceId, task);
|
|
236
236
|
if (counted.ok) return merge(workspaceId, task, counted, deps);
|
|
@@ -54,7 +54,7 @@ When the base moves and a task's PR conflicts with it, AgentOS rebases it onto t
|
|
|
54
54
|
|
|
55
55
|
Tasks are woken by AgentOS itself when a check fails on their PR's head (with the failing step and the log around the error) and when a collaborator reviews or comments on their PR, and you get one "task X woken: ..." line for each. Don't message a task about the same failure or review; act only if it doesn't fix it.
|
|
56
56
|
|
|
57
|
-
"Not yet" (CI running or settling, no review of this commit yet) is not a failure: wait for the event and try again. A failure counts against the task
|
|
57
|
+
"Not yet" (CI running or settling, no review of this commit yet) is not a failure: wait for the event and try again. A failure counts against the task once per commit, whichever path judges that commit and however often (a review that blocks counts when its verdict arrives); the same gate failing again on a different commit goes to Saad, with a note in your chat, and from then on only he merges or drops that task. Don't retry it; say so in your chat and move on. ${mergeApprovalsOn() ? APPROVALS_ON : APPROVALS_OFF} \`land\` merges a whole stack only if every open item passes the same gates, and judges each one again at its own head right before merging it.
|
|
58
58
|
|
|
59
59
|
## Brakes
|
|
60
60
|
|
|
@@ -6,7 +6,6 @@ import { AskRefused, raiseAsk, workSubject } from "./asks";
|
|
|
6
6
|
import {
|
|
7
7
|
failureOf,
|
|
8
8
|
markEscalated,
|
|
9
|
-
PER_COMMIT_GATES,
|
|
10
9
|
recordFailure,
|
|
11
10
|
type GateOutcome,
|
|
12
11
|
} from "./gates";
|
|
@@ -20,7 +19,8 @@ export function escalate(
|
|
|
20
19
|
url: string,
|
|
21
20
|
sha: string | null = null
|
|
22
21
|
): string {
|
|
23
|
-
if (!failureOf(task.id, gate))
|
|
22
|
+
if (!failureOf(task.id, gate))
|
|
23
|
+
recordFailure(workspaceId, task.id, gate, why, sha);
|
|
24
24
|
markEscalated(task.id, gate);
|
|
25
25
|
let created = false;
|
|
26
26
|
try {
|
|
@@ -50,8 +50,9 @@ export function escalate(
|
|
|
50
50
|
return `Escalated to Saad: ${why}. It's on his asks list; his answer reaches you as an event. Don't retry; say so in your chat.`;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
// Counts a gate's failure against the task; the
|
|
54
|
-
//
|
|
53
|
+
// Counts a gate's failure on this head against the task; the same gate
|
|
54
|
+
// failing on a second, different head goes to Saad. The same head judged
|
|
55
|
+
// again, from any path, isn't another failure.
|
|
55
56
|
export function countFailure(
|
|
56
57
|
workspaceId: string,
|
|
57
58
|
task: { id: string; name: string },
|
|
@@ -59,12 +60,11 @@ export function countFailure(
|
|
|
59
60
|
url: string,
|
|
60
61
|
sha: string
|
|
61
62
|
): { line: string; escalated: boolean } {
|
|
62
|
-
const
|
|
63
|
-
const n = recordFailure(workspaceId, task.id, o.gate, o.reason ?? "", key);
|
|
63
|
+
const n = recordFailure(workspaceId, task.id, o.gate, o.reason ?? "", sha);
|
|
64
64
|
const line = `- ${o.gate} failed (${n === 1 ? "first" : "again"}): ${o.reason}`;
|
|
65
65
|
if (n < 2 || failureOf(task.id, o.gate)?.escalated_at)
|
|
66
66
|
return { line, escalated: false };
|
|
67
|
-
const why = `the ${o.gate} gate failed
|
|
67
|
+
const why = `the ${o.gate} gate failed on two different commits: ${o.reason}`;
|
|
68
68
|
return {
|
|
69
69
|
line: `${line}\n ${escalate(workspaceId, task, o.gate, why, url, sha)}`,
|
|
70
70
|
escalated: true,
|
|
@@ -2,10 +2,12 @@
|
|
|
2
2
|
* The merge gates, ported from dispatch orchestrate. Each one passes, waits
|
|
3
3
|
* (not ready yet: CI running or settling, no review of this commit yet),
|
|
4
4
|
* fails, or goes straight to Saad (no CI at all). Only a failure counts
|
|
5
|
-
* against the task, and
|
|
6
|
-
* instead of being retried.
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* against the task, and a gate failing again on a different commit goes to
|
|
6
|
+
* Saad instead of being retried. A failure counts once per task, gate and
|
|
7
|
+
* head commit, whichever path judged it (the armed watcher, CI finishing,
|
|
8
|
+
* sign_off, done, land, a review's verdict arriving): judging the same head
|
|
9
|
+
* again is never another failure. A pass doesn't reset the count, so a gate
|
|
10
|
+
* that fails on A, passes on B and fails on C has failed twice.
|
|
9
11
|
*/
|
|
10
12
|
|
|
11
13
|
import { db } from "../db";
|
|
@@ -170,12 +172,10 @@ export interface FailureRow {
|
|
|
170
172
|
last_sha: string | null;
|
|
171
173
|
}
|
|
172
174
|
|
|
173
|
-
//
|
|
174
|
-
//
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
// Counts one failure; returns the count now. With `sha`, a failure of the
|
|
178
|
-
// commit last counted for this gate isn't counted again.
|
|
175
|
+
// Counts one failure; returns the count now. With `sha`, a failure is
|
|
176
|
+
// counted once per commit: the (task, gate, sha) key in
|
|
177
|
+
// orchestrator_gate_failure_commits decides, so neither a restart nor two
|
|
178
|
+
// evaluations at once count the same head twice.
|
|
179
179
|
export function recordFailure(
|
|
180
180
|
workspaceId: string,
|
|
181
181
|
sessionId: string,
|
|
@@ -183,14 +183,34 @@ export function recordFailure(
|
|
|
183
183
|
reason: string,
|
|
184
184
|
sha: string | null = null
|
|
185
185
|
): number {
|
|
186
|
+
return db
|
|
187
|
+
.transaction(countOnce)
|
|
188
|
+
.immediate(workspaceId, sessionId, gate, reason, sha);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function countOnce(
|
|
192
|
+
workspaceId: string,
|
|
193
|
+
sessionId: string,
|
|
194
|
+
gate: string,
|
|
195
|
+
reason: string,
|
|
196
|
+
sha: string | null
|
|
197
|
+
): number {
|
|
198
|
+
const fresh =
|
|
199
|
+
sha === null ||
|
|
200
|
+
db
|
|
201
|
+
.prepare(
|
|
202
|
+
`INSERT OR IGNORE INTO orchestrator_gate_failure_commits (session_id, gate, sha, workspace_id, reason)
|
|
203
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
204
|
+
)
|
|
205
|
+
.run(sessionId, gate, sha, workspaceId, reason).changes > 0;
|
|
186
206
|
db.prepare(
|
|
187
207
|
`INSERT INTO orchestrator_gate_failures (session_id, gate, workspace_id, count, last_reason, last_sha)
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
).run(sessionId, gate, workspaceId, reason, sha);
|
|
208
|
+
VALUES (?, ?, ?, 1, ?, ?)
|
|
209
|
+
ON CONFLICT(session_id, gate) DO UPDATE SET
|
|
210
|
+
count = count + ?,
|
|
211
|
+
last_reason = excluded.last_reason,
|
|
212
|
+
last_sha = excluded.last_sha`
|
|
213
|
+
).run(sessionId, gate, workspaceId, reason, sha, fresh ? 1 : 0);
|
|
194
214
|
return failureOf(sessionId, gate)!.count;
|
|
195
215
|
}
|
|
196
216
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* commit that was judged. With merge approvals on (merge-approvals.ts),
|
|
4
4
|
* anything touching CI config, build and hook scripts, agent config, deploys
|
|
5
5
|
* or secrets handling goes to Saad whatever the gates say. A repository with
|
|
6
|
-
* no CI and a gate failing
|
|
6
|
+
* no CI and a gate failing on two different commits go to him either way. Once a task
|
|
7
7
|
* is with Saad the orchestrator doesn't merge it; he signs it off or drops it.
|
|
8
8
|
*/
|
|
9
9
|
|
package/lib/tasks/finish.ts
CHANGED
|
@@ -75,8 +75,14 @@ async function cleanup(
|
|
|
75
75
|
export const signingOff = new Set<string>();
|
|
76
76
|
|
|
77
77
|
// What happens after a merge: restack anything stacked on it, then remove the
|
|
78
|
-
// session, worktree and branch. Kept per task so Land can wait for
|
|
79
|
-
|
|
78
|
+
// session, worktree and branch. Kept per task so Land and done can wait for
|
|
79
|
+
// it; on globalThis because the watcher's merges run in server.ts and done in
|
|
80
|
+
// a route bundle, which load this module separately.
|
|
81
|
+
const g = globalThis as unknown as {
|
|
82
|
+
__agentosAfterMerge?: Map<string, Promise<void>>;
|
|
83
|
+
};
|
|
84
|
+
const afterMerge: Map<string, Promise<void>> = (g.__agentosAfterMerge ??=
|
|
85
|
+
new Map());
|
|
80
86
|
|
|
81
87
|
async function finishMerge(
|
|
82
88
|
session: Session,
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Removals of a worktree in progress, visible to every reader. `git
|
|
3
|
+
* worktree remove` takes seconds on a tree with cloned dependencies, and a
|
|
4
|
+
* tree read halfway through lists every file it already deleted as an
|
|
5
|
+
* uncommitted change. The removal that a merge's cleanup starts runs in
|
|
6
|
+
* server.ts while done runs in a route bundle (each loads its own copy of
|
|
7
|
+
* a module) or another process, so besides the in-memory record a marker
|
|
8
|
+
* file beside the worktree says a removal is running.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import fs from "fs";
|
|
12
|
+
import os from "os";
|
|
13
|
+
import path from "path";
|
|
14
|
+
|
|
15
|
+
export interface Removal {
|
|
16
|
+
done: Promise<void>;
|
|
17
|
+
// The branch the worktree was on, read before anything was removed.
|
|
18
|
+
branch: Promise<string | null>;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const g = globalThis as unknown as { __agentosRemovals?: Map<string, Removal> };
|
|
22
|
+
export const removals: Map<string, Removal> = (g.__agentosRemovals ??=
|
|
23
|
+
new Map());
|
|
24
|
+
|
|
25
|
+
export const removalKey = (p: string) =>
|
|
26
|
+
path.resolve(p.replace(/^~/, os.homedir()));
|
|
27
|
+
|
|
28
|
+
// Beside the worktree, not in it: git never lists it and the removal
|
|
29
|
+
// doesn't delete it.
|
|
30
|
+
export const removalMarker = (key: string) =>
|
|
31
|
+
path.join(path.dirname(key), `.${path.basename(key)}.removing`);
|
|
32
|
+
|
|
33
|
+
// A marker older than this, or whose process is gone, is from a removal
|
|
34
|
+
// cut short (a restart) and no longer means anything.
|
|
35
|
+
const STALE_MS = 10 * 60_000;
|
|
36
|
+
|
|
37
|
+
// Returns what it wrote, so only the removal that wrote it clears it.
|
|
38
|
+
export function markRemoving(key: string): string | null {
|
|
39
|
+
const marker = removalMarker(key);
|
|
40
|
+
const tmp = `${marker}.${process.pid}`;
|
|
41
|
+
const body = JSON.stringify({ pid: process.pid, at: Date.now() });
|
|
42
|
+
try {
|
|
43
|
+
fs.writeFileSync(tmp, body);
|
|
44
|
+
fs.renameSync(tmp, marker);
|
|
45
|
+
return body;
|
|
46
|
+
} catch {
|
|
47
|
+
// No marker only means other processes can't see this removal.
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function unmarkRemoving(key: string, body: string): void {
|
|
53
|
+
const marker = removalMarker(key);
|
|
54
|
+
try {
|
|
55
|
+
if (fs.readFileSync(marker, "utf8") === body) fs.rmSync(marker);
|
|
56
|
+
} catch {
|
|
57
|
+
// Already gone.
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const alive = (pid: number) => {
|
|
62
|
+
try {
|
|
63
|
+
process.kill(pid, 0);
|
|
64
|
+
return true;
|
|
65
|
+
} catch (e) {
|
|
66
|
+
return (e as NodeJS.ErrnoException).code === "EPERM";
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
function markerLive(key: string): boolean {
|
|
71
|
+
let raw: string;
|
|
72
|
+
try {
|
|
73
|
+
raw = fs.readFileSync(removalMarker(key), "utf8");
|
|
74
|
+
} catch {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
try {
|
|
78
|
+
const { pid, at } = JSON.parse(raw) as { pid: number; at: number };
|
|
79
|
+
return Date.now() - at < STALE_MS && alive(pid);
|
|
80
|
+
} catch {
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Whether any code path or process is removing this worktree now.
|
|
86
|
+
export const removalRunning = (worktreePath: string) => {
|
|
87
|
+
const key = removalKey(worktreePath);
|
|
88
|
+
return removals.has(key) || markerLive(key);
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
const sleep = (ms: number) =>
|
|
92
|
+
new Promise<void>((r) => setTimeout(r, Math.max(0, ms)));
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Resolves once no removal of this worktree is running: true, or false
|
|
96
|
+
* when one still is after `timeoutMs`. `foreignOnly` waits only for one
|
|
97
|
+
* another process (or a copy of this module) marked.
|
|
98
|
+
*/
|
|
99
|
+
export async function removalSettled(
|
|
100
|
+
worktreePath: string,
|
|
101
|
+
timeoutMs = 60_000,
|
|
102
|
+
{ foreignOnly = false } = {}
|
|
103
|
+
): Promise<boolean> {
|
|
104
|
+
const key = removalKey(worktreePath);
|
|
105
|
+
const deadline = Date.now() + timeoutMs;
|
|
106
|
+
for (;;) {
|
|
107
|
+
const own = foreignOnly ? undefined : removals.get(key);
|
|
108
|
+
if (!own && !markerLive(key)) return true;
|
|
109
|
+
const left = deadline - Date.now();
|
|
110
|
+
if (left <= 0) return false;
|
|
111
|
+
await (own
|
|
112
|
+
? Promise.race([own.done.catch(() => {}), sleep(left)])
|
|
113
|
+
: sleep(Math.min(250, left)));
|
|
114
|
+
}
|
|
115
|
+
}
|
package/lib/worktrees.ts
CHANGED
|
@@ -14,6 +14,14 @@ import {
|
|
|
14
14
|
} from "./git";
|
|
15
15
|
import { installWorktreeHooks } from "./worktree-hooks";
|
|
16
16
|
import { gitResult, isGitError, worktreeList } from "./git-exec";
|
|
17
|
+
import {
|
|
18
|
+
markRemoving,
|
|
19
|
+
removalKey,
|
|
20
|
+
removalRunning,
|
|
21
|
+
removalSettled,
|
|
22
|
+
removals,
|
|
23
|
+
unmarkRemoving,
|
|
24
|
+
} from "./worktree-removal";
|
|
17
25
|
|
|
18
26
|
const git = (cwd: string, args: string[], timeout: number) =>
|
|
19
27
|
gitResult(cwd, args, { timeout });
|
|
@@ -252,28 +260,10 @@ async function undoFailedAdd(
|
|
|
252
260
|
await quiet(["branch", "-D", "--", branch]);
|
|
253
261
|
}
|
|
254
262
|
|
|
255
|
-
// Removals in progress, by path, with the branch each one's worktree was
|
|
256
|
-
// on. `git worktree remove` takes seconds on a tree with cloned
|
|
257
|
-
// dependencies, and a tree read halfway through looks like every file it
|
|
258
|
-
// already deleted is an uncommitted change.
|
|
259
|
-
const removing = new Map<
|
|
260
|
-
string,
|
|
261
|
-
{ done: Promise<void>; branch: Promise<string | null> }
|
|
262
|
-
>();
|
|
263
|
-
|
|
264
|
-
const removalKey = (p: string) => path.resolve(resolvePath(p));
|
|
265
|
-
|
|
266
|
-
// Resolves once no removal of this worktree is running.
|
|
267
|
-
export async function removalSettled(worktreePath: string): Promise<void> {
|
|
268
|
-
await removing.get(removalKey(worktreePath))?.done.catch(() => {});
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
export const removalRunning = (worktreePath: string) =>
|
|
272
|
-
removing.has(removalKey(worktreePath));
|
|
273
|
-
|
|
274
263
|
/**
|
|
275
264
|
* Delete a worktree and optionally its branch. A second call while one is
|
|
276
|
-
* running joins it, and still deletes the branch when it asks to
|
|
265
|
+
* running joins it, and still deletes the branch when it asks to; one
|
|
266
|
+
* another process is running is waited for first.
|
|
277
267
|
*/
|
|
278
268
|
export function deleteWorktree(
|
|
279
269
|
worktreePath: string,
|
|
@@ -282,22 +272,35 @@ export function deleteWorktree(
|
|
|
282
272
|
): Promise<void> {
|
|
283
273
|
const key = removalKey(worktreePath);
|
|
284
274
|
const repo = resolvePath(projectPath);
|
|
285
|
-
const running =
|
|
275
|
+
const running = removals.get(key);
|
|
286
276
|
if (running)
|
|
287
277
|
return deleteBranch
|
|
288
278
|
? running.done.then(async () => dropBranch(repo, await running.branch))
|
|
289
279
|
: running.done;
|
|
280
|
+
// One another process is running goes first, and keeps its own marker.
|
|
281
|
+
const foreign = removalRunning(key)
|
|
282
|
+
? removalSettled(key, undefined, { foreignOnly: true })
|
|
283
|
+
: null;
|
|
290
284
|
// Read before anything is removed.
|
|
291
|
-
const branch =
|
|
292
|
-
|
|
293
|
-
(
|
|
294
|
-
|
|
285
|
+
const branch = (async () => {
|
|
286
|
+
await foreign;
|
|
287
|
+
return git(key, ["rev-parse", "--abbrev-ref", "HEAD"], 5000).then(
|
|
288
|
+
({ stdout }) => stdout.trim() || null,
|
|
289
|
+
() => null
|
|
290
|
+
);
|
|
291
|
+
})();
|
|
292
|
+
let marker: string | null = null;
|
|
295
293
|
const done = (async () => {
|
|
296
294
|
await branch;
|
|
295
|
+
// Before anything is removed, so no reader catches it half-deleted.
|
|
296
|
+
marker = markRemoving(key);
|
|
297
297
|
await removeWorktree(key, repo);
|
|
298
298
|
if (deleteBranch) await dropBranch(repo, await branch);
|
|
299
|
-
})().finally(() =>
|
|
300
|
-
|
|
299
|
+
})().finally(() => {
|
|
300
|
+
removals.delete(key);
|
|
301
|
+
if (marker) unmarkRemoving(key, marker);
|
|
302
|
+
});
|
|
303
|
+
removals.set(key, { done, branch });
|
|
301
304
|
return done;
|
|
302
305
|
}
|
|
303
306
|
|