@saadnvd1/agent-os 0.3.631 → 0.3.637
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/api/tasks/[id]/approve-plan/route.ts +6 -1
- package/app/api/tasks/[id]/export/route.ts +3 -2
- package/app/api/tasks/[id]/plan/route.ts +37 -0
- package/app/api/tasks/route.ts +4 -1
- package/app/globals.css +28 -0
- package/components/Chat/ChatPanel.tsx +2 -11
- package/components/Chat/EventTurns.tsx +1 -7
- package/components/Chat/Markdown.tsx +1 -0
- package/components/Chat/Messages.tsx +1 -1
- package/lib/approvals/enable.ts +9 -2
- package/lib/chat/events.ts +4 -0
- package/lib/chat/group.ts +3 -1
- package/lib/chat/reading-order.ts +50 -0
- package/lib/chat/turns.ts +22 -3
- package/lib/chat/worker/host.ts +3 -0
- package/lib/db/migrations/20261015T1600_plan_waits_for_caller.ts +19 -0
- package/lib/db/types.ts +6 -1
- package/lib/env-setup.ts +17 -4
- package/lib/models/plan-step.ts +75 -24
- package/lib/tasks/held.ts +23 -0
- package/lib/tasks/hold-checks.ts +134 -0
- package/lib/tasks/holds.ts +281 -0
- package/lib/tasks/import.ts +11 -3
- package/lib/tasks/index.ts +25 -3
- package/lib/tasks/move-bundle.ts +17 -2
- package/lib/tasks/move-guard.ts +9 -1
- package/lib/tasks/move.ts +5 -4
- package/lib/tasks/pause.ts +2 -31
- package/lib/tasks/placement.ts +16 -16
- package/lib/tasks/plan-view.ts +11 -3
- package/lib/tasks/queue-candidates.ts +151 -0
- package/lib/tasks/queue-placement.ts +155 -0
- package/lib/tasks/queue-rows.ts +236 -0
- package/lib/tasks/queue-start.ts +211 -0
- package/lib/tasks/queue-tick.ts +148 -0
- package/lib/tasks/queue-types.ts +141 -0
- package/lib/tasks/queue-view.ts +112 -0
- package/lib/tasks/queue.ts +53 -1096
- package/lib/tasks/remote-plan-path.ts +7 -0
- package/lib/tasks/remote-plan.ts +308 -0
- package/lib/tasks/remote.ts +18 -1
- package/lib/tasks/schedule.ts +44 -147
- package/package.json +1 -1
- package/scripts/check +55 -7
- package/lib/tasks/auto-hold.ts +0 -17
package/lib/tasks/queue.ts
CHANGED
|
@@ -4,364 +4,71 @@
|
|
|
4
4
|
* (`max_running_tasks`, off unless set), or for another task to finish
|
|
5
5
|
* (`after`). The stack watcher's loop ticks it (lib/stacks/tick.ts), and the
|
|
6
6
|
* load monitor nudges it when a held machine calms down. Each tick scores
|
|
7
|
-
* the candidates (
|
|
8
|
-
* holds
|
|
9
|
-
*
|
|
10
|
-
* load rules out goes to a linked
|
|
7
|
+
* the candidates (schedule.ts: priority, age, a resume bonus), lists each
|
|
8
|
+
* one's holds (holds.ts: after, Pause, the usage window, machine load,
|
|
9
|
+
* simulators, start spacing, then the slot limit) and starts what has none,
|
|
10
|
+
* best first. A start the home machine's load rules out goes to a linked
|
|
11
|
+
* machine that is free (queue-placement.ts, placement.ts).
|
|
12
|
+
*
|
|
13
|
+
* This file is the request side (queueing, start now, priority, removal) and
|
|
14
|
+
* the public face: the parts live in queue-types, queue-rows, queue-view,
|
|
15
|
+
* queue-candidates, queue-placement, queue-start and queue-tick.
|
|
11
16
|
*
|
|
12
17
|
* A row's id is the session id it starts as, and it's claimed ('starting')
|
|
13
18
|
* before the task is created, so a restart never starts one twice. Nothing
|
|
14
19
|
* is deleted: a started or removed row stays as history.
|
|
15
20
|
*/
|
|
16
21
|
|
|
17
|
-
import type { ModelOverride } from "../models/resolve";
|
|
18
|
-
import type { AgentType } from "../providers";
|
|
19
22
|
import { randomUUID } from "crypto";
|
|
20
|
-
import { db
|
|
23
|
+
import { db } from "../db";
|
|
21
24
|
import { getProject } from "../projects";
|
|
22
25
|
import { isRemoteHost } from "../hosts";
|
|
23
|
-
import { getWorkspace } from "../workspaces";
|
|
24
26
|
import { isPaused } from "../orchestrator/pause";
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
recordStart,
|
|
31
|
-
} from "../orchestrator/brakes";
|
|
32
|
-
import { queueEvent } from "../orchestrator/events";
|
|
33
|
-
import { readUsage, windowRefusal } from "../orchestrator/usage";
|
|
34
|
-
import { createTask } from "./index";
|
|
35
|
-
import {
|
|
36
|
-
discardQueuedHeld,
|
|
37
|
-
markQueuedHeldDelivered,
|
|
38
|
-
queuedHeldMessages,
|
|
39
|
-
queuedLabel,
|
|
40
|
-
queuedMessagesSection,
|
|
41
|
-
} from "./queue-held";
|
|
42
|
-
import { expandHome } from "./session";
|
|
43
|
-
import {
|
|
44
|
-
DEFAULT_PRIORITY,
|
|
45
|
-
isPriority,
|
|
46
|
-
type Priority,
|
|
47
|
-
type WaitCode,
|
|
48
|
-
} from "./schedule-types";
|
|
49
|
-
import {
|
|
50
|
-
machineHold,
|
|
51
|
-
planStarts,
|
|
52
|
-
scoreParts,
|
|
53
|
-
type Candidate,
|
|
54
|
-
type Group,
|
|
55
|
-
type Hold,
|
|
56
|
-
} from "./schedule";
|
|
57
|
-
import {
|
|
58
|
-
heavyNeeds,
|
|
59
|
-
meetsNeeds,
|
|
60
|
-
needsFromSetting,
|
|
61
|
-
needsOf,
|
|
62
|
-
parseNeeds,
|
|
63
|
-
place,
|
|
64
|
-
unplaceable,
|
|
65
|
-
wantsPlacing,
|
|
66
|
-
type Need,
|
|
67
|
-
type Placement,
|
|
68
|
-
type PeerOption,
|
|
69
|
-
} from "./placement";
|
|
70
|
-
import {
|
|
71
|
-
LOCAL,
|
|
72
|
-
localView,
|
|
73
|
-
machineCapabilities,
|
|
74
|
-
recordAutoStart,
|
|
75
|
-
viewOf,
|
|
76
|
-
} from "../load/machines";
|
|
77
|
-
import { peerReading, peerSource, refreshPeers } from "../hosts/peer-load";
|
|
78
|
-
import { hostLink, linkedHostIds } from "../hosts/remote-api";
|
|
79
|
-
import { hostTasks } from "./remote";
|
|
80
|
-
import { projectMachines } from "../hosts/project-machines";
|
|
81
|
-
import {
|
|
82
|
-
type DueResume,
|
|
83
|
-
claimResume,
|
|
84
|
-
dueResumes,
|
|
85
|
-
resumeHoldFor,
|
|
86
|
-
setPauseNote,
|
|
87
|
-
} from "./pause";
|
|
88
|
-
import { orchestratorOf, type StartVia } from "./starter";
|
|
27
|
+
import { BrakeRefused, recordStart } from "../orchestrator/brakes";
|
|
28
|
+
import { discardQueuedHeld, queuedLabel } from "./queue-held";
|
|
29
|
+
import { DEFAULT_PRIORITY, isPriority, type Priority } from "./schedule-types";
|
|
30
|
+
import { meetsNeeds, needsFromSetting, parseNeeds } from "./placement";
|
|
31
|
+
import { machineCapabilities } from "../load/machines";
|
|
89
32
|
import { claimPiece } from "../plans/claim";
|
|
33
|
+
import { loadHeldStart, needsUnmetHere, requestNeeds } from "./hold-checks";
|
|
34
|
+
import {
|
|
35
|
+
limitOf,
|
|
36
|
+
patch,
|
|
37
|
+
reserved,
|
|
38
|
+
resolveAfter,
|
|
39
|
+
row,
|
|
40
|
+
runningIn,
|
|
41
|
+
sessionOf,
|
|
42
|
+
slotState,
|
|
43
|
+
startingIn,
|
|
44
|
+
stillWorking,
|
|
45
|
+
waitingIn,
|
|
46
|
+
workspaceOf,
|
|
47
|
+
} from "./queue-rows";
|
|
48
|
+
import { announce, startRow } from "./queue-start";
|
|
49
|
+
import { queuedView } from "./queue-view";
|
|
50
|
+
import { tickQueueSoon } from "./queue-tick";
|
|
51
|
+
import type {
|
|
52
|
+
ForcedStart,
|
|
53
|
+
QueueRequest,
|
|
54
|
+
QueueRow,
|
|
55
|
+
QueuedTaskView,
|
|
56
|
+
StartOutcome,
|
|
57
|
+
} from "./queue-types";
|
|
90
58
|
|
|
91
|
-
export
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
// A per-role model map (lib/models), when that's what it was given.
|
|
105
|
-
model_overrides: string | null;
|
|
106
|
-
// The starter's plan decision (1/0, null: not stated), its reason, and the
|
|
107
|
-
// orchestrator session that started it.
|
|
108
|
-
plan: number | null;
|
|
109
|
-
plan_reason: string | null;
|
|
110
|
-
plan_only: number | null;
|
|
111
|
-
started_by: string | null;
|
|
112
|
-
parent_session_id: string | null;
|
|
113
|
-
started_by_device: string | null;
|
|
114
|
-
started_via: string | null;
|
|
115
|
-
view: "chat" | "terminal" | null;
|
|
116
|
-
agent_type: string | null;
|
|
117
|
-
base_branch: string | null;
|
|
118
|
-
host_id: string | null;
|
|
119
|
-
// A task (or queued task) it waits on, by id.
|
|
120
|
-
after_task: string | null;
|
|
121
|
-
// "after any": the tasks running when it was queued, as JSON; the first
|
|
122
|
-
// of them to finish lets it go.
|
|
123
|
-
after_any: string | null;
|
|
124
|
-
// The finish (a task id) this "after any" waiter took; one finish lets one
|
|
125
|
-
// waiter go (a unique index keeps it so).
|
|
126
|
-
after_any_used: string | null;
|
|
127
|
-
priority: Priority;
|
|
128
|
-
// What it needs from a machine, as JSON, when the starter said.
|
|
129
|
-
machine_needs: string | null;
|
|
130
|
-
// The code behind `note`, set by the scheduler.
|
|
131
|
-
wait_code: WaitCode | null;
|
|
132
|
-
// The orchestrator's workspace, when the orchestrator queued it: its
|
|
133
|
-
// start goes through that orchestrator's brakes.
|
|
134
|
-
origin_workspace_id: string | null;
|
|
135
|
-
position: number;
|
|
136
|
-
status: QueueStatus;
|
|
137
|
-
note: string | null;
|
|
138
|
-
error: string | null;
|
|
139
|
-
attempts: number;
|
|
140
|
-
created_at: string;
|
|
141
|
-
started_at: string | null;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
// What the UI and tools see of a queued task. No server imports needed.
|
|
145
|
-
export interface QueuedTaskView {
|
|
146
|
-
id: string;
|
|
147
|
-
name: string;
|
|
148
|
-
prompt: string;
|
|
149
|
-
projectId: string;
|
|
150
|
-
projectName: string | null;
|
|
151
|
-
// The workspace whose line it's in, or null for a project without one.
|
|
152
|
-
workspaceId: string | null;
|
|
153
|
-
status: QueueStatus;
|
|
154
|
-
// 1-based place in its workspace's line.
|
|
155
|
-
position: number;
|
|
156
|
-
// What it waits on: a task's name, or "any running task".
|
|
157
|
-
after: string | null;
|
|
158
|
-
// The task it waits on, by id, when it waits on one.
|
|
159
|
-
afterId: string | null;
|
|
160
|
-
note: string | null;
|
|
161
|
-
error: string | null;
|
|
162
|
-
// The orchestrator session that queued it, when one did.
|
|
163
|
-
startedBy: string | null;
|
|
164
|
-
createdAt: string;
|
|
165
|
-
priority: Priority;
|
|
166
|
-
// Why it isn't starting, from the scheduler; null: nothing but its turn.
|
|
167
|
-
wait: WaitCode | null;
|
|
168
|
-
// Its place in the score order within its line, and how many are in it.
|
|
169
|
-
rank: number;
|
|
170
|
-
of: number;
|
|
171
|
-
// Messages held for it until it starts.
|
|
172
|
-
held: number;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
export const START_ATTEMPTS = 3;
|
|
176
|
-
const IN_FLIGHT = ["running", "moving"];
|
|
177
|
-
|
|
178
|
-
const row = (id: string) =>
|
|
179
|
-
db.prepare(`SELECT * FROM task_queue WHERE id = ?`).get(id) as
|
|
180
|
-
| QueueRow
|
|
181
|
-
| undefined;
|
|
182
|
-
|
|
183
|
-
const sessionOf = (id: string) =>
|
|
184
|
-
db.prepare(`SELECT * FROM sessions WHERE id = ?`).get(id) as
|
|
185
|
-
| Session
|
|
186
|
-
| undefined;
|
|
187
|
-
|
|
188
|
-
function patch(id: string, fields: Partial<QueueRow>): void {
|
|
189
|
-
const keys = Object.keys(fields);
|
|
190
|
-
if (!keys.length) return;
|
|
191
|
-
db.prepare(
|
|
192
|
-
`UPDATE task_queue SET ${keys.map((k) => `${k} = @${k}`).join(", ")} WHERE id = @id`
|
|
193
|
-
).run({ ...fields, id });
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
const workspaceOf = (projectId: string) =>
|
|
197
|
-
getProject(projectId)?.workspace_id ?? null;
|
|
198
|
-
|
|
199
|
-
// Running tasks hold their slot until they finish (merged, dropped, done),
|
|
200
|
-
// PR open or not; a queued task being started counts too. A paused task
|
|
201
|
-
// gives its slot back until it resumes. A quick change
|
|
202
|
-
// runs no agent, so it holds none. Without a
|
|
203
|
-
// workspace, the project is the scope.
|
|
204
|
-
function runningIn(
|
|
205
|
-
scope: { workspaceId: string | null; projectId: string },
|
|
206
|
-
forSlots = false
|
|
207
|
-
) {
|
|
208
|
-
const where = scope.workspaceId
|
|
209
|
-
? `p.workspace_id = @ws`
|
|
210
|
-
: `p.id = @project AND p.workspace_id IS NULL`;
|
|
211
|
-
return (
|
|
212
|
-
db
|
|
213
|
-
.prepare(
|
|
214
|
-
`SELECT s.id FROM sessions s JOIN projects p ON p.id = s.project_id
|
|
215
|
-
WHERE ${where} AND s.archived_at IS NULL AND s.quick_of IS NULL
|
|
216
|
-
AND (s.pause_state IS NULL OR s.pause_state != 'paused')
|
|
217
|
-
${forSlots ? `AND NOT (COALESCE(s.plan_only, 0) = 1 AND COALESCE(s.plan_phase, '') != 'implementing')` : ""}
|
|
218
|
-
AND s.task_status IN (${IN_FLIGHT.map((s) => `'${s}'`).join(", ")})`
|
|
219
|
-
)
|
|
220
|
-
.all({ ws: scope.workspaceId, project: scope.projectId }) as {
|
|
221
|
-
id: string;
|
|
222
|
-
}[]
|
|
223
|
-
).map((r) => r.id);
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
function startingIn(workspaceId: string): number {
|
|
227
|
-
return (
|
|
228
|
-
(reserved.get(workspaceId) ?? 0) +
|
|
229
|
-
(
|
|
230
|
-
db
|
|
231
|
-
.prepare(
|
|
232
|
-
`SELECT COUNT(*) AS n FROM task_queue q JOIN projects p ON p.id = q.project_id
|
|
233
|
-
WHERE q.status = 'starting' AND p.workspace_id = ?
|
|
234
|
-
AND q.plan_only IS NULL`
|
|
235
|
-
)
|
|
236
|
-
.get(workspaceId) as { n: number }
|
|
237
|
-
).n
|
|
238
|
-
);
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
const limitOf = (workspaceId: string | null) =>
|
|
242
|
-
workspaceId ? (getWorkspace(workspaceId)?.max_running_tasks ?? null) : null;
|
|
243
|
-
|
|
244
|
-
// Finished, as far as a wait on it goes: merged, dropped, done or handed to
|
|
245
|
-
// another machine, or a queued task that was removed or failed to start.
|
|
246
|
-
export function hasFinished(taskId: string): boolean {
|
|
247
|
-
const s = sessionOf(taskId);
|
|
248
|
-
if (s) return !!s.archived_at || !IN_FLIGHT.includes(s.task_status ?? "");
|
|
249
|
-
const q = row(taskId);
|
|
250
|
-
return !q || (q.status !== "queued" && q.status !== "starting");
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
// Still something to wait for under "any": in flight, and not a task whose
|
|
254
|
-
// PR was closed or merged and which nobody finished (that one never will).
|
|
255
|
-
function stillWorking(taskId: string): boolean {
|
|
256
|
-
if (hasFinished(taskId)) return false;
|
|
257
|
-
const s = sessionOf(taskId);
|
|
258
|
-
return !s || (s.pr_status !== "closed" && s.pr_status !== "merged");
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
export function afterMet(r: QueueRow): boolean {
|
|
262
|
-
if (r.after_task && !hasFinished(r.after_task)) return false;
|
|
263
|
-
if (r.after_any && !r.after_any_used) {
|
|
264
|
-
const ids = JSON.parse(r.after_any) as string[];
|
|
265
|
-
if (ids.length && !ids.some((id) => !stillWorking(id))) return false;
|
|
266
|
-
}
|
|
267
|
-
return true;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
// A task named by id, id prefix or name, running or queued; in one
|
|
271
|
-
// workspace when given (the orchestrator's).
|
|
272
|
-
export function resolveAfter(
|
|
273
|
-
ref: string,
|
|
274
|
-
workspaceId: string | null
|
|
275
|
-
): { id: string; name: string } {
|
|
276
|
-
const want = ref.trim();
|
|
277
|
-
const lower = want.toLowerCase();
|
|
278
|
-
const scope = workspaceId ? `AND p.workspace_id = @ws` : "";
|
|
279
|
-
const candidates = db
|
|
280
|
-
.prepare(
|
|
281
|
-
`SELECT s.id, s.name FROM sessions s JOIN projects p ON p.id = s.project_id
|
|
282
|
-
WHERE s.archived_at IS NULL AND s.task_status IN ('running', 'moving') ${scope}
|
|
283
|
-
UNION ALL
|
|
284
|
-
SELECT q.id, COALESCE(q.name, q.prompt) AS name FROM task_queue q
|
|
285
|
-
JOIN projects p ON p.id = q.project_id
|
|
286
|
-
WHERE q.status IN ('queued', 'starting') ${scope}`
|
|
287
|
-
)
|
|
288
|
-
.all({ ws: workspaceId }) as { id: string; name: string }[];
|
|
289
|
-
const exact = candidates.filter(
|
|
290
|
-
(c) => c.id === want || c.name.toLowerCase() === lower
|
|
291
|
-
);
|
|
292
|
-
const hits = exact.length
|
|
293
|
-
? exact
|
|
294
|
-
: want.length >= 4
|
|
295
|
-
? candidates.filter((c) => c.id.startsWith(want))
|
|
296
|
-
: [];
|
|
297
|
-
if (hits.length === 1) return hits[0];
|
|
298
|
-
if (hits.length > 1)
|
|
299
|
-
throw new Error(
|
|
300
|
-
`"${ref}" matches ${hits.length} tasks: ${hits.map((h) => `${h.name} (${h.id.slice(0, 8)})`).join(", ")}`
|
|
301
|
-
);
|
|
302
|
-
throw new Error(`No running or queued task matches "${ref}"`);
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
export interface QueueRequest {
|
|
306
|
-
projectId: string;
|
|
307
|
-
prompt: string;
|
|
308
|
-
name?: string;
|
|
309
|
-
model?: ModelOverride;
|
|
310
|
-
plan?: boolean;
|
|
311
|
-
planReason?: string | null;
|
|
312
|
-
planOnly?: boolean;
|
|
313
|
-
startedBy?: string | null;
|
|
314
|
-
parentSessionId?: string | null;
|
|
315
|
-
startedByDevice?: string | null;
|
|
316
|
-
startedVia: StartVia;
|
|
317
|
-
// The waiting plan piece it builds, claimed for its queue id.
|
|
318
|
-
pieceId?: number;
|
|
319
|
-
view?: "chat" | "terminal";
|
|
320
|
-
agent?: AgentType;
|
|
321
|
-
baseBranch?: string;
|
|
322
|
-
hostId?: string;
|
|
323
|
-
// A task's id or name, or "any" for whichever running task finishes first.
|
|
324
|
-
after?: string;
|
|
325
|
-
// Set when the orchestrator starts it.
|
|
326
|
-
originWorkspaceId?: string;
|
|
327
|
-
// Where it sits among the queued: urgent, blocking, normal (the default)
|
|
328
|
-
// or background.
|
|
329
|
-
priority?: Priority;
|
|
330
|
-
// "mac" pins it to a Mac; "any" says it needs nothing in particular.
|
|
331
|
-
machine?: "any" | "mac";
|
|
332
|
-
// Start at once, past the running-task limit, instead of queueing for a
|
|
333
|
-
// slot. Still needs the machine it needs, and an orchestrator's start
|
|
334
|
-
// still passes its brakes.
|
|
335
|
-
now?: boolean;
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
// How full the workspace's running-task limit is for this project.
|
|
339
|
-
export function slotState(projectId: string): {
|
|
340
|
-
running: number;
|
|
341
|
-
limit: number | null;
|
|
342
|
-
full: boolean;
|
|
343
|
-
} {
|
|
344
|
-
const workspaceId = workspaceOf(projectId);
|
|
345
|
-
const limit = limitOf(workspaceId);
|
|
346
|
-
if (!workspaceId || limit === null)
|
|
347
|
-
return { running: 0, limit: null, full: false };
|
|
348
|
-
const running =
|
|
349
|
-
runningIn({ workspaceId, projectId }, true).length +
|
|
350
|
-
startingIn(workspaceId);
|
|
351
|
-
return { running, limit, full: running >= limit };
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
const waitingIn = (workspaceId: string) =>
|
|
355
|
-
(
|
|
356
|
-
db
|
|
357
|
-
.prepare(
|
|
358
|
-
`SELECT COUNT(*) AS n FROM task_queue q JOIN projects p ON p.id = q.project_id
|
|
359
|
-
WHERE q.status = 'queued' AND p.workspace_id = ?
|
|
360
|
-
AND q.after_task IS NULL AND q.after_any IS NULL
|
|
361
|
-
AND q.plan_only IS NULL`
|
|
362
|
-
)
|
|
363
|
-
.get(workspaceId) as { n: number }
|
|
364
|
-
).n;
|
|
59
|
+
export { queuedLabel };
|
|
60
|
+
export { START_ATTEMPTS } from "./queue-types";
|
|
61
|
+
export type {
|
|
62
|
+
ForcedStart,
|
|
63
|
+
QueueRequest,
|
|
64
|
+
QueueRow,
|
|
65
|
+
QueueStatus,
|
|
66
|
+
QueuedTaskView,
|
|
67
|
+
StartOutcome,
|
|
68
|
+
} from "./queue-types";
|
|
69
|
+
export { afterMet, hasFinished, resolveAfter, slotState } from "./queue-rows";
|
|
70
|
+
export { findQueued, listQueue, queuedView } from "./queue-view";
|
|
71
|
+
export { tickOnce, tickQueue, tickQueueSoon } from "./queue-tick";
|
|
365
72
|
|
|
366
73
|
// Queues the task when it can't start now: it has an `after`, or its
|
|
367
74
|
// workspace is at its limit (or already has a line waiting for a slot).
|
|
@@ -463,16 +170,6 @@ export function queueIfNeeded(req: QueueRequest): QueuedTaskView | null {
|
|
|
463
170
|
return queuedView(id)!;
|
|
464
171
|
}
|
|
465
172
|
|
|
466
|
-
// Direct starts under way, by workspace: their task rows don't exist until
|
|
467
|
-
// createTask gets that far, so they hold their slot here meanwhile.
|
|
468
|
-
// On globalThis: the custom server's watcher and the route bundles each load
|
|
469
|
-
// their own copy of this module, and must see one set of reservations.
|
|
470
|
-
const g = globalThis as unknown as {
|
|
471
|
-
__agentosQueueReserved?: Map<string, number>;
|
|
472
|
-
__agentosQueueTick?: { ticking: Promise<void> | null };
|
|
473
|
-
};
|
|
474
|
-
const reserved = (g.__agentosQueueReserved ??= new Map<string, number>());
|
|
475
|
-
|
|
476
173
|
// Queues the task, or starts it with `start` holding its slot until its
|
|
477
174
|
// row exists, so two starts at once can't both take the last slot.
|
|
478
175
|
export async function startOrQueue<T>(
|
|
@@ -494,704 +191,6 @@ export async function startOrQueue<T>(
|
|
|
494
191
|
}
|
|
495
192
|
}
|
|
496
193
|
|
|
497
|
-
const label = queuedLabel;
|
|
498
|
-
|
|
499
|
-
function afterLabel(r: QueueRow): string | null {
|
|
500
|
-
if (r.after_task) {
|
|
501
|
-
const s = sessionOf(r.after_task);
|
|
502
|
-
const q = s ? null : row(r.after_task);
|
|
503
|
-
return s?.name ?? (q ? label(q) : r.after_task.slice(0, 8));
|
|
504
|
-
}
|
|
505
|
-
return r.after_any ? "any running task" : null;
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
// Every task still in line (and any that failed to start), in line order.
|
|
509
|
-
export function listQueue(): QueuedTaskView[] {
|
|
510
|
-
const rows = db
|
|
511
|
-
.prepare(
|
|
512
|
-
`SELECT q.*, p.name AS project_name, p.workspace_id AS workspace_id,
|
|
513
|
-
(SELECT COUNT(*) FROM bus_messages b WHERE b.to_id = q.id
|
|
514
|
-
AND b.held_queued = 1 AND b.delivered_at IS NULL AND b.read_at IS NULL) AS held_count
|
|
515
|
-
FROM task_queue q LEFT JOIN projects p ON p.id = q.project_id
|
|
516
|
-
WHERE q.status IN ('queued', 'starting', 'failed')
|
|
517
|
-
ORDER BY q.position`
|
|
518
|
-
)
|
|
519
|
-
.all() as (QueueRow & {
|
|
520
|
-
project_name: string | null;
|
|
521
|
-
workspace_id: string | null;
|
|
522
|
-
held_count: number;
|
|
523
|
-
})[];
|
|
524
|
-
const places = new Map<string, number>();
|
|
525
|
-
// The score order within each line, by priority and age alone: holds
|
|
526
|
-
// don't move a task's rank.
|
|
527
|
-
const now = Date.now();
|
|
528
|
-
const ranked = new Map<string, { rank: number; of: number }>();
|
|
529
|
-
const lines = new Map<string, typeof rows>();
|
|
530
|
-
for (const r of rows.filter((r) => r.status === "queued")) {
|
|
531
|
-
const key = r.workspace_id ?? `project:${r.project_id}`;
|
|
532
|
-
lines.set(key, [...(lines.get(key) ?? []), r]);
|
|
533
|
-
}
|
|
534
|
-
for (const line of lines.values()) {
|
|
535
|
-
const order = line
|
|
536
|
-
.map((r) => ({ r, score: scoreOf(r, now) }))
|
|
537
|
-
.sort((a, b) => b.score - a.score || a.r.position - b.r.position);
|
|
538
|
-
order.forEach(({ r }, i) =>
|
|
539
|
-
ranked.set(r.id, { rank: i + 1, of: order.length })
|
|
540
|
-
);
|
|
541
|
-
}
|
|
542
|
-
return rows.map((r) => {
|
|
543
|
-
const key = r.workspace_id ?? `project:${r.project_id}`;
|
|
544
|
-
const place = (places.get(key) ?? 0) + 1;
|
|
545
|
-
places.set(key, place);
|
|
546
|
-
const rank = ranked.get(r.id);
|
|
547
|
-
return {
|
|
548
|
-
id: r.id,
|
|
549
|
-
name: label(r),
|
|
550
|
-
prompt: r.prompt,
|
|
551
|
-
projectId: r.project_id,
|
|
552
|
-
projectName: r.project_name,
|
|
553
|
-
workspaceId: r.workspace_id,
|
|
554
|
-
status: r.status,
|
|
555
|
-
position: place,
|
|
556
|
-
after: afterLabel(r),
|
|
557
|
-
afterId: r.after_task,
|
|
558
|
-
note: r.note,
|
|
559
|
-
error: r.error,
|
|
560
|
-
startedBy: r.started_by,
|
|
561
|
-
createdAt: r.created_at,
|
|
562
|
-
priority: isPriority(r.priority) ? r.priority : DEFAULT_PRIORITY,
|
|
563
|
-
wait: r.wait_code,
|
|
564
|
-
rank: rank?.rank ?? place,
|
|
565
|
-
of: rank?.of ?? place,
|
|
566
|
-
held: r.held_count,
|
|
567
|
-
};
|
|
568
|
-
});
|
|
569
|
-
}
|
|
570
|
-
|
|
571
|
-
function scoreOf(r: QueueRow, now: number): number {
|
|
572
|
-
return scoreParts(
|
|
573
|
-
{
|
|
574
|
-
kind: "start",
|
|
575
|
-
priority: isPriority(r.priority) ? r.priority : DEFAULT_PRIORITY,
|
|
576
|
-
waitingSinceMs: sqliteMs(r.created_at),
|
|
577
|
-
},
|
|
578
|
-
now
|
|
579
|
-
).score;
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
// SQLite's datetime('now') is UTC without a zone.
|
|
583
|
-
const sqliteMs = (v: string) =>
|
|
584
|
-
Date.parse(v.includes("T") ? v : `${v.replace(" ", "T")}Z`) || 0;
|
|
585
|
-
|
|
586
|
-
export const queuedView = (id: string) =>
|
|
587
|
-
listQueue().find((q) => q.id === id) ?? null;
|
|
588
|
-
|
|
589
|
-
const requestNeeds = (
|
|
590
|
-
projectId: string,
|
|
591
|
-
workingDirectory: string,
|
|
592
|
-
prompt: string,
|
|
593
|
-
setting: Need[] | null
|
|
594
|
-
): Need[] =>
|
|
595
|
-
setting ??
|
|
596
|
-
needsOf({
|
|
597
|
-
projectId,
|
|
598
|
-
projectPath: expandHome(workingDirectory),
|
|
599
|
-
prompt,
|
|
600
|
-
}).needs;
|
|
601
|
-
|
|
602
|
-
// Whether the task needs something this machine lacks (a Mac).
|
|
603
|
-
function needsUnmetHere(
|
|
604
|
-
projectId: string,
|
|
605
|
-
prompt: string,
|
|
606
|
-
setting: Need[] | null
|
|
607
|
-
): boolean {
|
|
608
|
-
const project = getProject(projectId);
|
|
609
|
-
if (!project || isRemoteHost(project.host_id)) return false;
|
|
610
|
-
return !meetsNeeds(
|
|
611
|
-
requestNeeds(projectId, project.working_directory, prompt, setting),
|
|
612
|
-
machineCapabilities()
|
|
613
|
-
);
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
// Whether this machine's load rules out starting a task for this project now.
|
|
617
|
-
function loadHeldStart(
|
|
618
|
-
projectId: string,
|
|
619
|
-
prompt: string,
|
|
620
|
-
setting: Need[] | null
|
|
621
|
-
): boolean {
|
|
622
|
-
const project = getProject(projectId);
|
|
623
|
-
if (!project || isRemoteHost(project.host_id)) return false;
|
|
624
|
-
const needs = requestNeeds(
|
|
625
|
-
projectId,
|
|
626
|
-
project.working_directory,
|
|
627
|
-
prompt,
|
|
628
|
-
setting
|
|
629
|
-
);
|
|
630
|
-
const hold = machineHold(
|
|
631
|
-
{
|
|
632
|
-
heavy: heavyNeeds(needs),
|
|
633
|
-
needsUnmet: !meetsNeeds(needs, machineCapabilities()),
|
|
634
|
-
remote: false,
|
|
635
|
-
priority: DEFAULT_PRIORITY,
|
|
636
|
-
},
|
|
637
|
-
localView(),
|
|
638
|
-
Date.now()
|
|
639
|
-
);
|
|
640
|
-
return !!hold && (hold.code === "load_red" || hold.code === "heavy");
|
|
641
|
-
}
|
|
642
|
-
|
|
643
|
-
export type StartOutcome = "started" | "held" | "retry" | "failed";
|
|
644
|
-
|
|
645
|
-
// Tells the orchestrator that queued it that it started.
|
|
646
|
-
function announce(
|
|
647
|
-
r: Pick<QueueRow, "id" | "origin_workspace_id">,
|
|
648
|
-
session: Pick<Session, "name" | "branch_name">,
|
|
649
|
-
placed?: { text: string } | null
|
|
650
|
-
): void {
|
|
651
|
-
if (!r.origin_workspace_id) return;
|
|
652
|
-
queueEvent(
|
|
653
|
-
r.origin_workspace_id,
|
|
654
|
-
`queued-start:${r.id}`,
|
|
655
|
-
r.id,
|
|
656
|
-
placed
|
|
657
|
-
? `Queued task "${session.name}" (id ${r.id.slice(0, 8)}) ${placed.text}.`
|
|
658
|
-
: `Queued task "${session.name}" (id ${r.id.slice(0, 8)}) started on ${session.branch_name ?? "its branch"}.`
|
|
659
|
-
);
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
interface StartOpts {
|
|
663
|
-
force?: boolean;
|
|
664
|
-
// The finish an "after any" start takes (null: none left to take).
|
|
665
|
-
consume?: string | null;
|
|
666
|
-
// Start on this linked machine instead of the project's.
|
|
667
|
-
placement?: Placement | null;
|
|
668
|
-
heavy?: boolean;
|
|
669
|
-
// Pass this workspace's brakes even for a row no orchestrator queued.
|
|
670
|
-
brakeAs?: string;
|
|
671
|
-
}
|
|
672
|
-
|
|
673
|
-
async function startRow(
|
|
674
|
-
r: QueueRow,
|
|
675
|
-
opts: StartOpts = {}
|
|
676
|
-
): Promise<StartOutcome> {
|
|
677
|
-
const project = getProject(r.project_id);
|
|
678
|
-
const workspaceId = project?.workspace_id ?? null;
|
|
679
|
-
const origin = r.origin_workspace_id;
|
|
680
|
-
// The orchestrator starts only in its own workspace's projects on this
|
|
681
|
-
// machine; the project may have moved since it was queued.
|
|
682
|
-
if (
|
|
683
|
-
origin &&
|
|
684
|
-
(!project ||
|
|
685
|
-
workspaceId !== origin ||
|
|
686
|
-
isRemoteHost(r.host_id ?? project.host_id))
|
|
687
|
-
) {
|
|
688
|
-
patch(r.id, {
|
|
689
|
-
status: "failed",
|
|
690
|
-
error:
|
|
691
|
-
"Not started: its project left the orchestrator's workspace or this machine",
|
|
692
|
-
});
|
|
693
|
-
return "failed";
|
|
694
|
-
}
|
|
695
|
-
if (!opts.force) {
|
|
696
|
-
const held = autoHold(workspaceId, !!r.plan_only);
|
|
697
|
-
if (held) {
|
|
698
|
-
if (r.note !== held.text)
|
|
699
|
-
patch(r.id, { note: held.text, wait_code: held.code });
|
|
700
|
-
return "held";
|
|
701
|
-
}
|
|
702
|
-
// A start that came in during this tick's earlier starts may have taken
|
|
703
|
-
// the slot it planned on. The check and the claim are both synchronous.
|
|
704
|
-
const limit = limitOf(workspaceId);
|
|
705
|
-
if (
|
|
706
|
-
workspaceId &&
|
|
707
|
-
limit !== null &&
|
|
708
|
-
!r.plan_only &&
|
|
709
|
-
runningIn({ workspaceId, projectId: r.project_id }, true).length +
|
|
710
|
-
startingIn(workspaceId) >=
|
|
711
|
-
limit
|
|
712
|
-
)
|
|
713
|
-
return "held";
|
|
714
|
-
}
|
|
715
|
-
let claimed = 0;
|
|
716
|
-
try {
|
|
717
|
-
claimed = db
|
|
718
|
-
.prepare(
|
|
719
|
-
`UPDATE task_queue SET status = 'starting', note = NULL, wait_code = NULL,
|
|
720
|
-
after_any_used = COALESCE(after_any_used, @used)
|
|
721
|
-
WHERE id = @id AND status = 'queued'`
|
|
722
|
-
)
|
|
723
|
-
.run({ id: r.id, used: opts.consume ?? null }).changes;
|
|
724
|
-
} catch (error) {
|
|
725
|
-
// Another waiter took that finish between the plan and the claim.
|
|
726
|
-
if (String(error).includes("UNIQUE")) return "held";
|
|
727
|
-
throw error;
|
|
728
|
-
}
|
|
729
|
-
if (!claimed) return "held";
|
|
730
|
-
const placed = opts.placement ?? null;
|
|
731
|
-
if (!opts.force && !r.plan_only)
|
|
732
|
-
recordAutoStart(placed?.hostId ?? LOCAL, !!opts.heavy, Date.now());
|
|
733
|
-
// A linked machine's start never sees this machine's held messages: they
|
|
734
|
-
// go with the brief it is sent.
|
|
735
|
-
const remote = isRemoteHost(placed?.hostId ?? r.host_id ?? project?.host_id);
|
|
736
|
-
const remoteHeld = remote ? queuedHeldMessages(r.id) : [];
|
|
737
|
-
const task = {
|
|
738
|
-
id: r.id,
|
|
739
|
-
projectId: r.project_id,
|
|
740
|
-
prompt: r.prompt + queuedMessagesSection(remoteHeld),
|
|
741
|
-
name: r.name ?? undefined,
|
|
742
|
-
model: r.model_overrides
|
|
743
|
-
? (JSON.parse(r.model_overrides) as ModelOverride)
|
|
744
|
-
: (r.model ?? undefined),
|
|
745
|
-
plan: r.plan == null ? undefined : !!r.plan,
|
|
746
|
-
planReason: r.plan_reason,
|
|
747
|
-
planOnly: !!r.plan_only,
|
|
748
|
-
startedBy: r.started_by,
|
|
749
|
-
parentSessionId: r.parent_session_id,
|
|
750
|
-
startedByDevice: r.started_by_device,
|
|
751
|
-
startedVia: (r.started_via ??
|
|
752
|
-
(r.started_by ? "session" : "person")) as StartVia,
|
|
753
|
-
view: r.view ?? undefined,
|
|
754
|
-
agent: (r.agent_type as AgentType | null) ?? undefined,
|
|
755
|
-
baseBranch: r.base_branch ?? undefined,
|
|
756
|
-
hostId: placed?.hostId ?? r.host_id ?? undefined,
|
|
757
|
-
reclaim: true,
|
|
758
|
-
};
|
|
759
|
-
const placedText = placed
|
|
760
|
-
? `started on ${placed.name} · ${placed.why}`
|
|
761
|
-
: null;
|
|
762
|
-
try {
|
|
763
|
-
// The orchestrator's queued starts pass its brakes when they start,
|
|
764
|
-
// a person's "start now" included: it skips only the limit and `after`.
|
|
765
|
-
const brakeWs = origin ?? opts.brakeAs;
|
|
766
|
-
const session = brakeWs
|
|
767
|
-
? await braked(
|
|
768
|
-
brakeWs,
|
|
769
|
-
"task",
|
|
770
|
-
() => createTask(task),
|
|
771
|
-
(s) => s.id,
|
|
772
|
-
{ spendApproval: false }
|
|
773
|
-
)
|
|
774
|
-
: await createTask(task);
|
|
775
|
-
if (remoteHeld.length) markQueuedHeldDelivered(r.id);
|
|
776
|
-
patch(r.id, {
|
|
777
|
-
status: "started",
|
|
778
|
-
started_at: new Date().toISOString(),
|
|
779
|
-
error: null,
|
|
780
|
-
note: null,
|
|
781
|
-
});
|
|
782
|
-
stampStarted(r, placed);
|
|
783
|
-
announce(r, session, placedText ? { text: placedText } : null);
|
|
784
|
-
return "started";
|
|
785
|
-
} catch (error) {
|
|
786
|
-
if (error instanceof BrakeRefused) {
|
|
787
|
-
patch(r.id, { status: "queued", note: `Waiting: ${error.reason}` });
|
|
788
|
-
if (opts.force) throw error;
|
|
789
|
-
return "held";
|
|
790
|
-
}
|
|
791
|
-
const message = (
|
|
792
|
-
error instanceof Error ? error.message : String(error)
|
|
793
|
-
).slice(0, 300);
|
|
794
|
-
// Its row exists: the task is there, only its launch failed.
|
|
795
|
-
// The brakes never got to count it: count and announce it here.
|
|
796
|
-
const session = sessionOf(r.id);
|
|
797
|
-
if (session?.task_status) {
|
|
798
|
-
patch(r.id, { status: "started", started_at: new Date().toISOString() });
|
|
799
|
-
if (origin) recordStart(origin, "task", r.id);
|
|
800
|
-
announce(r, session);
|
|
801
|
-
return "started";
|
|
802
|
-
}
|
|
803
|
-
const attempts = r.attempts + 1;
|
|
804
|
-
const retry = attempts < START_ATTEMPTS && !opts.force;
|
|
805
|
-
patch(r.id, {
|
|
806
|
-
status: retry ? "queued" : "failed",
|
|
807
|
-
attempts,
|
|
808
|
-
error: retry
|
|
809
|
-
? `Start failed (try ${attempts} of ${START_ATTEMPTS}), trying again: ${message}`
|
|
810
|
-
: `Could not start: ${message}`,
|
|
811
|
-
});
|
|
812
|
-
return retry ? "retry" : "failed";
|
|
813
|
-
}
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
// Marks the task a queue row became: its priority, and where it was placed.
|
|
817
|
-
function stampStarted(r: QueueRow, placed: Placement | null): void {
|
|
818
|
-
const note = placed ? `Started on ${placed.name} · ${placed.why}` : null;
|
|
819
|
-
const priority =
|
|
820
|
-
r.priority && r.priority !== DEFAULT_PRIORITY ? r.priority : null;
|
|
821
|
-
if (!note && !priority) return;
|
|
822
|
-
db.prepare(
|
|
823
|
-
`UPDATE sessions SET priority = COALESCE(?, priority), placed_note = COALESCE(?, placed_note) WHERE id = ?`
|
|
824
|
-
).run(priority, note, r.id);
|
|
825
|
-
}
|
|
826
|
-
|
|
827
|
-
type QueueRowWithWorkspace = QueueRow & { workspace_id: string | null };
|
|
828
|
-
|
|
829
|
-
const groupKey = (workspaceId: string | null, projectId: string) =>
|
|
830
|
-
workspaceId ?? `project:${projectId}`;
|
|
831
|
-
|
|
832
|
-
function groupFor(
|
|
833
|
-
workspaceId: string | null,
|
|
834
|
-
projectId: string,
|
|
835
|
-
startingRows: number
|
|
836
|
-
): Group {
|
|
837
|
-
const scope = { workspaceId, projectId };
|
|
838
|
-
return {
|
|
839
|
-
inFlight:
|
|
840
|
-
runningIn(scope, true).length +
|
|
841
|
-
(workspaceId ? (reserved.get(workspaceId) ?? 0) : 0) +
|
|
842
|
-
startingRows,
|
|
843
|
-
limit: limitOf(workspaceId) ?? Number.POSITIVE_INFINITY,
|
|
844
|
-
};
|
|
845
|
-
}
|
|
846
|
-
|
|
847
|
-
// Finishes other waiters have taken, so one finish lets one waiter go.
|
|
848
|
-
const usedFinishes = (): string[] =>
|
|
849
|
-
(
|
|
850
|
-
db
|
|
851
|
-
.prepare(
|
|
852
|
-
`SELECT after_any_used AS id FROM task_queue WHERE after_any_used IS NOT NULL`
|
|
853
|
-
)
|
|
854
|
-
.all() as { id: string }[]
|
|
855
|
-
).map((x) => x.id);
|
|
856
|
-
|
|
857
|
-
function candidateOf(
|
|
858
|
-
r: QueueRowWithWorkspace,
|
|
859
|
-
used: string[]
|
|
860
|
-
): { c: Candidate; needs: Need[] } {
|
|
861
|
-
const project = getProject(r.project_id);
|
|
862
|
-
const hostId = r.host_id ?? project?.host_id;
|
|
863
|
-
const remote = isRemoteHost(hostId);
|
|
864
|
-
const needs = remote
|
|
865
|
-
? []
|
|
866
|
-
: (parseNeeds(r.machine_needs) ??
|
|
867
|
-
(project
|
|
868
|
-
? needsOf({
|
|
869
|
-
projectId: project.id,
|
|
870
|
-
projectPath: expandHome(project.working_directory),
|
|
871
|
-
prompt: r.prompt,
|
|
872
|
-
}).needs
|
|
873
|
-
: []));
|
|
874
|
-
let before: Hold | null = null;
|
|
875
|
-
if (r.after_task && !hasFinished(r.after_task))
|
|
876
|
-
before = {
|
|
877
|
-
code: "after",
|
|
878
|
-
text: `After ${afterLabel({ ...r, after_any: null }) ?? "its task"}`,
|
|
879
|
-
};
|
|
880
|
-
let any: Candidate["any"] = null;
|
|
881
|
-
if (r.after_any) {
|
|
882
|
-
const ids = JSON.parse(r.after_any) as string[];
|
|
883
|
-
any = {
|
|
884
|
-
ids,
|
|
885
|
-
finished: ids.filter((id) => !stillWorking(id)),
|
|
886
|
-
used: used.filter((id) => id !== r.after_any_used),
|
|
887
|
-
released: !!r.after_any_used,
|
|
888
|
-
label:
|
|
889
|
-
ids
|
|
890
|
-
.filter(stillWorking)
|
|
891
|
-
.map((id) => sessionOf(id)?.name ?? id.slice(0, 8))
|
|
892
|
-
.join(", ") || "its set",
|
|
893
|
-
};
|
|
894
|
-
}
|
|
895
|
-
return {
|
|
896
|
-
needs,
|
|
897
|
-
c: {
|
|
898
|
-
id: r.id,
|
|
899
|
-
kind: "start",
|
|
900
|
-
group: groupKey(r.workspace_id, r.project_id),
|
|
901
|
-
priority: isPriority(r.priority) ? r.priority : DEFAULT_PRIORITY,
|
|
902
|
-
position: r.position,
|
|
903
|
-
waitingSinceMs: sqliteMs(r.created_at),
|
|
904
|
-
heavy: heavyNeeds(needs),
|
|
905
|
-
needsUnmet: !remote && !meetsNeeds(needs, machineCapabilities()),
|
|
906
|
-
remote,
|
|
907
|
-
planOnly: !!r.plan_only,
|
|
908
|
-
before,
|
|
909
|
-
auto: autoHold(r.workspace_id, !!r.plan_only),
|
|
910
|
-
any,
|
|
911
|
-
},
|
|
912
|
-
};
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
// The linked machines a task could be placed on, with what they report.
|
|
916
|
-
async function peerOptions(
|
|
917
|
-
rows: QueueRowWithWorkspace[],
|
|
918
|
-
now: number
|
|
919
|
-
): Promise<Map<string, PeerOption[]>> {
|
|
920
|
-
const out = new Map<string, PeerOption[]>();
|
|
921
|
-
const hostIds = [...linkedHostIds()];
|
|
922
|
-
if (!hostIds.length || !rows.length) return out;
|
|
923
|
-
await refreshPeers(hostIds, now);
|
|
924
|
-
// Whether each machine can hold an orchestrator's task (an older AgentOS
|
|
925
|
-
// can't, and would fail its start); unreadable counts as no.
|
|
926
|
-
const holds = new Map<string, boolean>();
|
|
927
|
-
await Promise.all(
|
|
928
|
-
hostIds.map(async (hostId) => {
|
|
929
|
-
const link = hostLink(hostId);
|
|
930
|
-
const caps = link
|
|
931
|
-
? await hostTasks(link)
|
|
932
|
-
.then((t) => t.capabilities ?? [])
|
|
933
|
-
.catch(() => [])
|
|
934
|
-
: [];
|
|
935
|
-
holds.set(hostId, caps.includes("held-start"));
|
|
936
|
-
})
|
|
937
|
-
);
|
|
938
|
-
const perProject = new Map<string, Record<string, string | null>>();
|
|
939
|
-
for (const r of rows) {
|
|
940
|
-
if (out.has(r.project_id)) continue;
|
|
941
|
-
const project = getProject(r.project_id);
|
|
942
|
-
let can: Record<string, string | null> = {};
|
|
943
|
-
if (project) {
|
|
944
|
-
can =
|
|
945
|
-
perProject.get(project.id) ??
|
|
946
|
-
(await projectMachines(project).catch(() => ({})));
|
|
947
|
-
perProject.set(project.id, can);
|
|
948
|
-
}
|
|
949
|
-
out.set(
|
|
950
|
-
r.project_id,
|
|
951
|
-
hostIds.map((hostId) => {
|
|
952
|
-
const src = peerSource(hostId, now);
|
|
953
|
-
const reading = peerReading(hostId);
|
|
954
|
-
return {
|
|
955
|
-
hostId,
|
|
956
|
-
name: src.label,
|
|
957
|
-
view: viewOf(src, now),
|
|
958
|
-
capabilities: src.capabilities,
|
|
959
|
-
holds: holds.get(hostId) ?? false,
|
|
960
|
-
hasProject: can[hostId] === null,
|
|
961
|
-
loadPerCore: reading ? reading.load1 / reading.cores : 99,
|
|
962
|
-
};
|
|
963
|
-
})
|
|
964
|
-
);
|
|
965
|
-
}
|
|
966
|
-
return out;
|
|
967
|
-
}
|
|
968
|
-
|
|
969
|
-
// One look at the line: every queued task and every paused task that came
|
|
970
|
-
// due, scored, held or started. Machine readings come from the monitor and
|
|
971
|
-
// the peer poll; the clock is passed in.
|
|
972
|
-
export async function tickOnce(now = Date.now()): Promise<"stale" | void> {
|
|
973
|
-
const rows = db
|
|
974
|
-
.prepare(
|
|
975
|
-
`SELECT q.*, p.workspace_id AS workspace_id FROM task_queue q
|
|
976
|
-
JOIN projects p ON p.id = q.project_id
|
|
977
|
-
WHERE q.status IN ('queued', 'starting') ORDER BY q.position`
|
|
978
|
-
)
|
|
979
|
-
.all() as QueueRowWithWorkspace[];
|
|
980
|
-
const due = dueResumes();
|
|
981
|
-
// A deliberate resume starts past the slot limit; only a red machine holds
|
|
982
|
-
// it, and the task says so.
|
|
983
|
-
for (const d of due.filter((x) => x.resumeBy)) {
|
|
984
|
-
const s = db
|
|
985
|
-
.prepare(`SELECT * FROM sessions WHERE id = ?`)
|
|
986
|
-
.get(d.sessionId) as Session | undefined;
|
|
987
|
-
const why = s ? resumeHoldFor(s, d.resumeBy!) : null;
|
|
988
|
-
if (why) setPauseNote(d.sessionId, `held: ${why}`);
|
|
989
|
-
else await claimResume(d.sessionId, { reason: d.resumeBy!, force: true });
|
|
990
|
-
}
|
|
991
|
-
const resumes = due.filter((x) => !x.resumeBy);
|
|
992
|
-
if (!rows.length && !resumes.length) return;
|
|
993
|
-
const queued = rows.filter((r) => r.status === "queued");
|
|
994
|
-
const used = usedFinishes();
|
|
995
|
-
const built = new Map(
|
|
996
|
-
queued.map((r) => [r.id, candidateOf(r, used)] as const)
|
|
997
|
-
);
|
|
998
|
-
const candidates: Candidate[] = [...built.values()].map((b) => b.c);
|
|
999
|
-
for (const d of resumes) {
|
|
1000
|
-
const workspaceId = workspaceOf(d.projectId);
|
|
1001
|
-
candidates.push({
|
|
1002
|
-
id: `resume:${d.sessionId}`,
|
|
1003
|
-
kind: "resume",
|
|
1004
|
-
group: groupKey(workspaceId, d.projectId),
|
|
1005
|
-
priority: d.priority,
|
|
1006
|
-
position: -1_000_000 + d.pauseId,
|
|
1007
|
-
waitingSinceMs: d.dueAtMs,
|
|
1008
|
-
heavy: false,
|
|
1009
|
-
needsUnmet: false,
|
|
1010
|
-
remote: false,
|
|
1011
|
-
before: null,
|
|
1012
|
-
// A person's timed resume goes ahead of an orchestrator Pause.
|
|
1013
|
-
auto: d.byKind === "person" ? null : autoHold(workspaceId),
|
|
1014
|
-
any: null,
|
|
1015
|
-
});
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
// Placement: a start the home machine's load rules out goes to a free
|
|
1019
|
-
// linked machine, best score first, one per machine per look.
|
|
1020
|
-
const placements = new Map<string, Placement>();
|
|
1021
|
-
const home = localView(now);
|
|
1022
|
-
const toPlace = candidates.filter(
|
|
1023
|
-
(c) =>
|
|
1024
|
-
c.kind === "start" &&
|
|
1025
|
-
!c.remote &&
|
|
1026
|
-
!unplaceable(row(c.id)!) &&
|
|
1027
|
-
wantsPlacing(c, home, now)
|
|
1028
|
-
);
|
|
1029
|
-
if (toPlace.length) {
|
|
1030
|
-
const before = lineSignature();
|
|
1031
|
-
const peers = await peerOptions(
|
|
1032
|
-
toPlace.map((c) => rows.find((r) => r.id === c.id)!),
|
|
1033
|
-
now
|
|
1034
|
-
);
|
|
1035
|
-
// The line moved while the machines were asked: this look is stale, and
|
|
1036
|
-
// the one a change during it asks for sees the line as it is now.
|
|
1037
|
-
if (lineSignature() !== before) return "stale";
|
|
1038
|
-
const taken = new Set<string>();
|
|
1039
|
-
for (const c of [...toPlace].sort(
|
|
1040
|
-
(a, b) =>
|
|
1041
|
-
scoreParts(b, now).score - scoreParts(a, now).score ||
|
|
1042
|
-
a.position - b.position
|
|
1043
|
-
)) {
|
|
1044
|
-
const r = rows.find((x) => x.id === c.id)!;
|
|
1045
|
-
const options = (peers.get(r.project_id) ?? []).filter(
|
|
1046
|
-
(p) =>
|
|
1047
|
-
!taken.has(p.hostId) &&
|
|
1048
|
-
(!orchestratorOf(r.started_by) || p.holds !== false)
|
|
1049
|
-
);
|
|
1050
|
-
const spot = place(c, home, options, built.get(c.id)!.needs, now);
|
|
1051
|
-
if (!spot) continue;
|
|
1052
|
-
taken.add(spot.hostId);
|
|
1053
|
-
placements.set(c.id, spot);
|
|
1054
|
-
c.remote = true;
|
|
1055
|
-
}
|
|
1056
|
-
}
|
|
1057
|
-
|
|
1058
|
-
const groups = new Map<string, Group>();
|
|
1059
|
-
for (const c of candidates) {
|
|
1060
|
-
if (groups.has(c.group)) continue;
|
|
1061
|
-
const r = rows.find(
|
|
1062
|
-
(x) => groupKey(x.workspace_id, x.project_id) === c.group
|
|
1063
|
-
);
|
|
1064
|
-
const projectId =
|
|
1065
|
-
r?.project_id ??
|
|
1066
|
-
resumes.find((d) => `resume:${d.sessionId}` === c.id)!.projectId;
|
|
1067
|
-
const workspaceId = r?.workspace_id ?? workspaceOf(projectId);
|
|
1068
|
-
groups.set(
|
|
1069
|
-
c.group,
|
|
1070
|
-
groupFor(
|
|
1071
|
-
workspaceId,
|
|
1072
|
-
projectId,
|
|
1073
|
-
rows.filter(
|
|
1074
|
-
(x) =>
|
|
1075
|
-
x.status === "starting" &&
|
|
1076
|
-
groupKey(x.workspace_id, x.project_id) === c.group
|
|
1077
|
-
).length
|
|
1078
|
-
)
|
|
1079
|
-
);
|
|
1080
|
-
}
|
|
1081
|
-
|
|
1082
|
-
const plan = planStarts(candidates, groups, home, now);
|
|
1083
|
-
for (const r of queued)
|
|
1084
|
-
noteWaiting(r, withStayReason(r, plan.explain.get(r.id)));
|
|
1085
|
-
for (const d of resumes)
|
|
1086
|
-
setPauseNote(
|
|
1087
|
-
d.sessionId,
|
|
1088
|
-
resumeWaitNote(d, plan.explain.get(`resume:${d.sessionId}`))
|
|
1089
|
-
);
|
|
1090
|
-
|
|
1091
|
-
for (const id of plan.start) {
|
|
1092
|
-
if (id.startsWith("resume:")) {
|
|
1093
|
-
await claimResume(id.slice("resume:".length), { reason: "due" });
|
|
1094
|
-
continue;
|
|
1095
|
-
}
|
|
1096
|
-
const r = row(id);
|
|
1097
|
-
if (r?.status === "queued")
|
|
1098
|
-
await startRow(r, {
|
|
1099
|
-
consume: plan.consume.get(id) ?? null,
|
|
1100
|
-
placement: placements.get(id) ?? null,
|
|
1101
|
-
heavy: built.get(id)?.c.heavy,
|
|
1102
|
-
});
|
|
1103
|
-
}
|
|
1104
|
-
}
|
|
1105
|
-
|
|
1106
|
-
// What a timed resume still waiting for its turn says on its task.
|
|
1107
|
-
function resumeWaitNote(
|
|
1108
|
-
d: DueResume,
|
|
1109
|
-
explain: { code: WaitCode | null; text: string | null } | undefined
|
|
1110
|
-
): string | null {
|
|
1111
|
-
if (!explain) return null;
|
|
1112
|
-
if (explain.code && explain.code !== "slot" && explain.text)
|
|
1113
|
-
return `held: ${explain.text.replace(/^Waiting: /, "")}`;
|
|
1114
|
-
const slots = slotState(d.projectId);
|
|
1115
|
-
return slots.limit === null
|
|
1116
|
-
? "resuming: waiting for its turn"
|
|
1117
|
-
: `resuming: waiting for a slot, ${slots.running} of ${slots.limit} running`;
|
|
1118
|
-
}
|
|
1119
|
-
|
|
1120
|
-
// Writes why a row waits when that changed: its hold text and code. A bare
|
|
1121
|
-
// rank isn't stored (the view computes it), so the stream stays quiet.
|
|
1122
|
-
const MACHINE_HOLDS = new Set<WaitCode>([
|
|
1123
|
-
"load_red",
|
|
1124
|
-
"load_amber",
|
|
1125
|
-
"heavy",
|
|
1126
|
-
"needs",
|
|
1127
|
-
]);
|
|
1128
|
-
|
|
1129
|
-
// A hold on this machine's load says, for a task that can't go elsewhere,
|
|
1130
|
-
// why it waits here instead of starting on a linked machine.
|
|
1131
|
-
function withStayReason(
|
|
1132
|
-
r: QueueRow,
|
|
1133
|
-
explain: { code: WaitCode | null; text: string | null } | undefined
|
|
1134
|
-
) {
|
|
1135
|
-
if (!explain?.code || !explain.text || !MACHINE_HOLDS.has(explain.code))
|
|
1136
|
-
return explain;
|
|
1137
|
-
const why = unplaceable(r);
|
|
1138
|
-
return why ? { ...explain, text: `${explain.text} (${why})` } : explain;
|
|
1139
|
-
}
|
|
1140
|
-
|
|
1141
|
-
function noteWaiting(
|
|
1142
|
-
r: QueueRow,
|
|
1143
|
-
explain: { code: WaitCode | null; text: string | null } | undefined
|
|
1144
|
-
): void {
|
|
1145
|
-
if (!explain) return;
|
|
1146
|
-
const hold = explain.code && explain.code !== "slot" ? explain : null;
|
|
1147
|
-
if (hold) {
|
|
1148
|
-
if (r.note !== hold.text || r.wait_code !== hold.code)
|
|
1149
|
-
patch(r.id, { note: hold.text, wait_code: hold.code });
|
|
1150
|
-
return;
|
|
1151
|
-
}
|
|
1152
|
-
// A hold we wrote before is cleared; a brake's note isn't ours to clear.
|
|
1153
|
-
if (r.wait_code && r.wait_code !== "slot")
|
|
1154
|
-
patch(r.id, { note: null, wait_code: explain.code });
|
|
1155
|
-
else if (r.wait_code !== explain.code)
|
|
1156
|
-
patch(r.id, { wait_code: explain.code });
|
|
1157
|
-
}
|
|
1158
|
-
|
|
1159
|
-
function lineSignature(): string {
|
|
1160
|
-
const r = db
|
|
1161
|
-
.prepare(
|
|
1162
|
-
`SELECT COUNT(*) AS n, COALESCE(MAX(position), 0) AS p,
|
|
1163
|
-
COALESCE(SUM(priority = 'normal'), 0) AS pr,
|
|
1164
|
-
(SELECT COUNT(*) FROM sessions WHERE task_status IS NOT NULL
|
|
1165
|
-
AND task_status NOT IN ('merged', 'dropped')) AS running
|
|
1166
|
-
FROM task_queue WHERE status IN ('queued', 'starting')`
|
|
1167
|
-
)
|
|
1168
|
-
.get() as { n: number; p: number; pr: number; running: number };
|
|
1169
|
-
return `${r.n}:${r.p}:${r.pr}:${r.running}`;
|
|
1170
|
-
}
|
|
1171
|
-
|
|
1172
|
-
// One look at a time; a call during one shares it, and a look that went
|
|
1173
|
-
// stale (the line changed while it waited on a machine) is run again.
|
|
1174
|
-
const tickState = (g.__agentosQueueTick ??= { ticking: null });
|
|
1175
|
-
|
|
1176
|
-
export function tickQueue(): Promise<void> {
|
|
1177
|
-
tickState.ticking ??= (async () => {
|
|
1178
|
-
try {
|
|
1179
|
-
for (let i = 0; i < 3; i++) {
|
|
1180
|
-
if ((await tickOnce()) !== "stale") break;
|
|
1181
|
-
}
|
|
1182
|
-
} finally {
|
|
1183
|
-
tickState.ticking = null;
|
|
1184
|
-
}
|
|
1185
|
-
})();
|
|
1186
|
-
return tickState.ticking;
|
|
1187
|
-
}
|
|
1188
|
-
|
|
1189
|
-
export function tickQueueSoon(): void {
|
|
1190
|
-
void tickQueue().catch((error: unknown) =>
|
|
1191
|
-
console.error("[queue] tick:", error)
|
|
1192
|
-
);
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
194
|
// A person's "start now": past the limit, its `after` and (for a task a
|
|
1196
195
|
// person queued) Pause. The orchestrator's queued tasks still pass its
|
|
1197
196
|
// brakes; a refusal is thrown.
|
|
@@ -1202,18 +201,6 @@ export async function startQueuedNow(id: string): Promise<StartOutcome> {
|
|
|
1202
201
|
return startRow(r, { force: true });
|
|
1203
202
|
}
|
|
1204
203
|
|
|
1205
|
-
export interface ForcedStart {
|
|
1206
|
-
outcome: "started" | "held" | "failed";
|
|
1207
|
-
// Why it is held, or what failed; empty when it started.
|
|
1208
|
-
text: string;
|
|
1209
|
-
// The running-task limit was full when it started.
|
|
1210
|
-
wasFull: boolean;
|
|
1211
|
-
running: number;
|
|
1212
|
-
limit: number | null;
|
|
1213
|
-
// It skipped an `after`.
|
|
1214
|
-
skippedAfter: boolean;
|
|
1215
|
-
}
|
|
1216
|
-
|
|
1217
204
|
// The orchestrator's start now: past the limit and `after`, never past
|
|
1218
205
|
// Pause, its brakes or a machine need. A held task stays queued.
|
|
1219
206
|
export async function startQueuedForOrchestrator(
|
|
@@ -1263,36 +250,6 @@ export async function startQueuedForOrchestrator(
|
|
|
1263
250
|
}
|
|
1264
251
|
}
|
|
1265
252
|
|
|
1266
|
-
// A task still waiting in the line (or failed to start) by id, an id prefix
|
|
1267
|
-
// of 4+ characters or, when asked, its name; in one workspace.
|
|
1268
|
-
export function findQueued(
|
|
1269
|
-
workspaceId: string,
|
|
1270
|
-
ref: string,
|
|
1271
|
-
opts: { byName?: boolean } = {}
|
|
1272
|
-
): QueueRow | null {
|
|
1273
|
-
const want = ref.trim();
|
|
1274
|
-
const rows = db
|
|
1275
|
-
.prepare(
|
|
1276
|
-
`SELECT q.* FROM task_queue q JOIN projects p ON p.id = q.project_id
|
|
1277
|
-
WHERE p.workspace_id = ? AND q.status IN ('queued', 'failed')`
|
|
1278
|
-
)
|
|
1279
|
-
.all(workspaceId) as QueueRow[];
|
|
1280
|
-
const lower = want.toLowerCase();
|
|
1281
|
-
const hits = rows.filter(
|
|
1282
|
-
(r) =>
|
|
1283
|
-
r.id === want ||
|
|
1284
|
-
(want.length >= 4 && r.id.startsWith(want)) ||
|
|
1285
|
-
(opts.byName && label(r).toLowerCase() === lower)
|
|
1286
|
-
);
|
|
1287
|
-
if (hits.length > 1)
|
|
1288
|
-
throw new Error(
|
|
1289
|
-
`"${ref}" matches ${hits.length} queued tasks: ${hits.map((h) => `${label(h)} (${h.id.slice(0, 8)})`).join(", ")}`
|
|
1290
|
-
);
|
|
1291
|
-
return hits[0] ?? null;
|
|
1292
|
-
}
|
|
1293
|
-
|
|
1294
|
-
export { queuedLabel };
|
|
1295
|
-
|
|
1296
253
|
// Changes the priority of a queued task or a running one (they share an
|
|
1297
254
|
// id). Age is kept. Throws when no such task is waiting or running.
|
|
1298
255
|
export function setPriority(id: string, priority: Priority): void {
|