@cmgmyr/hive 1.11.9 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agentResume.js +222 -0
- package/dist/build-info.json +1 -1
- package/dist/cli.js +265 -2
- package/dist/codexHome.js +105 -1
- package/dist/context.js +1 -0
- package/dist/db.js +25 -0
- package/dist/help.js +39 -5
- package/dist/hook.js +61 -5
- package/dist/hooks.js +1 -0
- package/dist/leadMessageDelivery.js +77 -3
- package/dist/projectYml.js +30 -2
- package/dist/scheduler.js +165 -27
- package/dist/spawn.js +35 -39
- package/dist/todoCompletionGate.js +124 -0
- package/dist/tools/agents.js +449 -131
- package/dist/tools/todos.js +84 -9
- package/dist/usageLimit.js +470 -0
- package/dist/wakeSql.js +2 -0
- package/dist/workerRecovery.js +16 -0
- package/package.json +1 -1
package/dist/tools/todos.js
CHANGED
|
@@ -11,6 +11,8 @@ import { findUnsafeControlChar } from "../tmux.js";
|
|
|
11
11
|
import { SLUG_MAX_LEN, fallbackSlug } from "../slug.js";
|
|
12
12
|
import { idParam, limitParam, offsetParam, projectIdParam } from "./params.js";
|
|
13
13
|
import { NOT_COMPLETED_TODO_SQL, OPEN_BLOCKERS_SQL } from "../todoScope.js";
|
|
14
|
+
import { loadProjectYml } from "../projectYml.js";
|
|
15
|
+
import { assertTodoCompletionGateCurrent, assertValidTodoCompletionGate, runTodoCompletionGate, } from "../todoCompletionGate.js";
|
|
14
16
|
const priorityParam = z.enum(["high", "medium", "low"]);
|
|
15
17
|
const slugParam = z
|
|
16
18
|
.string()
|
|
@@ -25,6 +27,35 @@ export { SLUG_MAX_LEN, fallbackSlug };
|
|
|
25
27
|
const statusParam = z.enum(TODO_STATUSES);
|
|
26
28
|
const LIVE_DEPENDENTS_SQL = `SELECT t.id AS todo_id FROM todo_blockers b JOIN todos t ON t.id = b.todo_id
|
|
27
29
|
WHERE b.blocker_id = ? AND t.${NOT_COMPLETED_TODO_SQL} AND t.archived_at IS NULL`;
|
|
30
|
+
const TODO_ROW_FIELDS = [
|
|
31
|
+
"id", "project_id", "title", "body", "priority", "status", "locked_by", "tags", "slug",
|
|
32
|
+
"created_at", "completed_at", "archived_at", "updated_at",
|
|
33
|
+
];
|
|
34
|
+
function assertTodoSnapshot(actual, expected) {
|
|
35
|
+
if (TODO_ROW_FIELDS.some((field) => actual[field] !== expected[field])) {
|
|
36
|
+
throw new Error(`[todo_completion_gate:changed] todo ${expected.id}; exit unavailable`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
async function throughCompletionGate(projectId, snapshot, title, apply) {
|
|
40
|
+
const project = resolveProject(projectId);
|
|
41
|
+
const loaded = loadProjectYml(project.path);
|
|
42
|
+
const argv = loaded.config?.completion_gate;
|
|
43
|
+
const input = {
|
|
44
|
+
projectId,
|
|
45
|
+
projectPath: project.path,
|
|
46
|
+
todoId: snapshot.id,
|
|
47
|
+
title,
|
|
48
|
+
actorId: "",
|
|
49
|
+
argv: argv ?? [],
|
|
50
|
+
};
|
|
51
|
+
assertValidTodoCompletionGate(input, loaded.completionGateError);
|
|
52
|
+
if (!argv)
|
|
53
|
+
return apply();
|
|
54
|
+
const gatedInput = { ...input, actorId: currentActor(), argv };
|
|
55
|
+
await runTodoCompletionGate(gatedInput);
|
|
56
|
+
assertTodoCompletionGateCurrent(gatedInput, argv);
|
|
57
|
+
return apply(snapshot);
|
|
58
|
+
}
|
|
28
59
|
function transitiveBlockers(startId) {
|
|
29
60
|
const seen = new Set();
|
|
30
61
|
const stack = [startId];
|
|
@@ -73,8 +104,18 @@ const archiveTodo = db.transaction((projectId, todoId, archived) => {
|
|
|
73
104
|
updated_at = datetime('now') WHERE id = ?`).run(archived ? 1 : 0, todo.id);
|
|
74
105
|
return { todo_id: todo.id, archived };
|
|
75
106
|
});
|
|
76
|
-
const updateTodo = db.transaction((projectId, todoId, patch) => {
|
|
77
|
-
|
|
107
|
+
const updateTodo = db.transaction((projectId, todoId, patch, expected) => {
|
|
108
|
+
let todo;
|
|
109
|
+
try {
|
|
110
|
+
todo = getTodo(projectId, todoId);
|
|
111
|
+
}
|
|
112
|
+
catch (error) {
|
|
113
|
+
if (expected)
|
|
114
|
+
throw new Error(`[todo_completion_gate:changed] todo ${todoId}; exit unavailable`);
|
|
115
|
+
throw error;
|
|
116
|
+
}
|
|
117
|
+
if (expected)
|
|
118
|
+
assertTodoSnapshot(todo, expected);
|
|
78
119
|
if (todo.archived_at != null && todo.status === "completed" && patch.status && patch.status !== "completed") {
|
|
79
120
|
throw new Error(`Cannot change todo ${todo.id}'s status away from completed while archived. Unarchive it first.`);
|
|
80
121
|
}
|
|
@@ -93,8 +134,18 @@ const updateTodo = db.transaction((projectId, todoId, patch) => {
|
|
|
93
134
|
WHERE id = ?`).run(patch.title ?? null, patch.body ?? null, patch.priority ?? null, patch.status ?? null, patch.tags ? JSON.stringify(patch.tags) : null, patch.slug ?? null, patch.status ?? null, patch.status ?? null, todo.id);
|
|
94
135
|
return { todo_id: todo.id };
|
|
95
136
|
});
|
|
96
|
-
const completeTodo = db.transaction((projectId, todoId, completed) => {
|
|
97
|
-
|
|
137
|
+
const completeTodo = db.transaction((projectId, todoId, completed, expected) => {
|
|
138
|
+
let todo;
|
|
139
|
+
try {
|
|
140
|
+
todo = getTodo(projectId, todoId);
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
if (expected)
|
|
144
|
+
throw new Error(`[todo_completion_gate:changed] todo ${todoId}; exit unavailable`);
|
|
145
|
+
throw error;
|
|
146
|
+
}
|
|
147
|
+
if (expected)
|
|
148
|
+
assertTodoSnapshot(todo, expected);
|
|
98
149
|
if (!completed && todo.archived_at != null) {
|
|
99
150
|
throw new Error(`Cannot reopen archived todo ${todo.id}. Unarchive it first.`);
|
|
100
151
|
}
|
|
@@ -217,16 +268,28 @@ export function registerTodos(server) {
|
|
|
217
268
|
project_id: projectIdParam,
|
|
218
269
|
},
|
|
219
270
|
outputSchema: { project_id: idParam, todo_id: idParam },
|
|
220
|
-
}, (args) => run(() => {
|
|
271
|
+
}, (args) => run(async () => {
|
|
221
272
|
const projectId = effectiveProjectId(args.project_id);
|
|
222
|
-
const
|
|
273
|
+
const patch = {
|
|
223
274
|
title: args.title,
|
|
224
275
|
body: args.body,
|
|
225
276
|
priority: args.priority,
|
|
226
277
|
status: args.status,
|
|
227
278
|
tags: args.tags,
|
|
228
279
|
slug: args.slug,
|
|
229
|
-
}
|
|
280
|
+
};
|
|
281
|
+
if (args.status !== "completed") {
|
|
282
|
+
const result = updateTodo.immediate(projectId, args.todo_id, patch);
|
|
283
|
+
return { project_id: projectId, ...result };
|
|
284
|
+
}
|
|
285
|
+
const snapshot = getTodo(projectId, args.todo_id);
|
|
286
|
+
if (snapshot.status === "completed") {
|
|
287
|
+
const loaded = loadProjectYml(resolveProject(projectId).path);
|
|
288
|
+
const expected = loaded.completionGateError || loaded.config?.completion_gate ? snapshot : undefined;
|
|
289
|
+
const result = updateTodo.immediate(projectId, args.todo_id, patch, expected);
|
|
290
|
+
return { project_id: projectId, ...result };
|
|
291
|
+
}
|
|
292
|
+
const result = await throughCompletionGate(projectId, snapshot, args.title ?? snapshot.title, (expected) => updateTodo.immediate(projectId, args.todo_id, patch, expected));
|
|
230
293
|
return { project_id: projectId, ...result };
|
|
231
294
|
}));
|
|
232
295
|
server.registerTool("todo_archive", {
|
|
@@ -267,9 +330,21 @@ export function registerTodos(server) {
|
|
|
267
330
|
completed: z.boolean(),
|
|
268
331
|
newly_unblocked: z.array(idParam),
|
|
269
332
|
},
|
|
270
|
-
}, (args) => run(() => {
|
|
333
|
+
}, (args) => run(async () => {
|
|
271
334
|
const projectId = effectiveProjectId(args.project_id);
|
|
272
|
-
const
|
|
335
|
+
const completed = args.completed ?? true;
|
|
336
|
+
if (!completed) {
|
|
337
|
+
const result = completeTodo.immediate(projectId, args.todo_id, false);
|
|
338
|
+
return { project_id: projectId, ...result };
|
|
339
|
+
}
|
|
340
|
+
const snapshot = getTodo(projectId, args.todo_id);
|
|
341
|
+
if (snapshot.status === "completed") {
|
|
342
|
+
const loaded = loadProjectYml(resolveProject(projectId).path);
|
|
343
|
+
const expected = loaded.completionGateError || loaded.config?.completion_gate ? snapshot : undefined;
|
|
344
|
+
const result = completeTodo.immediate(projectId, args.todo_id, true, expected);
|
|
345
|
+
return { project_id: projectId, ...result };
|
|
346
|
+
}
|
|
347
|
+
const result = await throughCompletionGate(projectId, snapshot, snapshot.title, (expected) => completeTodo.immediate(projectId, args.todo_id, true, expected));
|
|
273
348
|
return { project_id: projectId, ...result };
|
|
274
349
|
}));
|
|
275
350
|
server.registerTool("todo_comment", {
|
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
import { closeSync, fstatSync, openSync, readSync, realpathSync } from "node:fs";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { isAbsolute, join, relative, sep } from "node:path";
|
|
4
|
+
import { db } from "./db.js";
|
|
5
|
+
import { codexHomeDir } from "./codexHome.js";
|
|
6
|
+
import { transcriptDir } from "./transcript.js";
|
|
7
|
+
import { harnessFor } from "./harnesses.js";
|
|
8
|
+
import { awaitingFirstPrompt } from "./firstPrompt.js";
|
|
9
|
+
import { readHandoffGate } from "./leadHandoffState.js";
|
|
10
|
+
import { PANE_LEASE_SETTLE_MS, releasePaneLease, tryAcquirePaneLease } from "./leaseStore.js";
|
|
11
|
+
import { capturePane, liveTargets, paneInCopyMode, rowOwnership, sendText, sleep } from "./tmux.js";
|
|
12
|
+
const UNKNOWN_RESET_MS = 6 * 60 * 60 * 1000;
|
|
13
|
+
const RESET_GRACE_MS = 60 * 1000;
|
|
14
|
+
const MAX_TAIL_BYTES = 256 * 1024;
|
|
15
|
+
function currentTranscriptOwner(row) {
|
|
16
|
+
if (row.kind !== "lead")
|
|
17
|
+
return row;
|
|
18
|
+
const turn = db.prepare("SELECT pane_pid, session_id, transcript_path FROM lead_turn_state WHERE agent_id = ?")
|
|
19
|
+
.get(row.id);
|
|
20
|
+
if (!turn || turn.pane_pid !== row.pane_pid || !turn.session_id)
|
|
21
|
+
return null;
|
|
22
|
+
return { ...row, session_id: turn.session_id, transcript_path: turn.transcript_path };
|
|
23
|
+
}
|
|
24
|
+
function object(value) {
|
|
25
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
|
|
26
|
+
}
|
|
27
|
+
function canonical(path) {
|
|
28
|
+
try {
|
|
29
|
+
return requireRealpath(path);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function requireRealpath(path) {
|
|
36
|
+
return realpathSync(path);
|
|
37
|
+
}
|
|
38
|
+
function isWithin(root, path) {
|
|
39
|
+
if (!isAbsolute(path) || path.includes("\0"))
|
|
40
|
+
return false;
|
|
41
|
+
const base = canonical(root);
|
|
42
|
+
const target = canonical(path);
|
|
43
|
+
if (base === null || target === null)
|
|
44
|
+
return false;
|
|
45
|
+
const rel = relative(base, target);
|
|
46
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
|
|
47
|
+
}
|
|
48
|
+
function tail(path) {
|
|
49
|
+
let fd;
|
|
50
|
+
try {
|
|
51
|
+
fd = openSync(path, "r");
|
|
52
|
+
const size = fstatSync(fd).size;
|
|
53
|
+
const start = Math.max(0, size - MAX_TAIL_BYTES);
|
|
54
|
+
const buffer = Buffer.alloc(size - start);
|
|
55
|
+
const bytes = readSync(fd, buffer, 0, buffer.length, start);
|
|
56
|
+
const lines = buffer.subarray(0, bytes).toString("utf8").split("\n");
|
|
57
|
+
if (start > 0)
|
|
58
|
+
lines.shift();
|
|
59
|
+
return lines.filter(Boolean);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return [];
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
if (fd !== undefined)
|
|
66
|
+
closeSync(fd);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function parsedLines(path) {
|
|
70
|
+
const records = [];
|
|
71
|
+
const lines = tail(path);
|
|
72
|
+
for (let i = 0; i < lines.length; i++) {
|
|
73
|
+
const line = lines[i];
|
|
74
|
+
try {
|
|
75
|
+
const value = object(JSON.parse(line));
|
|
76
|
+
if (value)
|
|
77
|
+
records.push(value);
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
if (i === lines.length - 1)
|
|
81
|
+
return [];
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return records;
|
|
85
|
+
}
|
|
86
|
+
function last(values) { return values.at(-1); }
|
|
87
|
+
function claudeEvidence(row, nowMs) {
|
|
88
|
+
const expected = transcriptDir(row.cwd);
|
|
89
|
+
const path = row.transcript_path || (row.session_id ? join(expected, `${row.session_id}.jsonl`) : "");
|
|
90
|
+
if (!path || !isWithin(expected, path))
|
|
91
|
+
return null;
|
|
92
|
+
const records = parsedLines(path);
|
|
93
|
+
const record = last(records.filter((candidate) => candidate.type === "assistant"));
|
|
94
|
+
if (!record)
|
|
95
|
+
return null;
|
|
96
|
+
if (typeof record.sessionId === "string" && record.sessionId !== row.session_id)
|
|
97
|
+
return null;
|
|
98
|
+
const quota = object(record.quotaLimits);
|
|
99
|
+
if (record.type === "assistant" && record.error === "rate_limit" && quota?.status === "rejected" && record.isApiErrorMessage === true) {
|
|
100
|
+
const reset = quota.resetsAt;
|
|
101
|
+
const resetMs = typeof reset === "number" && Number.isFinite(reset) && reset > 0 ? reset * 1000 : null;
|
|
102
|
+
return { signal: `claude:${row.session_id}:${createHash("sha256").update(JSON.stringify(record)).digest("hex")}`, resetMs, progressed: false };
|
|
103
|
+
}
|
|
104
|
+
if (record.error === "rate_limit" && record.apiError === "usage_limit_reached") {
|
|
105
|
+
const ownership = rowOwnership(row, liveTargets());
|
|
106
|
+
if (ownership !== "live")
|
|
107
|
+
return null;
|
|
108
|
+
let screen = "";
|
|
109
|
+
try {
|
|
110
|
+
screen = capturePane(row.tmux_target, 80);
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
if (!/usage limit reached\s*[·•]\s*limit resets/i.test(screen))
|
|
116
|
+
return null;
|
|
117
|
+
const content = object(record.message)?.content;
|
|
118
|
+
const text = Array.isArray(content)
|
|
119
|
+
? content.map((part) => typeof object(part)?.text === "string" ? object(part)?.text : "").join("\n")
|
|
120
|
+
: "";
|
|
121
|
+
const resetValue = quota?.resetsAt;
|
|
122
|
+
const resetMs = typeof resetValue === "number" && Number.isFinite(resetValue) && resetValue > 0
|
|
123
|
+
? resetValue * 1000
|
|
124
|
+
: parseResetText(text || screen, nowMs);
|
|
125
|
+
return { signal: `claude:${row.session_id}:${createHash("sha256").update(JSON.stringify(record)).digest("hex")}`, resetMs, progressed: false };
|
|
126
|
+
}
|
|
127
|
+
if (record.type === "assistant" && record.error !== "rate_limit" && record.isApiErrorMessage !== true) {
|
|
128
|
+
return { signal: "", resetMs: null, progressed: true };
|
|
129
|
+
}
|
|
130
|
+
return { signal: "", resetMs: null, progressed: false };
|
|
131
|
+
}
|
|
132
|
+
function codexEvidence(row) {
|
|
133
|
+
const root = join(codexHomeDir(row.codex_home), "sessions");
|
|
134
|
+
if (!row.codex_home || !row.transcript_path || !isWithin(root, row.transcript_path))
|
|
135
|
+
return null;
|
|
136
|
+
const name = row.transcript_path.split(/[\\/]/).at(-1) ?? "";
|
|
137
|
+
if (!/^rollout-.*\.jsonl$/.test(name) || (row.session_id && !name.includes(row.session_id)))
|
|
138
|
+
return null;
|
|
139
|
+
const ownership = rowOwnership(row, liveTargets());
|
|
140
|
+
if (ownership !== "live")
|
|
141
|
+
return null;
|
|
142
|
+
let screen = "";
|
|
143
|
+
try {
|
|
144
|
+
screen = capturePane(row.tmux_target, 80);
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
const token = last(parsedLines(row.transcript_path).filter((record) => {
|
|
150
|
+
const payload = object(record.payload);
|
|
151
|
+
return record.type === "event_msg" && payload?.type === "token_count" && object(payload.rate_limits) !== null;
|
|
152
|
+
}));
|
|
153
|
+
const rateLimits = object(object(token?.payload)?.rate_limits);
|
|
154
|
+
if (!rateLimits)
|
|
155
|
+
return null;
|
|
156
|
+
const primary = object(rateLimits.primary);
|
|
157
|
+
const secondary = object(rateLimits.secondary);
|
|
158
|
+
const reached = typeof rateLimits.rate_limit_reached_type === "string" && rateLimits.rate_limit_reached_type !== "";
|
|
159
|
+
const exhausted = [primary, secondary].some((window) => typeof window?.used_percent === "number" && window.used_percent >= 100);
|
|
160
|
+
if (!/hit your usage limit/i.test(screen)) {
|
|
161
|
+
return !reached && !exhausted ? { signal: "", resetMs: null, progressed: true } : { signal: "", resetMs: null, progressed: false };
|
|
162
|
+
}
|
|
163
|
+
if (!reached && !exhausted)
|
|
164
|
+
return null;
|
|
165
|
+
const resets = [primary, secondary].flatMap((window) => {
|
|
166
|
+
if (typeof window?.used_percent !== "number" || window.used_percent < 100)
|
|
167
|
+
return [];
|
|
168
|
+
const value = window?.resets_at;
|
|
169
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? [value * 1000] : [];
|
|
170
|
+
});
|
|
171
|
+
const resetMs = resets.length ? Math.max(...resets) : null;
|
|
172
|
+
return { signal: `codex:${row.session_id}:${createHash("sha256").update(JSON.stringify(token)).digest("hex")}`, resetMs, progressed: false };
|
|
173
|
+
}
|
|
174
|
+
function parseResetText(text, nowMs) {
|
|
175
|
+
const match = /resets\s+(?:(\w{3})\s+)?(?:(\w{3})\s+(\d{1,2})\s+at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)(?:\s*\(([^)]+)\))?/i.exec(text);
|
|
176
|
+
if (!match)
|
|
177
|
+
return null;
|
|
178
|
+
const [, weekday, month, day, hourRaw, minuteRaw, meridiem, zone] = match;
|
|
179
|
+
const hour = Number(hourRaw) % 12 + (/pm/i.test(meridiem) ? 12 : 0);
|
|
180
|
+
const minute = Number(minuteRaw ?? 0);
|
|
181
|
+
let formatter;
|
|
182
|
+
let current;
|
|
183
|
+
try {
|
|
184
|
+
formatter = new Intl.DateTimeFormat("en-US", {
|
|
185
|
+
...(zone ? { timeZone: zone } : {}),
|
|
186
|
+
year: "numeric", month: "numeric", day: "numeric", weekday: "short",
|
|
187
|
+
});
|
|
188
|
+
current = Object.fromEntries(formatter.formatToParts(new Date(nowMs)).map((part) => [part.type, part.value]));
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
let year = Number(current.year);
|
|
194
|
+
let monthIndex = Number(current.month) - 1;
|
|
195
|
+
let date = Number(current.day);
|
|
196
|
+
if (month && day) {
|
|
197
|
+
monthIndex = new Date(`${month} 1, 2000`).getMonth();
|
|
198
|
+
date = Number(day);
|
|
199
|
+
if (monthIndex < Number(current.month) - 1 || (monthIndex === Number(current.month) - 1 && date < Number(current.day)))
|
|
200
|
+
year++;
|
|
201
|
+
}
|
|
202
|
+
else if (weekday) {
|
|
203
|
+
const weekdays = ["sun", "mon", "tue", "wed", "thu", "fri", "sat"];
|
|
204
|
+
const target = weekdays.indexOf(weekday.toLowerCase().slice(0, 3));
|
|
205
|
+
if (target < 0)
|
|
206
|
+
return null;
|
|
207
|
+
const currentDay = weekdays.indexOf(current.weekday.toLowerCase().slice(0, 3));
|
|
208
|
+
date += (target - currentDay + 7) % 7;
|
|
209
|
+
}
|
|
210
|
+
const wallTime = Date.UTC(year, monthIndex, date, hour, minute);
|
|
211
|
+
if (zone) {
|
|
212
|
+
try {
|
|
213
|
+
const clock = new Intl.DateTimeFormat("en-US", { timeZone: zone, year: "numeric", month: "numeric", day: "numeric", hour: "numeric", minute: "numeric", hourCycle: "h23" });
|
|
214
|
+
let candidate = wallTime;
|
|
215
|
+
for (let i = 0; i < 4; i++) {
|
|
216
|
+
const parts = Object.fromEntries(clock.formatToParts(new Date(candidate)).map((part) => [part.type, part.value]));
|
|
217
|
+
const represented = Date.UTC(Number(parts.year), Number(parts.month) - 1, Number(parts.day), Number(parts.hour), Number(parts.minute));
|
|
218
|
+
const correction = wallTime - represented;
|
|
219
|
+
if (correction === 0)
|
|
220
|
+
break;
|
|
221
|
+
candidate += correction;
|
|
222
|
+
}
|
|
223
|
+
const check = Object.fromEntries(clock.formatToParts(new Date(candidate)).map((part) => [part.type, part.value]));
|
|
224
|
+
if (Number(check.year) !== year || Number(check.month) !== monthIndex + 1 || Number(check.day) !== date ||
|
|
225
|
+
Number(check.hour) !== hour || Number(check.minute) !== minute)
|
|
226
|
+
return null;
|
|
227
|
+
if (!month && !weekday && candidate <= nowMs)
|
|
228
|
+
candidate += 24 * 60 * 60 * 1000;
|
|
229
|
+
return candidate;
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
const result = new Date(year, monthIndex, date, hour, minute).getTime();
|
|
236
|
+
if (!month && !weekday && result <= nowMs)
|
|
237
|
+
return result + 24 * 60 * 60 * 1000;
|
|
238
|
+
return result;
|
|
239
|
+
}
|
|
240
|
+
function timestamp(value) {
|
|
241
|
+
if (!value)
|
|
242
|
+
return null;
|
|
243
|
+
const normalized = /(?:Z|[+-]\d{2}:?\d{2})$/i.test(value) ? value : `${value.replace(" ", "T")}Z`;
|
|
244
|
+
const parsed = Date.parse(normalized);
|
|
245
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
246
|
+
}
|
|
247
|
+
function writeEpisode(row, harness, signal, resetMs, nowMs) {
|
|
248
|
+
const existing = db.prepare("SELECT * FROM agent_usage_limits WHERE agent_id = ?").get(row.id);
|
|
249
|
+
const identityMatches = existing?.project_id === row.project_id && existing.pane_pid === row.pane_pid && existing.session_id === row.session_id;
|
|
250
|
+
if (identityMatches && existing?.signal_key === signal)
|
|
251
|
+
return;
|
|
252
|
+
const expiresMs = resetMs === null ? nowMs + UNKNOWN_RESET_MS : resetMs + RESET_GRACE_MS;
|
|
253
|
+
db.prepare(`INSERT INTO agent_usage_limits
|
|
254
|
+
(agent_id, project_id, harness, pane_pid, session_id, signal_key, observed_at, reset_at, expires_at, released_at, continue_claimed_at)
|
|
255
|
+
VALUES (?, ?, ?, ?, ?, ?, datetime(?, 'unixepoch'), ?, datetime(?, 'unixepoch'), NULL, NULL)
|
|
256
|
+
ON CONFLICT(agent_id) DO UPDATE SET project_id=excluded.project_id, harness=excluded.harness,
|
|
257
|
+
pane_pid=excluded.pane_pid, session_id=excluded.session_id, signal_key=excluded.signal_key,
|
|
258
|
+
observed_at=excluded.observed_at, reset_at=excluded.reset_at, expires_at=excluded.expires_at,
|
|
259
|
+
released_at=NULL, continue_claimed_at=NULL
|
|
260
|
+
WHERE EXISTS (
|
|
261
|
+
SELECT 1 FROM agents a
|
|
262
|
+
WHERE a.id = excluded.agent_id AND a.project_id = excluded.project_id
|
|
263
|
+
AND a.pane_pid = excluded.pane_pid AND a.status = 'running'
|
|
264
|
+
AND ((a.kind = 'agent' AND a.session_id = excluded.session_id) OR (a.kind = 'lead' AND EXISTS (
|
|
265
|
+
SELECT 1 FROM lead_turn_state t
|
|
266
|
+
WHERE t.agent_id = a.id AND t.pane_pid = a.pane_pid AND t.session_id = excluded.session_id
|
|
267
|
+
)))
|
|
268
|
+
)`)
|
|
269
|
+
.run(row.id, row.project_id, harness, row.pane_pid, row.session_id, signal, nowMs / 1000, resetMs === null ? null : new Date(resetMs).toISOString(), expiresMs / 1000);
|
|
270
|
+
}
|
|
271
|
+
export function observeClaudeStopFailure(agentId, payload, nowMs) {
|
|
272
|
+
const row = db.prepare("SELECT * FROM agents WHERE id = ? AND status = 'running'").get(agentId);
|
|
273
|
+
const input = object(payload);
|
|
274
|
+
if (!row || !["agent", "lead"].includes(row.kind) || harnessFor(row.command).name !== "claude" || input?.error !== "rate_limit")
|
|
275
|
+
return;
|
|
276
|
+
const owner = currentTranscriptOwner(row);
|
|
277
|
+
if (!owner)
|
|
278
|
+
return;
|
|
279
|
+
const transcript = typeof input.transcript_path === "string" ? input.transcript_path : owner.transcript_path;
|
|
280
|
+
const expected = transcriptDir(owner.cwd);
|
|
281
|
+
if (!transcript || !isWithin(expected, transcript))
|
|
282
|
+
return;
|
|
283
|
+
const sessionId = typeof input.session_id === "string" ? input.session_id : owner.session_id;
|
|
284
|
+
if (!sessionId || (owner.session_id && sessionId !== owner.session_id))
|
|
285
|
+
return;
|
|
286
|
+
const text = typeof input.last_assistant_message === "string" ? input.last_assistant_message : "";
|
|
287
|
+
const records = parsedLines(transcript);
|
|
288
|
+
const latestAssistant = last(records.filter((record) => record.type === "assistant"));
|
|
289
|
+
if (typeof latestAssistant?.sessionId === "string" && latestAssistant.sessionId !== sessionId)
|
|
290
|
+
return;
|
|
291
|
+
const rejected = latestAssistant?.error === "rate_limit" && latestAssistant.isApiErrorMessage === true &&
|
|
292
|
+
object(latestAssistant.quotaLimits)?.status === "rejected" ? latestAssistant : undefined;
|
|
293
|
+
if (!rejected && !/(?:session|weekly|opus|sonnet|usage) limit|limit resets/i.test(text))
|
|
294
|
+
return;
|
|
295
|
+
const reset = object(rejected?.quotaLimits)?.resetsAt;
|
|
296
|
+
const resetMs = typeof reset === "number" && Number.isFinite(reset) && reset > 0
|
|
297
|
+
? reset * 1000
|
|
298
|
+
: parseResetText(text, nowMs);
|
|
299
|
+
const signal = `claude:${sessionId}:${createHash("sha256").update(JSON.stringify(rejected ?? text)).digest("hex")}`;
|
|
300
|
+
writeEpisode(owner, "claude", signal, resetMs, nowMs);
|
|
301
|
+
}
|
|
302
|
+
export function observeUsageLimit(agentId, nowMs) {
|
|
303
|
+
const row = db.prepare("SELECT * FROM agents WHERE id = ? AND status = 'running'").get(agentId);
|
|
304
|
+
if (!row || !["agent", "lead"].includes(row.kind))
|
|
305
|
+
return;
|
|
306
|
+
const owner = currentTranscriptOwner(row);
|
|
307
|
+
if (!owner)
|
|
308
|
+
return;
|
|
309
|
+
const harness = harnessFor(row.command).name;
|
|
310
|
+
const evidence = harness === "claude" ? claudeEvidence(owner, nowMs) : harness === "codex" ? codexEvidence(owner) : null;
|
|
311
|
+
const existing = db.prepare("SELECT * FROM agent_usage_limits WHERE agent_id = ?").get(agentId);
|
|
312
|
+
if (existing && (existing.project_id !== owner.project_id || existing.pane_pid !== owner.pane_pid || existing.session_id !== owner.session_id)) {
|
|
313
|
+
db.prepare(`UPDATE agent_usage_limits SET released_at = datetime(?, 'unixepoch')
|
|
314
|
+
WHERE agent_id = ? AND project_id = ? AND pane_pid = ? AND session_id = ? AND released_at IS NULL
|
|
315
|
+
AND EXISTS (
|
|
316
|
+
SELECT 1 FROM agents a
|
|
317
|
+
WHERE a.id = ? AND a.project_id = ? AND a.pane_pid = ? AND a.status = 'running'
|
|
318
|
+
AND ((a.kind = 'agent' AND a.session_id = ?) OR (a.kind = 'lead' AND EXISTS (
|
|
319
|
+
SELECT 1 FROM lead_turn_state t
|
|
320
|
+
WHERE t.agent_id = a.id AND t.pane_pid = a.pane_pid AND t.session_id = ?
|
|
321
|
+
)))
|
|
322
|
+
)`)
|
|
323
|
+
.run(nowMs / 1000, agentId, existing.project_id, existing.pane_pid, existing.session_id, owner.id, owner.project_id, owner.pane_pid, owner.session_id, owner.session_id);
|
|
324
|
+
}
|
|
325
|
+
if (evidence?.progressed && existing?.released_at === null && existing.project_id === owner.project_id &&
|
|
326
|
+
existing.pane_pid === owner.pane_pid && existing.session_id === owner.session_id) {
|
|
327
|
+
db.prepare(`UPDATE agent_usage_limits SET released_at = datetime(?, 'unixepoch')
|
|
328
|
+
WHERE agent_id = ? AND project_id = ? AND pane_pid = ? AND session_id = ? AND released_at IS NULL
|
|
329
|
+
AND EXISTS (
|
|
330
|
+
SELECT 1 FROM agents a
|
|
331
|
+
WHERE a.id = ? AND a.project_id = ? AND a.pane_pid = ? AND a.status = 'running'
|
|
332
|
+
AND ((a.kind = 'agent' AND a.session_id = ?) OR (a.kind = 'lead' AND EXISTS (
|
|
333
|
+
SELECT 1 FROM lead_turn_state t
|
|
334
|
+
WHERE t.agent_id = a.id AND t.pane_pid = a.pane_pid AND t.session_id = ?
|
|
335
|
+
)))
|
|
336
|
+
)`)
|
|
337
|
+
.run(nowMs / 1000, agentId, existing.project_id, existing.pane_pid, existing.session_id, owner.id, owner.project_id, owner.pane_pid, owner.session_id, owner.session_id);
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
if (!evidence?.signal)
|
|
341
|
+
return;
|
|
342
|
+
writeEpisode(owner, harness, evidence.signal, evidence.resetMs, nowMs);
|
|
343
|
+
}
|
|
344
|
+
export function usageLimitGate(agentId, nowMs) {
|
|
345
|
+
const episode = db.prepare("SELECT expires_at, released_at FROM agent_usage_limits WHERE agent_id = ?")
|
|
346
|
+
.get(agentId);
|
|
347
|
+
const until = episode?.expires_at ?? null;
|
|
348
|
+
return { held: episode?.released_at == null && (timestamp(until) ?? 0) > nowMs, until };
|
|
349
|
+
}
|
|
350
|
+
export function limitedHarness(projectId, harness, nowMs) {
|
|
351
|
+
const row = db.prepare(`SELECT u.expires_at FROM agent_usage_limits u JOIN agents a ON a.id = u.agent_id
|
|
352
|
+
WHERE u.project_id = ? AND u.harness = ? AND u.released_at IS NULL AND a.status = 'running'
|
|
353
|
+
ORDER BY u.expires_at, u.agent_id LIMIT 1`).get(projectId, harness);
|
|
354
|
+
return row && (timestamp(row.expires_at) ?? 0) > nowMs ? { until: row.expires_at } : null;
|
|
355
|
+
}
|
|
356
|
+
export function observeProjectUsageLimits(projectId, harness, nowMs) {
|
|
357
|
+
const rows = db.prepare(`SELECT id FROM agents WHERE project_id = ? AND status = 'running' AND kind IN ('agent', 'lead')`)
|
|
358
|
+
.all(projectId);
|
|
359
|
+
for (const row of rows) {
|
|
360
|
+
const current = db.prepare("SELECT command FROM agents WHERE id = ?").get(row.id);
|
|
361
|
+
if (current && harnessFor(current.command).name === harness)
|
|
362
|
+
observeUsageLimit(row.id, nowMs);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
export async function continueLimitedCodexWorkers(nowMs) {
|
|
366
|
+
const workers = db.prepare("SELECT id, command FROM agents WHERE status = 'running' AND kind = 'agent'")
|
|
367
|
+
.all();
|
|
368
|
+
for (const worker of workers) {
|
|
369
|
+
if (harnessFor(worker.command).name === "codex")
|
|
370
|
+
observeUsageLimit(worker.id, nowMs);
|
|
371
|
+
}
|
|
372
|
+
const due = db.prepare(`SELECT u.agent_id, u.signal_key FROM agent_usage_limits u
|
|
373
|
+
JOIN agents a ON a.id = u.agent_id
|
|
374
|
+
WHERE u.harness = 'codex' AND u.reset_at IS NOT NULL AND julianday(u.reset_at) <= julianday(datetime(?, 'unixepoch'))
|
|
375
|
+
AND u.continue_claimed_at IS NULL AND u.released_at IS NULL AND a.kind = 'agent' AND a.status = 'running'
|
|
376
|
+
ORDER BY u.expires_at, u.agent_id`).all(nowMs / 1000);
|
|
377
|
+
for (const candidate of due) {
|
|
378
|
+
try {
|
|
379
|
+
const row = db.prepare("SELECT * FROM agents WHERE id = ? AND kind = 'agent' AND status = 'running'")
|
|
380
|
+
.get(candidate.agent_id);
|
|
381
|
+
if (!row || harnessFor(row.command).name !== "codex" || awaitingFirstPrompt(row))
|
|
382
|
+
continue;
|
|
383
|
+
if (readHandoffGate(row.id)?.holdAutomation)
|
|
384
|
+
continue;
|
|
385
|
+
const episode = db.prepare("SELECT * FROM agent_usage_limits WHERE agent_id = ? AND signal_key = ?")
|
|
386
|
+
.get(row.id, candidate.signal_key);
|
|
387
|
+
if (!episode || episode.project_id !== row.project_id || episode.pane_pid !== row.pane_pid ||
|
|
388
|
+
episode.session_id !== row.session_id || episode.reset_at === null || episode.released_at !== null ||
|
|
389
|
+
(timestamp(episode.reset_at) ?? Infinity) > nowMs || episode.continue_claimed_at !== null)
|
|
390
|
+
continue;
|
|
391
|
+
const snapshot = liveTargets();
|
|
392
|
+
if (rowOwnership(row, snapshot) !== "live")
|
|
393
|
+
continue;
|
|
394
|
+
const classifier = harnessFor(row.command).paneClassifier;
|
|
395
|
+
if (!classifier || !limitStillVisible(row.tmux_target, classifier, true))
|
|
396
|
+
continue;
|
|
397
|
+
const lease = tryAcquirePaneLease(row.project_id, row.tmux_socket, row.tmux_target, {
|
|
398
|
+
id: `hive:usage-limit:${row.id}`,
|
|
399
|
+
name: "hive usage-limit continuation",
|
|
400
|
+
kind: "scheduler",
|
|
401
|
+
});
|
|
402
|
+
if (lease === null)
|
|
403
|
+
continue;
|
|
404
|
+
try {
|
|
405
|
+
const current = db.prepare("SELECT * FROM agents WHERE id = ? AND kind = 'agent' AND status = 'running'")
|
|
406
|
+
.get(row.id);
|
|
407
|
+
const currentEpisode = db.prepare("SELECT * FROM agent_usage_limits WHERE agent_id = ? AND signal_key = ?")
|
|
408
|
+
.get(row.id, candidate.signal_key);
|
|
409
|
+
if (!current || current.tmux_target !== row.tmux_target || current.pane_pid !== row.pane_pid ||
|
|
410
|
+
current.tmux_socket !== row.tmux_socket || current.session_id !== row.session_id ||
|
|
411
|
+
awaitingFirstPrompt(current) || !currentEpisode || currentEpisode.continue_claimed_at !== null ||
|
|
412
|
+
currentEpisode.released_at !== null || currentEpisode.reset_at === null ||
|
|
413
|
+
(timestamp(currentEpisode.reset_at) ?? Infinity) > nowMs || readHandoffGate(row.id)?.holdAutomation ||
|
|
414
|
+
rowOwnership(current, liveTargets()) !== "live" || !limitStillVisible(row.tmux_target, classifier, true))
|
|
415
|
+
continue;
|
|
416
|
+
const claim = db.prepare(`UPDATE agent_usage_limits SET continue_claimed_at = datetime(?, 'unixepoch')
|
|
417
|
+
WHERE agent_id = ? AND signal_key = ? AND harness = 'codex'
|
|
418
|
+
AND julianday(reset_at) <= julianday(datetime(?, 'unixepoch'))
|
|
419
|
+
AND continue_claimed_at IS NULL AND released_at IS NULL`)
|
|
420
|
+
.run(nowMs / 1000, row.id, candidate.signal_key, nowMs / 1000);
|
|
421
|
+
if (claim.changes !== 1)
|
|
422
|
+
continue;
|
|
423
|
+
let pasted = false;
|
|
424
|
+
await sendText(row.tmux_target, "continue", true, () => { pasted = true; }, undefined, () => {
|
|
425
|
+
const latest = db.prepare("SELECT * FROM agents WHERE id = ? AND kind = 'agent' AND status = 'running'")
|
|
426
|
+
.get(row.id);
|
|
427
|
+
if (!latest || latest.tmux_target !== row.tmux_target || latest.pane_pid !== row.pane_pid ||
|
|
428
|
+
latest.tmux_socket !== row.tmux_socket || latest.session_id !== row.session_id ||
|
|
429
|
+
rowOwnership(latest, liveTargets()) !== "live" || awaitingFirstPrompt(latest) ||
|
|
430
|
+
readHandoffGate(row.id)?.holdAutomation ||
|
|
431
|
+
!limitStillVisible(row.tmux_target, classifier, !pasted, pasted ? "continue" : undefined)) {
|
|
432
|
+
throw new Error("Codex usage-limit continuation lost its live-pane, assignment, or visible-limit guard");
|
|
433
|
+
}
|
|
434
|
+
});
|
|
435
|
+
await sleep(PANE_LEASE_SETTLE_MS);
|
|
436
|
+
}
|
|
437
|
+
finally {
|
|
438
|
+
releasePaneLease(row.project_id, row.tmux_socket, row.tmux_target, lease);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
catch (error) {
|
|
442
|
+
console.error(`[usage-limit] Codex continuation for agent ${candidate.agent_id} was claimed and will not be retried: ${error instanceof Error ? error.message : String(error)}`);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
function limitStillVisible(target, classifier, requireEmpty, ownSubmittedText) {
|
|
447
|
+
if (!classifier)
|
|
448
|
+
return false;
|
|
449
|
+
if (paneInCopyMode(target) !== false)
|
|
450
|
+
return false;
|
|
451
|
+
const choice = classifier.choiceCheck(target).awaitingChoice;
|
|
452
|
+
if (choice !== false)
|
|
453
|
+
return false;
|
|
454
|
+
if (requireEmpty) {
|
|
455
|
+
const box = classifier.inputBoxState(target);
|
|
456
|
+
if (box?.state !== "empty")
|
|
457
|
+
return false;
|
|
458
|
+
}
|
|
459
|
+
else if (ownSubmittedText !== undefined) {
|
|
460
|
+
const box = classifier.inputBoxState(target);
|
|
461
|
+
if (box?.state !== "empty" && (box?.state !== "pending" || box.text !== ownSubmittedText))
|
|
462
|
+
return false;
|
|
463
|
+
}
|
|
464
|
+
try {
|
|
465
|
+
return /hit your usage limit/i.test(capturePane(target, 80));
|
|
466
|
+
}
|
|
467
|
+
catch {
|
|
468
|
+
return false;
|
|
469
|
+
}
|
|
470
|
+
}
|
package/dist/wakeSql.js
CHANGED
|
@@ -116,6 +116,7 @@ export const HELD_REASON_UNCLASSIFIABLE_PANE = `${HELD_REASON_UNCLASSIFIABLE_PAN
|
|
|
116
116
|
export function isUnclassifiablePaneHold(heldReason) {
|
|
117
117
|
return heldReason != null && heldReason.startsWith(HELD_REASON_UNCLASSIFIABLE_PANE_PREFIX);
|
|
118
118
|
}
|
|
119
|
+
export const HELD_REASON_PANE_CAPTURE_UNKNOWN = "the pane could not be read, so its dialog state is unknown; wake held rather than typed blind";
|
|
119
120
|
export const HELD_REASON_COPY_MODE = "the pane is in tmux copy mode; tmux clears its bracketed-paste flag there, so the body would arrive " +
|
|
120
121
|
"with no paste markers (losing everything before the last 1022-byte write) and the Enter would be " +
|
|
121
122
|
"eaten by the mode instead of submitting - both silently, with tmux reporting success. It delivers " +
|
|
@@ -133,3 +134,4 @@ export const HELD_REASON_CONVERSATION = "a human talked to this lead more recent
|
|
|
133
134
|
"ceiling is reached, whichever comes first";
|
|
134
135
|
export const HELD_REASON_HANDOFF = "the lead is handing off to a fresh session; held through the wind-down and the successor's boot, and delivered to " +
|
|
135
136
|
"the lead row once the successor completes its first turn, or when the handoff fails or releases its hold";
|
|
137
|
+
export const HELD_REASON_USAGE_LIMIT = "the target harness is stopped by a usage limit; held until ";
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { db } from "./db.js";
|
|
2
|
+
export const WORKER_RECOVERY_RETENTION_SECONDS = 7 * 24 * 60 * 60;
|
|
3
|
+
export function markWorkerRetirementIntent(agentId, expected) {
|
|
4
|
+
return (db
|
|
5
|
+
.prepare(`UPDATE agents SET recovery_state = 'excluded'
|
|
6
|
+
WHERE id = ? AND kind = 'agent' AND status IN ('running', 'closed')
|
|
7
|
+
AND tmux_target = ? AND tmux_socket = ? AND pane_pid = ?`)
|
|
8
|
+
.run(agentId, expected.tmux_target, expected.tmux_socket, expected.pane_pid).changes > 0);
|
|
9
|
+
}
|
|
10
|
+
export function markWorkerLost(agentId, expected) {
|
|
11
|
+
return (db
|
|
12
|
+
.prepare(`UPDATE agents SET status = 'closed', closed_at = datetime('now'), recovery_state = 'pending'
|
|
13
|
+
WHERE id = ? AND kind = 'agent' AND status = 'running' AND recovery_state != 'excluded'
|
|
14
|
+
AND tmux_target = ? AND tmux_socket = ? AND pane_pid = ?`)
|
|
15
|
+
.run(agentId, expected.tmux_target, expected.tmux_socket, expected.pane_pid).changes > 0);
|
|
16
|
+
}
|
package/package.json
CHANGED